Files
nexus/sreweekly/articles/511/05-reliability-engineering-for-air-gapped-systems.html
2026-09-12 17:23:01 +08:00

490 lines
234 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en" dir="ltr">
<head>
<meta charset="utf-8" />
<meta name="norton-safeweb-site-verification" content="24usqpep0ejc5w6hod3dulxwciwp0djs6c6ufp96av3t4whuxovj72wfkdjxu82yacb7430qjm8adbd5ezlt4592dq4zrvadcn9j9n-0btgdzpiojfzno16-fnsnu7xd" />
<link rel="preconnect" href="https://substackcdn.com" />
<title data-rh="true">Reliability Engineering for Air-Gapped Systems</title>
<meta data-rh="true" name="theme-color" content="#001426"/><meta data-rh="true" property="og:type" content="article"/><meta data-rh="true" property="og:title" content="Reliability Engineering for Air-Gapped Systems"/><meta data-rh="true" name="twitter:title" content="Reliability Engineering for Air-Gapped Systems"/><meta data-rh="true" name="description" content="Tips and tricks to work around delayed or partial observability for remote systems. Build a side page to visualize information, alert on anomalies and support automated recovery and self-service in order to reduce time to resolution and increase confidence in the software vendor."/><meta data-rh="true" property="og:description" content="Tips and tricks to work around inaccessible observability"/><meta data-rh="true" name="twitter:description" content="Tips and tricks to work around inaccessible observability"/><meta data-rh="true" property="og:image" content="https://substackcdn.com/image/fetch/$s_!a6ED!,w_1200,h_675,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png"/><meta data-rh="true" name="twitter:image" content="https://substackcdn.com/image/fetch/$s_!Nlgq!,f_auto,q_auto:best,fl_progressive:steep/https%3A%2F%2Falexewerlof.substack.com%2Fapi%2Fv1%2Fpost_preview%2F193103401%2Ftwitter.jpg%3Fversion%3D4"/><meta data-rh="true" name="twitter:card" content="summary_large_image"/><meta data-rh="true" property="article:modified_time" content="2026-06-16T18:13:21.741Z"/>
<style>
@layer legacy, pencraftReset, pencraft;
</style>
<link rel="preload" as="style" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
<link rel="preload" as="font" href="https://fonts.gstatic.com/s/merriweather/v33/u-4D0qyriQwlOrhSvowK_l5UcA6zuSYEqOzpPe3HOZJ5eX1WtLaQwmYiScCmDxhtNOKl8yDr3icaFF31CPDaYKfF.woff2" crossorigin />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/11100.cee9fbee.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/86118.c25b9f67.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/93519.6640a5b2.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/42118.cba90b72.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/39522.25128f14.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/47358.668d8a3c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/94881.cb76fe04.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27386.72f00430.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/76768.f15fc605.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/41932.97d2663c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/48409.0588981c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/32552.313415d5.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27313.92c631cb.css" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=0, viewport-fit=cover" />
<meta name="author" content="Alex Ewerlöf" />
<meta property="og:url" content="https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped" />
<link rel="canonical" href="https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped" />
<meta name="google-site-verification" content="HIy4u2xUDZkIs6P0KJw9TSYeq7KqXSPAQvZIs9911qU" />
<link rel="shortcut icon" href="https://substackcdn.com/image/fetch/$s_!j4nj!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa0c1d559-cfc9-48dc-9e78-57b4c52d5b70%2Ffavicon.ico">
<link rel="icon" type="image/png" sizes="16x16" href="https://substackcdn.com/image/fetch/$s_!VSMg!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa0c1d559-cfc9-48dc-9e78-57b4c52d5b70%2Ffavicon-16x16.png">
<link rel="icon" type="image/png" sizes="32x32" href="https://substackcdn.com/image/fetch/$s_!CJdc!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa0c1d559-cfc9-48dc-9e78-57b4c52d5b70%2Ffavicon-32x32.png">
<link rel="icon" type="image/png" sizes="48x48" href="https://substackcdn.com/image/fetch/$s_!5Qve!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa0c1d559-cfc9-48dc-9e78-57b4c52d5b70%2Ffavicon-48x48.png">
<link rel="apple-touch-icon" sizes="57x57" href="https://substackcdn.com/image/fetch/$s_!gV2k!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa0c1d559-cfc9-48dc-9e78-57b4c52d5b70%2Fapple-touch-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="https://substackcdn.com/image/fetch/$s_!L4Uu!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa0c1d559-cfc9-48dc-9e78-57b4c52d5b70%2Fapple-touch-icon-60x60.png">
<link rel="apple-touch-icon" sizes="72x72" href="https://substackcdn.com/image/fetch/$s_!X3nD!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa0c1d559-cfc9-48dc-9e78-57b4c52d5b70%2Fapple-touch-icon-72x72.png">
<link rel="apple-touch-icon" sizes="76x76" href="https://substackcdn.com/image/fetch/$s_!Xe9B!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa0c1d559-cfc9-48dc-9e78-57b4c52d5b70%2Fapple-touch-icon-76x76.png">
<link rel="apple-touch-icon" sizes="114x114" href="https://substackcdn.com/image/fetch/$s_!ef9e!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa0c1d559-cfc9-48dc-9e78-57b4c52d5b70%2Fapple-touch-icon-114x114.png">
<link rel="apple-touch-icon" sizes="120x120" href="https://substackcdn.com/image/fetch/$s_!B-Gw!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa0c1d559-cfc9-48dc-9e78-57b4c52d5b70%2Fapple-touch-icon-120x120.png">
<link rel="apple-touch-icon" sizes="144x144" href="https://substackcdn.com/image/fetch/$s_!PA8p!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa0c1d559-cfc9-48dc-9e78-57b4c52d5b70%2Fapple-touch-icon-144x144.png">
<link rel="apple-touch-icon" sizes="152x152" href="https://substackcdn.com/image/fetch/$s_!x3Pg!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa0c1d559-cfc9-48dc-9e78-57b4c52d5b70%2Fapple-touch-icon-152x152.png">
<link rel="apple-touch-icon" sizes="167x167" href="https://substackcdn.com/image/fetch/$s_!7ISu!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa0c1d559-cfc9-48dc-9e78-57b4c52d5b70%2Fapple-touch-icon-167x167.png">
<link rel="apple-touch-icon" sizes="180x180" href="https://substackcdn.com/image/fetch/$s_!k1Y3!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa0c1d559-cfc9-48dc-9e78-57b4c52d5b70%2Fapple-touch-icon-180x180.png">
<link rel="apple-touch-icon" sizes="1024x1024" href="https://substackcdn.com/image/fetch/$s_!hatU!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa0c1d559-cfc9-48dc-9e78-57b4c52d5b70%2Fapple-touch-icon-1024x1024.png">
<link rel="alternate" type="application/rss+xml" href="/feed?sectionId=67723" title="Alex Ewerlöf Notes"/>
<style>
@font-face{font-family:'Merriweather';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/merriweather/v33/u-4B0qyriQwlOrhSvowK_l5-eTxCVx0ZbwLvKH2Gk9hLmp0v5yA-xXPqCzLvPee1XYk_XSf-FmTCUG36AvH4ZL_Ean2DTA.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Merriweather';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/merriweather/v33/u-4B0qyriQwlOrhSvowK_l5-eTxCVx0ZbwLvKH2Gk9hLmp0v5yA-xXPqCzLvPee1XYk_XSf-FmTCUG3zAvH4ZL_Ean2DTA.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Merriweather';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/merriweather/v33/u-4B0qyriQwlOrhSvowK_l5-eTxCVx0ZbwLvKH2Gk9hLmp0v5yA-xXPqCzLvPee1XYk_XSf-FmTCUG34AvH4ZL_Ean2DTA.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Merriweather';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/merriweather/v33/u-4B0qyriQwlOrhSvowK_l5-eTxCVx0ZbwLvKH2Gk9hLmp0v5yA-xXPqCzLvPee1XYk_XSf-FmTCUG35AvH4ZL_Ean2DTA.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Merriweather';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/merriweather/v33/u-4B0qyriQwlOrhSvowK_l5-eTxCVx0ZbwLvKH2Gk9hLmp0v5yA-xXPqCzLvPee1XYk_XSf-FmTCUG33AvH4ZL_Ean0.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Merriweather';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/merriweather/v33/u-4D0qyriQwlOrhSvowK_l5UcA6zuSYEqOzpPe3HOZJ5eX1WtLaQwmYiScCmDxhtNOKl8yDr3icaGV31CPDaYKfFQn0.woff2) format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F}@font-face{font-family:'Merriweather';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/merriweather/v33/u-4D0qyriQwlOrhSvowK_l5UcA6zuSYEqOzpPe3HOZJ5eX1WtLaQwmYiScCmDxhtNOKl8yDr3icaEF31CPDaYKfFQn0.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Merriweather';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/merriweather/v33/u-4D0qyriQwlOrhSvowK_l5UcA6zuSYEqOzpPe3HOZJ5eX1WtLaQwmYiScCmDxhtNOKl8yDr3icaG131CPDaYKfFQn0.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Merriweather';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/merriweather/v33/u-4D0qyriQwlOrhSvowK_l5UcA6zuSYEqOzpPe3HOZJ5eX1WtLaQwmYiScCmDxhtNOKl8yDr3icaGl31CPDaYKfFQn0.woff2) format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Merriweather';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/merriweather/v33/u-4D0qyriQwlOrhSvowK_l5UcA6zuSYEqOzpPe3HOZJ5eX1WtLaQwmYiScCmDxhtNOKl8yDr3icaFF31CPDaYKfF.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
</style>
<style data-pub-theme-prefix>:root{--color_theme_bg_pop:#FFBC52;--background_pop:#FFBC52;--color_theme_bg_web:#001426;--cover_bg_color:#001426;--cover_bg_color_secondary:#0f2233;--background_pop_darken:#ffb238;--print_on_pop:#ffffff;--color_theme_bg_pop_darken:#ffb238;--color_theme_print_on_pop:#ffffff;--color_theme_bg_pop_20:rgba(255, 188, 82, 0.2);--color_theme_bg_pop_30:rgba(255, 188, 82, 0.3);--print_pop:#ffbc52;--color_theme_accent:#ffbc52;--cover_print_primary:#ffffff;--cover_print_secondary:#d9d9d9;--cover_print_tertiary:#d9d9d9;--cover_border_color:#ffffff;--font_family_headings_preset:'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';--font_weight_headings_preset:900;--font_family_body_preset:Merriweather,sans-serif;--font_weight_body_preset:400;--font_preset_heading:sans;--font_preset_body:fancy_serif;--home_hero:feature-media;--home_posts:custom;--home_show_top_posts:true;--web_bg_color:#001426;--background_contrast_1:#0f2233;--background_contrast_2:#223443;--background_contrast_3:#485663;--background_contrast_4:#6d7882;--background_contrast_5:#aeb5ba;--color_theme_bg_contrast_1:#0f2233;--color_theme_bg_contrast_2:#223443;--color_theme_bg_contrast_3:#485663;--color_theme_bg_contrast_4:#6d7882;--color_theme_bg_contrast_5:#aeb5ba;--color_theme_bg_elevated:#0f2233;--color_theme_bg_elevated_secondary:#223443;--color_theme_bg_elevated_tertiary:#485663;--color_theme_detail:#192b3c;--background_contrast_pop:rgba(255, 188, 82, 0.4);--color_theme_bg_contrast_pop:rgba(255, 188, 82, 0.4);--theme_bg_is_dark:1;--print_on_web_bg_color:hsl(208.4210526315789, 25.00000000000006%, 90.74509803921569%);--print_secondary_on_web_bg_color:#99a1a8;--footer_bg_color:#000C17;--footer_bg_color_h:208.69565217391306;--footer_bg_color_s:100%;--footer_bg_color_l:4.509803921568627%;--print_on_footer_bg_color:hsl(208.69565217391306, 25.000000000000057%, 90.45098039215686%);--print_secondary_on_footer_bg_color:#999ea2;--footer_bg_contrast_1:#0f1b25;--footer_bg_contrast_2:#222d36;--footer_bg_contrast_3:#485158;--footer_bg_contrast_4:#6d737a;--footer_bg_elevated:#0f1b25;--footer_bg_elevated_secondary:#222d36;--footer_bg_elevated_tertiary:#485158;--footer_detail_color:#19242e;--background_pop_rgb:255, 188, 82;--color_theme_bg_pop_rgb:255, 188, 82;--color_theme_accent_rgb:255, 188, 82;}</style>
<link rel="stylesheet" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
<style data-pub-theme-postfix></style>
<script async="true" src="https://www.googletagmanager.com/gtag/js?id=G-G8RXW18CW7&l=GA4DataLayer" >
</script>
</head>
<body class="">
<div id="entry">
<div id="main" style="--post-preview-border-radius:var(--border-radius-xs);" class="main typography use-theme-bg"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div data-testid="navbar" class="main-menu"><div class="mainMenuContent-DME8DR"><div style="position:relative;height:93px;" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-20 pc-paddingRight-20 pc-paddingTop-0 pc-paddingBottom-0 pc-justifyContent-space-between pc-alignItems-center pc-reset border-bottom-detail-k1F6C4 topBar-pIF0J1"><div style="visibility:visible;flex:1 0 0;" class="logoContainer-p12gJb"><a href="/" native class="pencraft pc-display-contents pc-reset"><div draggable="false" class="pencraft pc-display-flex pc-position-relative pc-reset"><div style="width:40px;height:40px;" class="pencraft pc-display-flex pc-reset bg-white-ZBV5av pc-borderRadius-sm overflow-hidden-WdpwT6 sizing-border-box-DggLA4"><picture class="pencraft pc-display-contents pc-reset"><source type="image/webp" sizes="40px" srcset="https://substackcdn.com/image/fetch/$s_!_Ur2!,w_40,h_40,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png 40w, https://substackcdn.com/image/fetch/$s_!_Ur2!,w_80,h_80,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png 80w, https://substackcdn.com/image/fetch/$s_!_Ur2!,w_120,h_120,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png 120w" style="display:none;"/><img style="object-fit:cover;width:40px;height:40px;" src="https://substackcdn.com/image/fetch/$s_!_Ur2!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png" srcset="https://substackcdn.com/image/fetch/$s_!_Ur2!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png 40w, https://substackcdn.com/image/fetch/$s_!_Ur2!,w_80,h_80,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png 80w, https://substackcdn.com/image/fetch/$s_!_Ur2!,w_120,h_120,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png 120w" sizes="40px" alt="Alex Ewerlöf Notes" draggable="false" class="pencraft pc-reset"/></picture></div></div></a></div><div style="flex-basis:0px;flex-grow:2;min-width:0px;display:flex;justify-content:center;" class="logoTopCenterSlot-hss6WZ"><div style="flex-grow:0;" class="titleContainer-DJYq5v overflowVisible-joA5t0"><h1 id="wordlogo" class="pencraft pc-reset font-pub-headings-FE5byy reset-IxiVJZ title-oOnUGd titleWithWordmark-GfqxEZ"><a href="/" class="pencraft pc-display-contents pc-reset"><img alt="Alex Ewerlöf Notes" src="https://substackcdn.com/image/fetch/$s_!3m6V!,e_trim:10:white/e_trim:10:transparent/h_84,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fbd56dc7c-4b9f-43f2-99a1-ecc995284940_1200x256.png" style="max-height:42px;" class="wordmark-WcDTTu"/></a></h1></div></div><div style="flex-basis:0px;flex-grow:1;min-width:fit-content;" class="pencraft pc-display-flex pc-justifyContent-flex-end pc-alignItems-center pc-reset buttonsContainerContainer-ThaN_w"><div class="buttonsContainer-SJBuep"><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-flex-end pc-alignItems-center pc-reset navbar-buttons"><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><span data-state="closed"><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></span><button tabindex="0" type="button" aria-label="Share Publication" id="radix-P0-4" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg></button></div><button tabindex="0" type="button" data-testid="noncontributor-cta-button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_primary-RfbeYt size_md-gCDS3o">Subscribe</button><button tabindex="0" data-native="true" type="button" data-href="https://substack.com/sign-in?redirect=%2Fp%2Freliability-engineering-for-air-gapped&amp;for_pub=alexewerlof" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_tertiary-rlke8z size_md-gCDS3o">Sign in</button></div></div></div></div></div><div style="background-color:#001426;height:94px;"></div></div></div><div><script nonce="nQGtn4EHptwtQVZSM4a71A" type="application/ld+json">{"@context":"https://schema.org","@type":"NewsArticle","url":"https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped","mainEntityOfPage":"https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped","headline":"Reliability Engineering for Air-Gapped Systems","description":"Tips and tricks to work around delayed or partial observability for remote systems. Build a side page to visualize information, alert on anomalies and support automated recovery and self-service in order to reduce time to resolution and increase confidence in the software vendor.","image":[{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!a6ED!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png"}],"datePublished":"2026-04-03T21:03:06+00:00","dateModified":"2026-04-03T21:03:06+00:00","isAccessibleForFree":true,"author":[{"@type":"Person","name":"Alex Ewerl\u00F6f","url":"https://substack.com/@alexewerlof","description":"Writes about technical leadership, growth mindset, and system reliability engineering. Senior Staff Engineer, MSc Systems Engineering from KTH, Stockholmer, dad, amateur artist. Read more here: https://www.alexewerlof.com/who","identifier":"user:87732486","sameAs":["https://twitter.com/alexewerlof"],"image":{"@type":"ImageObject","contentUrl":"https://substackcdn.com/image/fetch/$s_!IRtW!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!IRtW!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg"}}],"publisher":{"@type":"Organization","name":"Alex Ewerl\u00F6f Notes","url":"https://blog.alexewerlof.com","description":"Technical Leadership, Reliability Engineering, Growth","interactionStatistic":{"@type":"InteractionCounter","name":"Subscribers","interactionType":"https://schema.org/SubscribeAction","userInteractionCount":10000},"identifier":"pub:1002265","logo":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!_Ur2!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!_Ur2!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!_Ur2!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png"},"image":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!_Ur2!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!_Ur2!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!_Ur2!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png"},"sameAs":["https://twitter.com/alexewerlof"]},"interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":4},{"@type":"InteractionCounter","interactionType":"https://schema.org/ShareAction","userInteractionCount":1},{"@type":"InteractionCounter","interactionType":"https://schema.org/CommentAction","userInteractionCount":3}]}</script><script nonce="nQGtn4EHptwtQVZSM4a71A" type="application/ld+json">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Alex Ewerl\u00F6f Notes","item":"https://blog.alexewerlof.com"},{"@type":"ListItem","position":2,"name":"Reliability Engineering for Air-Gapped Systems","item":"https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped"}]}</script><div aria-label="Post" role="main" class="single-post-container"><div class="container"><div class="single-post"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><article class="typography newsletter-post post"><div role="region" aria-label="Post header" class="post-header"><div class="pencraft pc-display-flex pc-reset post-label"><a href="https://blog.alexewerlof.com/s/sre/?utm_source=substack&amp;utm_medium=menu" class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-14-MLPa7j weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA">Reliability Engineering</a></div><h1 dir="auto" class="post-title published title-X77sOw">Reliability Engineering for Air-Gapped Systems</h1><h3 dir="auto" class="subtitle subtitle-HEEcLo">Tips and tricks to work around inaccessible observability</h3><div aria-label="Post UFI" role="region" class="pencraft pc-display-flex pc-flexDirection-column pc-paddingBottom-16 pc-reset"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-gap-12 pc-alignItems-center pc-reset byline-wrapper"><div class="pencraft pc-display-flex pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-row pc-gap-8 pc-alignItems-center pc-justifyContent-flex-start pc-reset"><div style="--scale:36px;--offset:9px;--border-width:4.5px;" class="pencraft pc-display-flex pc-flexDirection-row pc-alignItems-center pc-justifyContent-flex-start pc-reset ltr-qDBmby"><a href="https://substack.com/@alexewerlof" aria-label="View Alex Ewerlöf's profile" class="pencraft pc-display-contents pc-reset"><div style="--scale:36px;" tabindex="0" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA animate-XFJxE4 outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 pressable-sm-YIJFKJ showFocus-sk_vEm container-TAtrWj interactive-UkK0V6 avatar-u8q6xB last-JfNEJ_"><div style="--scale:36px;" title="Alex Ewerlöf" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!IRtW!,w_36,h_36,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg 1x, https://substackcdn.com/image/fetch/$s_!IRtW!,w_72,h_72,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg 2x, https://substackcdn.com/image/fetch/$s_!IRtW!,w_108,h_108,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg 3x"/><img src="https://substackcdn.com/image/fetch/$s_!IRtW!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg" alt="Alex Ewerlöf's avatar" srcset="https://substackcdn.com/image/fetch/$s_!IRtW!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg 1x, https://substackcdn.com/image/fetch/$s_!IRtW!,w_72,h_72,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg 2x, https://substackcdn.com/image/fetch/$s_!IRtW!,w_108,h_108,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg 3x" width="36" height="36" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div></a></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><div class="pencraft pc-reset color-primary-zABazT line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><span style="min-width:0;" data-state="closed"><a href="https://substack.com/@alexewerlof" class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ">Alex Ewerlöf</a></span></div><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><div class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><time datetime="2026-04-03T21:03:06.206Z" title="2026-04-03T21:03:06.206Z">Apr 03, 2026</time></div></div></div></div></div><div class="pencraft pc-display-flex pc-gap-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (4)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">4</div></button></div><button tabindex="0" type="button" aria-label="View comments (3)" data-href="https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg><div class="label">3</div></button><button tabindex="0" type="button" id="radix-P0-27" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg><div class="label">1</div></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div><div class="visibility-check"></div><div><div class="dt-post-body"><div class="available-content"><div dir="auto" class="body markup"><p>Back in February, I helped a few teams from defense sector to measure the right thing (SLI), set reasonable expectations (SLO) and tie reliability to accountability through alerting. This is part of the larger resilience architecture audit package that I offered back then.</p><p>The details is behind an NDA (non-disclosure agreement) but there was an interesting aspect that is worth discussing:</p><p>All those systems were air-gapped, meaning the team that builds the software has no access to metrics, logs or runtime.</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!a6ED!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!a6ED!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png 424w, https://substackcdn.com/image/fetch/$s_!a6ED!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png 848w, https://substackcdn.com/image/fetch/$s_!a6ED!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png 1272w, https://substackcdn.com/image/fetch/$s_!a6ED!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!a6ED!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png" width="851" height="861" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/cf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:861,&quot;width&quot;:851,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:86496,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:false,&quot;topImage&quot;:true,&quot;internalRedirect&quot;:&quot;https://blog.alexewerlof.com/i/193103401?img=https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png&quot;,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!a6ED!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png 424w, https://substackcdn.com/image/fetch/$s_!a6ED!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png 848w, https://substackcdn.com/image/fetch/$s_!a6ED!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png 1272w, https://substackcdn.com/image/fetch/$s_!a6ED!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png 1456w" sizes="100vw" fetchpriority="high" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a><figcaption class="image-caption">An air-gapped system may be operated by someone other than the developers. The observability may be absent, delayed or partial </figcaption></figure></div><p>How could they measure SLI/SLO without real time data? How could they do that without any data? How would they know something is broken? And how would they diagnose and fix the problem?</p><p>That’s what this post is about. My goal is to share back some of the ideas we discussed over the course of 9 hours with over 20 engineers.</p><p><strong>Disclaimer: no AI is used in this post other than my browser’s built-in spell checker. Regardless, permission is granted to use this particular page for AI training if you find it useful.</strong></p><div data-component-name="SubscribeWidget" class="subscribe-widget"><div class="pencraft pc-display-flex pc-justifyContent-center pc-reset"><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe-widget"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><input type="hidden" name="source_post_id" value="193103401"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div><h2 class="header-anchor-post">Strict Requirements<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§strict-requirements" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://blog.alexewerlof.com/i/193103401/strict-requirements" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>I have worked at heavily regulated industries like healthcare, banking and automotive but this was the first time that I came across a system that was completely cut off from the internet for security and privacy reasons.</p><div data-component-name="DigestPostEmbed" class="digestPostEmbed-flwiST"><a href="https://blog.alexewerlof.com/p/nfr" rel="noopener" target="_blank"><div class="pencraft pc-display-flex pc-gap-16 pc-reset"><div style="width:70px;height:70px;" class="pencraft pc-reset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!GpLJ!,w_140,h_140,c_fill,f_webp,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1aac3c2c-3d39-4dca-864d-9a28f39a93e9_961x446.png"/><img src="https://substackcdn.com/image/fetch/$s_!GpLJ!,w_140,h_140,c_fill,f_auto,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1aac3c2c-3d39-4dca-864d-9a28f39a93e9_961x446.png" sizes="100vw" alt="NFR" width="140" height="140" class="img-OACg1c smSquare-NGbPBa pencraft pc-reset"/></picture></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><h4 class="pencraft pc-reset color-pub-primary-text-NyXPlw line-height-24-jnGwiv font-display-nhmvtD size-20-P_cSRT weight-bold-DmI9lw reset-IxiVJZ">NFR</h4><div class="pencraft pc-display-flex pc-gap-4 pc-alignItems-center pc-reset"><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><a href="https://substack.com/profile/87732486-alex-ewerlof" class="inheritColor-WetTGJ">Alex Ewerlöf</a></div><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T reset-IxiVJZ">·</div><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA">April 22, 2024</div></div><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-0 pc-paddingBottom-0 pc-alignItems-center pc-reset"><a href="https://blog.alexewerlof.com/p/nfr" class="pencraft pc-reset align-center-y7ZD4w line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-medium-fw81nC reset-IxiVJZ"><div class="pencraft pc-display-flex pc-gap-8 pc-alignItems-center pc-reset link-HREYZo"><span class="pencraft pc-reset color-accent-BVX_7M line-height-20-t4M0El font-text-qe4AeH size-14-MLPa7j weight-semibold-uqA4FV reset-IxiVJZ">Read full story</span><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-right"><path d="M5 12h14"></path><path d="m12 5 7 7-7 7"></path></svg></div></a></div></div></div></a></div><p>The NFR (non-functional requirements) dictated the system designed to:</p><ul><li><p>Be deployed on-prem (at the military facility’s premises)</p></li><li><p>Operate completely without internet to minimize the attack surface but also because it was primarily used internally inside military operations.</p></li><li><p>Be deployed using an archive file that was physically handed over to the operator (on the military consumer’s side) to reduce the risk of man-in-the-middle or supply chain attacks.</p></li><li><p>Any problem with the system was resolved over secure phone calls because sending diagnostics data over email or other channels was against security protocol.</p></li><li><p>In some occasions the software vendor had to physically dispatch an engineer to the client side to diagnose and fix issues onsite.</p></li></ul><p>Yeah, not your typical cloud service for sure! 😄</p><p>But that was exactly the type of problem that got me involved with them. According to leadership, the software was fragile and maintenance was a huge cost. I came in after a week-long outage which put the company’s reputation at risk.</p><h2 class="header-anchor-post">Engineers<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§engineers" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://blog.alexewerlof.com/i/193103401/engineers" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>You might think that the system quality is a direct reflection of the talent behind it and you’d be forgiven.</p><p>When I met the team, I felt their frustration but also their commitment to improve reliability and openness to new ideas.</p><p>It’s not an exaggeration to say that most of what I’m about to share came out of intense discussion with the team who knew their domain and constraints very well in conjunction with the service level (SLI/SLO/SLA) model that I specialize in.</p><div data-component-name="DigestPostEmbed" class="digestPostEmbed-flwiST"><a href="https://blog.alexewerlof.com/p/service-level-terminology" rel="noopener" target="_blank"><div class="pencraft pc-display-flex pc-gap-16 pc-reset"><div style="width:70px;height:70px;" class="pencraft pc-reset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!MlI7!,w_140,h_140,c_fill,f_webp,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F5bb74595-f0d9-4d19-9a4c-23d596b721a8_1010x1122.png"/><img src="https://substackcdn.com/image/fetch/$s_!MlI7!,w_140,h_140,c_fill,f_auto,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F5bb74595-f0d9-4d19-9a4c-23d596b721a8_1010x1122.png" sizes="100vw" alt="Service Level Terminology" width="140" height="140" class="img-OACg1c smSquare-NGbPBa pencraft pc-reset"/></picture></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><h4 class="pencraft pc-reset color-pub-primary-text-NyXPlw line-height-24-jnGwiv font-display-nhmvtD size-20-P_cSRT weight-bold-DmI9lw reset-IxiVJZ">Service Level Terminology</h4><div class="pencraft pc-display-flex pc-gap-4 pc-alignItems-center pc-reset"><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><a href="https://substack.com/profile/87732486-alex-ewerlof" class="inheritColor-WetTGJ">Alex Ewerlöf</a></div><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T reset-IxiVJZ">·</div><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA">October 25, 2024</div></div><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-0 pc-paddingBottom-0 pc-alignItems-center pc-reset"><a href="https://blog.alexewerlof.com/p/service-level-terminology" class="pencraft pc-reset align-center-y7ZD4w line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-medium-fw81nC reset-IxiVJZ"><div class="pencraft pc-display-flex pc-gap-8 pc-alignItems-center pc-reset link-HREYZo"><span class="pencraft pc-reset color-accent-BVX_7M line-height-20-t4M0El font-text-qe4AeH size-14-MLPa7j weight-semibold-uqA4FV reset-IxiVJZ">Read full story</span><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-right"><path d="M5 12h14"></path><path d="m12 5 7 7-7 7"></path></svg></div></a></div></div></div></a></div><h3 class="header-anchor-post">Self-service<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§self-service" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://blog.alexewerlof.com/i/193103401/self-service" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>The team could not observe the system in real-time, but they could work around that limitation by helping the operator (usually the IT personnel at the military facility) act as SRE.</p><p>The idea was simple: provide a reliable page to:</p><ol><li><p><strong>Real time status:</strong><span> create simple dashboards to visualize CPU/Memory/Disk usage, Network connectivity, and load (e.g. request per second) and metric trends over time. This could help the operator quickly identify common problems. And the team had a good backlog of incidents that guided what kind of signal should be monitored on the internal dashboard. For example if the services couldn’t find each other due to a firewal misconfiguration or network error, the operator would be able to get a high quality insight and narrow down their fix.</span></p></li><li><p><strong>Proactive alerting:</strong><span> once we have that data, creating alerts (e.g. via email) is trivial. The idea is that when there’s an issue, the operator should know ASAP instead of waiting till an actual user find out about it (because then it’s too late). This severely improve TTD (time to detect) and reduce down time.</span></p></li><li><p><strong>Mainstreamed troubleshooting:</strong><span> once an error was identified (e.g. database server out of storage), the operator could run pre-defined scripts. You can think of them as automated runbooks. Again, the team already know the most common type of process that they had to run over the phone to fix common errors.</span></p></li><li><p><strong>Auto-repair:</strong><span> For the most common type of error, the system would auto-diagnose and auto-repair (e.g. restarting an instance or scaling horizontally). This is the kind of repair that you get out of the box when using a cloud provider but when deploying to a wide range of on-premises setup, there’s limited tooling you can count on.</span></p></li><li><p><strong>Intelligent Anomaly Detection:</strong><span> Observability is a measure of how well internal states of a system can be inferred from knowledge of its external outputs. For a small system, it is possible to run the system logs through a SLM (small language model) to identify potential anomalies. Something like </span><a href="https://www.liquid.ai/models">Liquid AI</a><span> can easily run on a side card node and be tasked to analyze a fraction of logs for signals. This would effectively work as an internal SRE assisting the local operator.</span></p></li><li><p><strong>Anonymization:</strong><span> stripping the data from critical information like PII (personally identifiable information) or sensitive military planning information, it’s theoretically possible to create diagnostic data that can be shared with the vendor to understand and improve the performance of the system.</span></p></li></ol><div data-component-name="DigestPostEmbed" class="digestPostEmbed-flwiST"><a href="https://blog.alexewerlof.com/p/ai-topology" rel="noopener" target="_blank"><div class="pencraft pc-display-flex pc-gap-16 pc-reset"><div style="width:70px;height:70px;" class="pencraft pc-reset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!4UFi!,w_140,h_140,c_fill,f_webp,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F55317f3d-e36b-449f-b4b6-f63db8d987ad_721x689.png"/><img src="https://substackcdn.com/image/fetch/$s_!4UFi!,w_140,h_140,c_fill,f_auto,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F55317f3d-e36b-449f-b4b6-f63db8d987ad_721x689.png" sizes="100vw" alt="AI topology" width="140" height="140" class="img-OACg1c smSquare-NGbPBa pencraft pc-reset"/></picture></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><h4 class="pencraft pc-reset color-pub-primary-text-NyXPlw line-height-24-jnGwiv font-display-nhmvtD size-20-P_cSRT weight-bold-DmI9lw reset-IxiVJZ">AI topology</h4><div class="pencraft pc-display-flex pc-gap-4 pc-alignItems-center pc-reset"><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><a href="https://substack.com/profile/87732486-alex-ewerlof" class="inheritColor-WetTGJ">Alex Ewerlöf</a></div><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T reset-IxiVJZ">·</div><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA">October 24, 2025</div></div><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-0 pc-paddingBottom-0 pc-alignItems-center pc-reset"><a href="https://blog.alexewerlof.com/p/ai-topology" class="pencraft pc-reset align-center-y7ZD4w line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-medium-fw81nC reset-IxiVJZ"><div class="pencraft pc-display-flex pc-gap-8 pc-alignItems-center pc-reset link-HREYZo"><span class="pencraft pc-reset color-accent-BVX_7M line-height-20-t4M0El font-text-qe4AeH size-14-MLPa7j weight-semibold-uqA4FV reset-IxiVJZ">Read full story</span><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-right"><path d="M5 12h14"></path><path d="m12 5 7 7-7 7"></path></svg></div></a></div></div></div></a></div><p>You may have noticed that the ideas are sorted incrementally based on how hard they are to implement and how risky they are.</p><h2 class="header-anchor-post">Bespoke Observability?<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§bespoke-observability" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://blog.alexewerlof.com/i/193103401/bespoke-observability" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>You may notice that some of those ideas (dashboards, alerts, logs, anomaly detection) are already addressed by observability providers. The team did not have to reinvent the wheel. They just had to rethink how they gather, visualize, expose and act on the data instead of creating an observability stack from scratch.</p><p>The goal of this exercise wasn’t to create a bespoke observability stack, but rather identify the requirements and be intentional about what data to gather, how to expose it, what kind of action to support and why.</p><p>Regardless, even a simple status page would be a big win because it helped the customers answer a simple question: is it just me?</p><div data-component-name="DigestPostEmbed" class="digestPostEmbed-flwiST"><a href="https://blog.alexewerlof.com/p/service-degradation-vs-disruption" rel="noopener" target="_blank"><div class="pencraft pc-display-flex pc-gap-16 pc-reset"><div style="width:70px;height:70px;" class="pencraft pc-reset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!wL5v!,w_140,h_140,c_fill,f_webp,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0bd88583-8b9e-4ce0-8cb6-befa958a794f_943x620.png"/><img src="https://substackcdn.com/image/fetch/$s_!wL5v!,w_140,h_140,c_fill,f_auto,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0bd88583-8b9e-4ce0-8cb6-befa958a794f_943x620.png" sizes="100vw" alt="Degradation vs disruption" width="140" height="140" class="img-OACg1c smSquare-NGbPBa pencraft pc-reset"/></picture></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><h4 class="pencraft pc-reset color-pub-primary-text-NyXPlw line-height-24-jnGwiv font-display-nhmvtD size-20-P_cSRT weight-bold-DmI9lw reset-IxiVJZ">Degradation vs disruption</h4><div class="pencraft pc-display-flex pc-gap-4 pc-alignItems-center pc-reset"><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><a href="https://substack.com/profile/87732486-alex-ewerlof" class="inheritColor-WetTGJ">Alex Ewerlöf</a></div><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T reset-IxiVJZ">·</div><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA">July 10, 2024</div></div><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-0 pc-paddingBottom-0 pc-alignItems-center pc-reset"><a href="https://blog.alexewerlof.com/p/service-degradation-vs-disruption" class="pencraft pc-reset align-center-y7ZD4w line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-medium-fw81nC reset-IxiVJZ"><div class="pencraft pc-display-flex pc-gap-8 pc-alignItems-center pc-reset link-HREYZo"><span class="pencraft pc-reset color-accent-BVX_7M line-height-20-t4M0El font-text-qe4AeH size-14-MLPa7j weight-semibold-uqA4FV reset-IxiVJZ">Read full story</span><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-right"><path d="M5 12h14"></path><path d="m12 5 7 7-7 7"></path></svg></div></a></div></div></div></a></div><p><span>The key idea here is to have the status page </span><strong>separate</strong><span> from the main application, ideally on a completely different release cadence. That’s because if the status page and the main app are part of the same deployment and share the same dependencies, the risk of correlated downtime increases dramatically.</span></p><p>That is not good because the status page is most needed when the main service is experiencing degradation or disruption.</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!jfie!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!jfie!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png 424w, https://substackcdn.com/image/fetch/$s_!jfie!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png 848w, https://substackcdn.com/image/fetch/$s_!jfie!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png 1272w, https://substackcdn.com/image/fetch/$s_!jfie!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!jfie!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png" width="950" height="532" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:532,&quot;width&quot;:950,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:69037,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:&quot;https://blog.alexewerlof.com/i/193103401?img=https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png&quot;,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!jfie!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png 424w, https://substackcdn.com/image/fetch/$s_!jfie!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png 848w, https://substackcdn.com/image/fetch/$s_!jfie!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png 1272w, https://substackcdn.com/image/fetch/$s_!jfie!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a><figcaption class="image-caption">A simple architecture showing how the status checker keeps testing the API and optionally front-end assets to populate the status DB which is in turn used to render the status page frontend</figcaption></figure></div><p><span>Again, the key is to treat the status page as a separate entity with its own isolation and lifecycle. Some vendors even keep it on a separate domain to decouple it from the main service DNS issues (For example: the status page for </span><code>github.com</code><span> is at </span><code>githubstatus.com</code><span>).</span></p><div data-component-name="SubscribeWidget" class="subscribe-widget"><div class="pencraft pc-display-flex pc-justifyContent-center pc-reset"><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe-widget"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><input type="hidden" name="source_post_id" value="193103401"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div><h2 class="header-anchor-post">Bonus ideas<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§bonus-ideas" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://blog.alexewerlof.com/i/193103401/bonus-ideas" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>We also discussed a couple more ideas that work around the unique limitations of this setup:</p><ol><li><p><strong>Cryptic but specific error codes:</strong><span> when a support call is made without useful information, it hurts the TTR (time to resolution). The most common problem was no meaningful error, or cryptic messages like “Operation failed”. This left both parties (the operator and vendor) wonder: what went wrong? Why? Where? How to fix that? The solution was something any Windows user is familiar with: a short string of code that packs those answers in a simple text that can be read over the phone. “Oh you got </span><code>0xAF6600BB</code><span>? That’s means the static web server has inconsistent state. Please go to the Application tab and clear storage!”. Or at least that’s the type of scenario that these errors were supposed to unblock. The idea was to assign a unique ID to </span><strong>each type of operation, module, component, and system</strong><span> to pinpoint exactly what failed and where. Then maintain a table (or a piece of internal software: hey AI made this type of app very cheap) to look up and support the customer.</span></p></li><li><p><strong>Metric batches:</strong><span> since we cannot rely on real-time monitoring due to security and privacy concerns, the next best thing is to create small batches of metric data (think time series, compressed) that can manually be exported by the operator, either as email attachments or via USB. </span></p></li><li><p><strong>Pseudonymized logs:</strong><span> we did mention that one of the things AI could do was to clean up the logs to be transferable to the software vendor for diagnostics but due to complexity and unpredictability, a simple regexp would do. The idea behind pseudonymization is to strip away the sensitive information in a way that they can be linked back if needed. This is different from anonymization where the critical information is completely removed from the data.</span></p></li></ol><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!U2b8!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!U2b8!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png 424w, https://substackcdn.com/image/fetch/$s_!U2b8!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png 848w, https://substackcdn.com/image/fetch/$s_!U2b8!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png 1272w, https://substackcdn.com/image/fetch/$s_!U2b8!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!U2b8!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png" width="1155" height="548" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:548,&quot;width&quot;:1155,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:75058,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:&quot;https://blog.alexewerlof.com/i/193103401?img=https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png&quot;,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!U2b8!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png 424w, https://substackcdn.com/image/fetch/$s_!U2b8!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png 848w, https://substackcdn.com/image/fetch/$s_!U2b8!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png 1272w, https://substackcdn.com/image/fetch/$s_!U2b8!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a><figcaption class="image-caption"><span>An example of error codes from </span><a href="https://www.samsung.com/ph/support/home-appliances/samsung-front-load-washer-error-codes/">Samsung Washing machines</a><span> demonstrates how the system can provide onsite help without internet connectivity.</span></figcaption></figure></div><h1 class="header-anchor-post">SLI/SLO/Alerting<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§slisloalerting" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://blog.alexewerlof.com/i/193103401/slisloalerting" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h1><p>So, did we solve the SLI/SLO problem for this type of air-gaped high security setup?</p><p>Not in the sense that Google preaches in their SRE books. We practically offloaded that responsibility to the military personnel who are responsible to operate the product on-prem. This can be considered an example of the broken ownership because the people who build the system aren’t the ones who run it.</p><div data-component-name="DigestPostEmbed" class="digestPostEmbed-flwiST"><a href="https://blog.alexewerlof.com/p/broken-ownership" rel="noopener" target="_blank"><div class="pencraft pc-display-flex pc-gap-16 pc-reset"><div style="width:70px;height:70px;" class="pencraft pc-reset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!E-8i!,w_140,h_140,c_fill,f_webp,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F72091ddc-6191-4ccb-8d06-15a7edbed199_1012x964.png"/><img src="https://substackcdn.com/image/fetch/$s_!E-8i!,w_140,h_140,c_fill,f_auto,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F72091ddc-6191-4ccb-8d06-15a7edbed199_1012x964.png" sizes="100vw" alt="6 Archetypes of Broken Ownership" width="140" height="140" class="img-OACg1c smSquare-NGbPBa pencraft pc-reset"/></picture></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><h4 class="pencraft pc-reset color-pub-primary-text-NyXPlw line-height-24-jnGwiv font-display-nhmvtD size-20-P_cSRT weight-bold-DmI9lw reset-IxiVJZ">6 Archetypes of Broken Ownership</h4><div class="pencraft pc-display-flex pc-gap-4 pc-alignItems-center pc-reset"><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><a href="https://substack.com/profile/87732486-alex-ewerlof" class="inheritColor-WetTGJ">Alex Ewerlöf</a></div><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T reset-IxiVJZ">·</div><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA">August 1, 2023</div></div><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-0 pc-paddingBottom-0 pc-alignItems-center pc-reset"><a href="https://blog.alexewerlof.com/p/broken-ownership" class="pencraft pc-reset align-center-y7ZD4w line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-medium-fw81nC reset-IxiVJZ"><div class="pencraft pc-display-flex pc-gap-8 pc-alignItems-center pc-reset link-HREYZo"><span class="pencraft pc-reset color-accent-BVX_7M line-height-20-t4M0El font-text-qe4AeH size-14-MLPa7j weight-semibold-uqA4FV reset-IxiVJZ">Read full story</span><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-right"><path d="M5 12h14"></path><path d="m12 5 7 7-7 7"></path></svg></div></a></div></div></div></a></div><p>To their defense, this is primarily a side effect of the strict NFRs.</p><p>The key insight was to provide them with bespoke compliant observability and repair tools to:</p><ol><li><p><strong>Reduce false alerts:</strong><span> by providing them with better signals about the health of the system</span></p></li><li><p><strong>Reduce support calls:</strong><span> by enabling them to diagnose and repair common errors that happen due to discrepancy of the on-prem setup.</span></p></li><li><p><strong>Increase ownership:</strong><span> This is in a way deflecting the on-prem problems to the people who are in the best position to fix it because they own that setup (</span><a href="https://blog.alexewerlof.com/p/you-build-it-you-own-it">know how it works, have the mandate, and are responsible</a><span>).</span></p></li><li><p><strong>Reduce TTR:</strong><span> from diagnostic to alerting and troubleshooting, the on-prem operators could get a lot done before even touching the support phone calls.</span></p></li><li><p><strong>Boost confidence:</strong><span> previously any time something broke, the operators blamed the software and vendor. Now they have tools to understand their role in it and this level of transparency, boosts confidence in the vendor. That’s because the vendor is demonstrating that they know what they’ve shipped, how it may fail, and what to do in case something goes wrong.</span></p></li></ol><div data-attrs="{&quot;url&quot;:&quot;https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped?utm_source=substack&amp;utm_medium=email&amp;utm_content=share&amp;action=share&quot;,&quot;text&quot;:&quot;Share&quot;}" data-component-name="CaptionedButtonToDOM" class="captioned-button-wrap"><div class="preamble"><p class="cta-caption">If you found this post insightful, please share it in your circles and on social media to inspire others</p></div><p data-attrs="{&quot;url&quot;:&quot;https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped?utm_source=substack&amp;utm_medium=email&amp;utm_content=share&amp;action=share&quot;,&quot;text&quot;:&quot;Share&quot;}" data-component-name="ButtonCreateButton" class="button-wrapper"><a href="https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped?utm_source=substack&amp;utm_medium=email&amp;utm_content=share&amp;action=share" class="button primary"><span>Share</span></a></p></div></div></div><div class="visibility-check"></div></div><div class="post-footer"><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG border-top-detail-themed-k9TZAY border-bottom-detail-themed-Ua9186 post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (4)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">4</div></button></div><button tabindex="0" type="button" aria-label="View comments (3)" data-href="https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg><div class="label">3</div></button><button tabindex="0" type="button" id="radix-P0-56" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg><div class="label">1</div></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div></article></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="visibility-check"></div><div id="discussion" class="pencraft pc-display-flex pc-flexDirection-column pc-gap-16 pc-paddingTop-32 pc-paddingBottom-32 pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-32 pc-reset container"><h4 class="pencraft pc-reset line-height-24-jnGwiv font-display-nhmvtD size-20-P_cSRT weight-bold-DmI9lw reset-IxiVJZ">Discussion about this post</h4><div class="pencraft pc-alignSelf-flex-start pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Select discussion type" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-70-content-comments" data-state="active" id="radix-P0-70-trigger-comments" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Comments</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-70-content-restacks" data-state="inactive" id="radix-P0-70-trigger-restacks" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Restacks</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div></div></div><div id="substack-comments" class="single-post-section comments-section"><div class="container"><div class="visibility-check"></div><div data-test-id="comment-input" class="pencraft pc-display-flex pc-reset flex-grow-rzmknG"><form class="comment-input-form form-CkZ7Kt"><div style="--scale:32px;" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><div style="--scale:32px;" title="User" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png" alt="User's avatar" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x" width="32" height="32" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-8 pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-flexDirection-column pc-boxShadow-xs pc-reset bg-primary-zk6FDl flex-grow-rzmknG border-detail-EGrm7T pc-borderRadius-sm"><textarea name="body" placeholder="Write a comment..." aria-label="Write a comment..." rows="4" class="pencraft input-qHk4bN autogrowing-_ipn9Y textarea-GbEjRX inputText-pV_yWb unstyled-IJLUJU"></textarea></div><div data-state="closed" class="pencraft pc-display-flex pc-flexDirection-column pc-reset overflow-hidden-WdpwT6"></div></div></form></div><div class="comment-list post-page-root-comment-list"><div class="comment-list-items"><div class="comment"><div id="comment-277164624" class="comment-anchor"></div><div id="comment-277164624-reply" class="comment-anchor"></div><div role="article" aria-label="Comment by jps" class="pencraft pc-display-flex pc-gap-12 pc-paddingBottom-12 pc-reset comment-content"><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><a href="https://substack.com/profile/30753764-jps?utm_source=comment" aria-label="View jps's profile" class="pencraft pc-display-contents pc-reset"><div style="--scale:32px;" tabindex="0" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA animate-XFJxE4 outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 pressable-sm-YIJFKJ showFocus-sk_vEm container-TAtrWj interactive-UkK0V6"><div style="--scale:32px;" title="jps" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!n7Iv!,w_32,h_32,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fblack.png 1x, https://substackcdn.com/image/fetch/$s_!n7Iv!,w_64,h_64,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fblack.png 2x, https://substackcdn.com/image/fetch/$s_!n7Iv!,w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fblack.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!n7Iv!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fblack.png" alt="jps's avatar" srcset="https://substackcdn.com/image/fetch/$s_!n7Iv!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fblack.png 1x, https://substackcdn.com/image/fetch/$s_!n7Iv!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fblack.png 2x, https://substackcdn.com/image/fetch/$s_!n7Iv!,w_96,h_96,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fblack.png 3x" width="32" height="32" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div></a></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-4 pc-reset"><div class="pencraft pc-display-flex pc-minWidth-0 pc-gap-8 pc-alignItems-center pc-justifyContent-space-between pc-reset line-height-20-t4M0El font-text-qe4AeH size-15-Psle70 weight-regular-mUq6Gb"><div class="pencraft pc-display-flex pc-minWidth-0 pc-gap-8 pc-alignItems-center pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-gap-6 pc-reset color-primary-zABazT line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-regular-mUq6Gb reset-IxiVJZ comment-author-name"><div class="pencraft pc-display-flex pc-gap-4 pc-minWidth-0 pc-alignItems-center pc-reset"><span class="pencraft pc-reset ellipsis-Jb8om8 line-height-20-t4M0El font-text-qe4AeH size-15-Psle70 weight-medium-fw81nC reset-IxiVJZ"><span style="min-width:0;" data-state="closed"><span class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ"><a href="https://substack.com/profile/30753764-jps?utm_source=substack-feed-item" showBack data-native="true" class="link-LIBpto">jps</a></span></span></span></div></div><a data-native="true" href="https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped/comment/277164624" rel="nofollow" title="Jun 16, 2026, 8:06 AM" class="pencraft pc-reset color-secondary-ls1g8s decoration-hover-underline-ClDVRM reset-IxiVJZ"><span class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-regular-mUq6Gb nowrap-QngyoB reset-IxiVJZ">Jun 16</span></a></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"></div><div class="pencraft pc-display-flex pc-reset comment-menu-trigger-container triggerContainer-eX588u"><button tabindex="0" type="button" aria-label="Ellipsis" id="radix-P0-83" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft trigger-j08Uop iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_quaternary-kpMibu"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-ellipsis"><circle cx="12" cy="12" r="1"></circle><circle cx="19" cy="12" r="1"></circle><circle cx="5" cy="12" r="1"></circle></svg></button></div></div><div class="comment-body expanded"><p><span>The ideas are quite useful for cloud services as well.</span></p><p><span>But without real time monitoring, how could such systems be continuously improved based on actual usage?</span></p></div><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-8 pc-justifyContent-flex-start pc-alignItems-center pc-reset comment-actions withShareButton-hQzuEn"><span class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ"><a class="pencraft pc-reset link-_X6et2 link-LIBpto"><div class="pencraft pc-display-flex pc-gap-6 pc-alignItems-center pc-reset"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="var(--color-fg-secondary-themed)" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA comment-actions-label">Reply</div></div></a></span><span class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ"><a class="pencraft pc-reset link-_X6et2 link-LIBpto"><div class="pencraft pc-display-flex pc-gap-6 pc-alignItems-center pc-reset"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="var(--color-fg-secondary-themed)" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg><div class="pencraft pc-reset color-pub-secondary-text-hGQ02T line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA comment-actions-label">Share</div></div></a></span></div><div data-state="closed" class="pencraft pc-display-flex pc-flexDirection-column pc-reset overflow-hidden-WdpwT6"></div></div></div><div class="more-replies-container"><a href="https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped/comment/277164624" class="more-replies">2 replies by Alex Ewerlöf and others</a></div></div></div></div><a href="https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped/comments" class="more-comments">2 more comments...</a></div></div></div><div class="single-post-section"><div class="container"><div class="visibility-check"></div><div style="margin-left:-8px;margin-right:-8px;" aria-label="Top Posts Footer" role="region" class="pencraft pc-paddingTop-24 pc-paddingBottom-24 pc-reset"><div class="portable-archive empty-list"><div aria-label="Archive sort tabs" role="navigation" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-8 pc-paddingRight-8 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Tabs" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-101-content-top" data-state="active" id="radix-P0-101-trigger-top" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Top</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-101-content-new" data-state="inactive" id="radix-P0-101-trigger-new" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Latest</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-101-content-community" data-state="inactive" id="radix-P0-101-trigger-community" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Discussions</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></div><div class="portable-archive-list"><p class="portable-archive-empty dt-archive-empty-state">No posts</p></div></div></div></div></div><div class="visibility-check"></div><div class="pencraft pc-display-contents pc-reset pubInvertedTheme-U483dz"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-justifyContent-center pc-padding-48 pc-mobile-padding-24 pc-reset bg-primary-zk6FDl container-jsOc9L"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-gap-24 pc-flexWrap-wrap pc-reset content-jLbYeh"><h3 class="pencraft pc-reset color-primary-zABazT align-center-y7ZD4w line-height-28-s562kJ font-display-nhmvtD size-24-lFU3ly weight-semibold-uqA4FV reset-IxiVJZ">Ready for more?</h3><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe_footer"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB emailInputOnAccentBackground-TfaCGr input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonOnAccentBackground-vmEt94 buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div></div></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset container-K5XPov"></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="footer-wrap compact-footer"><div class="visibility-check"></div><div class="footer custom-themed-background"><div class="pencraft pc-display-flex pc-gap-12 pc-alignItems-center pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-minWidth-0 pc-reset flex-grow-rzmknG nowrap-QngyoB"><span><span>© 2026 Alex Ewerlöf</span><span> · </span><a href="https://substack.com/privacy" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Privacy</a><span> ∙ </span><a href="https://substack.com/tos" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Terms</a><span> ∙ </span><a href="https://substack.com/ccpa#personal-data-collected" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Collection notice</a></span></div><div class="pencraft pc-display-flex pc-reset"><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-center pc-alignItems-center pc-reset footerButtons-ap9Sk7"><a data-native="true" href="https://substack.com/signup?utm_source=substack&amp;utm_medium=web&amp;utm_content=footer" class="footerSubstackCta-v5HWfj sizeSm-F_aC8H"><svg aria-hidden="true" width="1000" height="1000" viewBox="0 0 1000 1000" fill="#ff6719" stroke-width="1.8" stroke="none" xmlns="http://www.w3.org/2000/svg"><g><path d="M764.166 348.371H236.319V419.402H764.166V348.371Z"></path><path d="M236.319 483.752V813.999L500.231 666.512L764.19 813.999V483.752H236.319Z"></path><path d="M764.166 213H236.319V284.019H764.166V213Z"></path></g></svg> Start your Substack</a><a data-native="true" href="https://substack.com/app/app-store-redirect?utm_campaign=app-marketing&amp;utm_content=web-footer-button" class="footerSubstackCta-v5HWfj getTheApp-Yk3w1O noIcon-z7v9D8 sizeSm-F_aC8H sizeSmNoIcon-KRHkpZ">Get the app</a></div></div><div class="pencraft pc-display-flex pc-minWidth-0 pc-justifyContent-flex-end pc-reset flex-grow-rzmknG"><div translated class="pencraft pc-reset reset-IxiVJZ footer-slogan-blurb"><a href="https://substack.com" data-native="true">Substack</a> is the home for great culture</div></div></div></div></div></div></div><div role="region" aria-label="Notifications (F8)" tabindex="-1" style="pointer-events:none;"><ol tabindex="-1" style="--offset:0px;z-index:1000;" class="viewport-_BM4Bg"></ol></div><div></div><script suppressHydrationWarning nonce="nQGtn4EHptwtQVZSM4a71A">window.__staticRouterHydrationData = JSON.parse("{\"loaderData\":{},\"actionData\":null,\"errors\":null}");</script>
</div>
<script src="https://js.sentry-cdn.com/6c2ff3e3828e4017b7faf7b63e24cdf8.min.js" crossorigin="anonymous"></script>
<script nonce="nQGtn4EHptwtQVZSM4a71A">
window.Sentry && window.Sentry.onLoad(function() {
window.Sentry.init({
environment: window._preloads.sentry_environment,
dsn: window._preloads.sentry_dsn,
})
})
</script>
<script nonce="nQGtn4EHptwtQVZSM4a71A">window._preloads = JSON.parse("{\"cspNonce\":\"nQGtn4EHptwtQVZSM4a71A\",\"isEU\":false,\"language\":\"en\",\"country\":\"US\",\"leaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"enabledLeaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"userLocale\":{\"language\":\"en\",\"region\":\"US\",\"source\":\"accept-language\"},\"base_url\":\"https://blog.alexewerlof.com\",\"stripe_publishable_key\":\"pk_live_51QfnARLDSWi1i85FBpvw6YxfQHljOpWXw8IKi5qFWEzvW8HvoD8cqTulR9UWguYbYweLvA16P7LN6WZsGdZKrNkE00uGbFaOE3\",\"captcha_site_key\":\"6LeI15YsAAAAAPXyDcvuVqipba_jEFQCjz1PFQoz\",\"pub\":{\"apple_pay_disabled\":false,\"apex_domain\":null,\"author_id\":87732486,\"byline_images_enabled\":true,\"bylines_enabled\":true,\"chartable_token\":null,\"community_enabled\":true,\"copyright\":\"Alex Ewerl\u00F6f\",\"cover_photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/ffddf167-cd34-41aa-9334-ad26acca3ed2_600x600.png\",\"created_at\":\"2022-07-14T22:27:58.683Z\",\"custom_domain_optional\":false,\"custom_domain\":\"blog.alexewerlof.com\",\"default_comment_sort\":\"oldest_first\",\"default_coupon\":null,\"default_group_coupon\":\"d8f4bfde\",\"default_show_guest_bios\":true,\"email_banner_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/74f8e318-c490-4be2-b868-df0285fc2f3e_1100x220.png\",\"email_from_name\":\"Alex Ewerl\u00F6f Notes\",\"email_from\":null,\"embed_tracking_disabled\":false,\"explicit\":false,\"expose_paywall_content_to_search_engines\":true,\"fb_pixel_id\":null,\"fb_site_verification_token\":null,\"flagged_as_spam\":false,\"founding_subscription_benefits\":[\"1:1 chat, 60m video call\"],\"free_subscription_benefits\":[\"All the public posts\"],\"ga_pixel_id\":null,\"google_site_verification_token\":\"HIy4u2xUDZkIs6P0KJw9TSYeq7KqXSPAQvZIs9911qU\",\"google_tag_manager_token\":\"\",\"hero_image\":null,\"hero_text\":\"Technical Leadership, Reliability Engineering, Growth\",\"hide_intro_subtitle\":null,\"hide_intro_title\":null,\"hide_podcast_feed_link\":false,\"homepage_type\":\"newspaper\",\"id\":1002265,\"image_thumbnails_always_enabled\":false,\"invite_only\":false,\"hide_podcast_from_pub_listings\":false,\"hide_podcast_from_pub_listings_prior_to\":null,\"language\":\"en\",\"logo_url_wide\":\"https://substackcdn.com/image/fetch/$s_!3m6V!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fbd56dc7c-4b9f-43f2-99a1-ecc995284940_1200x256.png\",\"logo_url\":\"https://substackcdn.com/image/fetch/$s_!_Ur2!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png\",\"minimum_group_size\":2,\"moderation_enabled\":true,\"name\":\"Alex Ewerl\u00F6f Notes\",\"paid_subscription_benefits\":[\"The pro-tips on all articles\",\"Support spending my private time to write\",\"Access to full archive\"],\"parsely_pixel_id\":null,\"chartbeat_domain\":null,\"payments_state\":\"enabled\",\"paywall_free_trial_enabled\":true,\"podcast_art_url\":null,\"paid_podcast_episode_art_url\":null,\"podcast_byline\":null,\"podcast_description\":null,\"podcast_enabled\":false,\"podcast_feed_url\":null,\"podcast_title\":null,\"post_preview_limit\":null,\"primary_user_id\":87732486,\"require_clickthrough\":false,\"show_pub_podcast_tab\":false,\"show_recs_on_homepage\":true,\"subdomain\":\"alexewerlof\",\"subscriber_invites\":0,\"support_email\":null,\"theme_var_background_pop\":\"#786CFF\",\"theme_var_color_links\":false,\"theme_var_cover_bg_color\":null,\"trial_end_override\":null,\"twitter_pixel_id\":null,\"type\":\"newsletter\",\"post_reaction_faces_enabled\":true,\"is_personal_mode\":false,\"plans\":[{\"id\":\"monthly7_5eur\",\"object\":\"plan\",\"active\":true,\"aggregate_usage\":null,\"amount\":750,\"amount_decimal\":\"750\",\"billing_scheme\":\"per_unit\",\"created\":1728579524,\"currency\":\"eur\",\"interval\":\"month\",\"interval_count\":1,\"livemode\":true,\"metadata\":{\"substack\":\"yes\"},\"meter\":null,\"nickname\":\"\u20AC7.50 a month\",\"product\":\"prod_R0QYoetykRH93k\",\"tiers_mode\":null,\"transform_usage\":null,\"trial_period_days\":null,\"usage_type\":\"licensed\",\"currency_options\":{\"aud\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":1300,\"unit_amount_decimal\":\"1300\"},\"brl\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":4400,\"unit_amount_decimal\":\"4400\"},\"cad\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":1300,\"unit_amount_decimal\":\"1300\"},\"chf\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":800,\"unit_amount_decimal\":\"800\"},\"dkk\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":6000,\"unit_amount_decimal\":\"6000\"},\"eur\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":750,\"unit_amount_decimal\":\"750\"},\"gbp\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":700,\"unit_amount_decimal\":\"700\"},\"inr\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":82500,\"unit_amount_decimal\":\"82500\"},\"jpy\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":1500,\"unit_amount_decimal\":\"1500\"},\"mxn\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":15000,\"unit_amount_decimal\":\"15000\"},\"nok\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":8500,\"unit_amount_decimal\":\"8500\"},\"nzd\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":1500,\"unit_amount_decimal\":\"1500\"},\"pln\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":3300,\"unit_amount_decimal\":\"3300\"},\"sek\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":8500,\"unit_amount_decimal\":\"8500\"},\"usd\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":900,\"unit_amount_decimal\":\"900\"}}},{\"id\":\"yearly74eur\",\"object\":\"plan\",\"active\":true,\"aggregate_usage\":null,\"amount\":7400,\"amount_decimal\":\"7400\",\"billing_scheme\":\"per_unit\",\"created\":1728579524,\"currency\":\"eur\",\"interval\":\"year\",\"interval_count\":1,\"livemode\":true,\"metadata\":{\"substack\":\"yes\"},\"meter\":null,\"nickname\":\"\u20AC74 a year\",\"product\":\"prod_R0QYUJf9xqVpaq\",\"tiers_mode\":null,\"transform_usage\":null,\"trial_period_days\":null,\"usage_type\":\"licensed\",\"currency_options\":{\"aud\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":12500,\"unit_amount_decimal\":\"12500\"},\"brl\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":43500,\"unit_amount_decimal\":\"43500\"},\"cad\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":12000,\"unit_amount_decimal\":\"12000\"},\"chf\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":7000,\"unit_amount_decimal\":\"7000\"},\"dkk\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":55500,\"unit_amount_decimal\":\"55500\"},\"eur\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":7400,\"unit_amount_decimal\":\"7400\"},\"gbp\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":6500,\"unit_amount_decimal\":\"6500\"},\"inr\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":811000,\"unit_amount_decimal\":\"811000\"},\"jpy\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":14000,\"unit_amount_decimal\":\"14000\"},\"mxn\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":148000,\"unit_amount_decimal\":\"148000\"},\"nok\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":81500,\"unit_amount_decimal\":\"81500\"},\"nzd\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":15000,\"unit_amount_decimal\":\"15000\"},\"pln\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":32500,\"unit_amount_decimal\":\"32500\"},\"sek\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":82000,\"unit_amount_decimal\":\"82000\"},\"usd\":{\"custom_unit_amount\":null,\"tax_behavior\":\"unspecified\",\"unit_amount\":8500,\"unit_amount_decimal\":\"8500\"}}},{\"id\":\"founding20000eur\",\"name\":\"founding20000eur\",\"nickname\":\"founding20000eur\",\"active\":true,\"amount\":20000,\"currency\":\"eur\",\"interval\":\"year\",\"interval_count\":1,\"metadata\":{\"substack\":\"yes\",\"founding\":\"yes\",\"no_coupons\":\"yes\",\"short_description\":\"VIP supporter\",\"short_description_english\":\"VIP supporter\",\"minimum\":\"20000\",\"minimum_local\":{\"usd\":23500,\"gbp\":17500,\"eur\":20000,\"cad\":32500,\"mxn\":394000,\"aud\":32500,\"brl\":119000,\"sek\":225500,\"chf\":19000,\"dkk\":149500,\"nok\":215500,\"inr\":2217000,\"jpy\":36000,\"nzd\":40000,\"pln\":86500}},\"currency_options\":{\"usd\":{\"unit_amount\":23500,\"tax_behavior\":\"unspecified\"},\"gbp\":{\"unit_amount\":17500,\"tax_behavior\":\"unspecified\"},\"eur\":{\"unit_amount\":20000,\"tax_behavior\":\"unspecified\"},\"cad\":{\"unit_amount\":32500,\"tax_behavior\":\"unspecified\"},\"mxn\":{\"unit_amount\":394000,\"tax_behavior\":\"unspecified\"},\"aud\":{\"unit_amount\":32500,\"tax_behavior\":\"unspecified\"},\"brl\":{\"unit_amount\":119000,\"tax_behavior\":\"unspecified\"},\"sek\":{\"unit_amount\":225500,\"tax_behavior\":\"unspecified\"},\"chf\":{\"unit_amount\":19000,\"tax_behavior\":\"unspecified\"},\"dkk\":{\"unit_amount\":149500,\"tax_behavior\":\"unspecified\"},\"nok\":{\"unit_amount\":215500,\"tax_behavior\":\"unspecified\"},\"inr\":{\"unit_amount\":2217000,\"tax_behavior\":\"unspecified\"},\"jpy\":{\"unit_amount\":36000,\"tax_behavior\":\"unspecified\"},\"nzd\":{\"unit_amount\":40000,\"tax_behavior\":\"unspecified\"},\"pln\":{\"unit_amount\":86500,\"tax_behavior\":\"unspecified\"}}}],\"stripe_user_id\":\"acct_1MAFmbEv9TeVut3s\",\"stripe_country\":\"SE\",\"stripe_publishable_key\":\"pk_live_51MAFmbEv9TeVut3sg6LOLDoGpzdvqtqZvasczL18isiPGcezpoD3DRMPDmivD2Qu12m0h0mpKUgIqx5cJZisTK1500fjcalFnx\",\"stripe_platform_account\":\"US\",\"automatic_tax_enabled\":false,\"author_name\":\"Alex Ewerl\u00F6f\",\"author_handle\":\"alexewerlof\",\"author_photo_url\":\"https://substackcdn.com/image/fetch/$s_!IRtW!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg\",\"author_bio\":\"Writes about technical leadership, growth mindset, and system reliability engineering. Senior Staff Engineer, MSc Systems Engineering from KTH, Stockholmer, dad, amateur artist. Read more here: https://www.alexewerlof.com/who\",\"twitter_screen_name\":\"alexewerlof\",\"has_custom_tos\":false,\"has_custom_privacy\":false,\"theme\":{\"background_pop_color\":\"#FFBC52\",\"web_bg_color\":\"#001426\",\"cover_bg_color\":\"#001426\",\"publication_id\":1002265,\"color_links\":null,\"font_preset_heading\":\"sans\",\"font_preset_body\":\"fancy_serif\",\"font_family_headings\":null,\"font_family_body\":null,\"font_family_ui\":null,\"font_size_body_desktop\":null,\"print_secondary\":null,\"custom_css_web\":null,\"custom_css_email\":null,\"home_hero\":\"feature-media\",\"home_posts\":\"custom\",\"home_show_top_posts\":true,\"hide_images_from_list\":false,\"home_hero_alignment\":\"left\",\"home_hero_show_podcast_links\":true,\"default_post_header_variant\":null,\"custom_header\":{\"layout\":\"stacked\",\"navStyle\":\"text\",\"navPosition\":\"bottom\",\"wordmarkLogoSize\":42},\"custom_footer\":{\"layout\":\"two-column\",\"backgroundColor\":\"#000C17\",\"publicationNameSize\":50,\"showPublicationName\":false,\"publicationNameStyle\":\"text\",\"publicationNamePosition\":\"bottom\",\"publicationNameBackgroundColor\":null},\"social_media_links\":{\"bluesky\":\"https://bsky.app/profile/alexewerlof.com\",\"youtube\":\"https://www.youtube.com/@alexewerlof\",\"linkedin\":\"https://www.linkedin.com/in/alexewerlof/\"},\"font_options\":{\"body\":\"merriweather_400\",\"heading\":\"heavy_sans\"},\"section_template\":null,\"custom_subscribe\":null,\"design_template\":null,\"design_template_options\":null},\"threads_v2_settings\":{\"first_thread_email_sent_at\":\"2023-02-12T20:30:37.079+00:00\",\"activated_at\":\"2023-02-12T20:30:06.164+00:00\",\"reader_thread_notifications_enabled\":true,\"create_thread_minimum_role\":\"paid\",\"photo_replies_enabled\":true,\"boost_free_subscriber_chat_preview_enabled\":false,\"push_suppression_enabled\":false},\"default_group_coupon_percent_off\":\"50.00\",\"default_group_coupon_include_founding\":false,\"pause_return_date\":null,\"has_posts\":true,\"has_recommendations\":true,\"first_post_date\":\"2020-09-24T11:28:00.000Z\",\"has_podcast\":false,\"has_free_podcast\":false,\"has_subscriber_only_podcast\":false,\"has_community_content\":true,\"rankingDetail\":\"Launched 4 years ago\",\"rankingDetailFreeIncluded\":\"Tens of thousands of subscribers\",\"rankingDetailOrderOfMagnitude\":10,\"rankingDetailFreeIncludedOrderOfMagnitude\":10000,\"rankingDetailFreeSubscriberCount\":\"Over 19,000 subscribers\",\"rankingDetailByLanguage\":{\"ar\":{\"rankingDetail\":\"\u062A\u0645 \u0627\u0644\u0625\u0637\u0644\u0627\u0642 4 years ago\"},\"ca\":{\"rankingDetail\":\"S\u2019ha llan\u00E7at fa 4 anys\"},\"da\":{\"rankingDetail\":\"Lancering 4 \u00E5r\"},\"de\":{\"rankingDetail\":\"vor 4 Jahren gelauncht\"},\"es\":{\"rankingDetail\":\"Lanzado hace 4 a\u00F1os\"},\"fr\":{\"rankingDetail\":\"Lanc\u00E9 il y a 4 ann\u00E9es\"},\"id\":{\"rankingDetail\":\"Diluncurkan 4 tahun yang lalu\"},\"ja\":{\"rankingDetail\":\"\u958B\u59CB\u65E5 4\u5E74\u524D\"},\"ko\":{\"rankingDetail\":\"4\uB144 \uC804\uC5D0 \uCD9C\uC2DC\"},\"nb\":{\"rankingDetail\":\"Lansert 4 \u00E5r\"},\"nl\":{\"rankingDetail\":\"Gelanceerd 4 jaar geleden\"},\"pl\":{\"rankingDetail\":\"Uruchomiono 4 lat temu\"},\"pt\":{\"rankingDetail\":\"Lan\u00E7ado 4 anos\"},\"pt-br\":{\"rankingDetail\":\"Lan\u00E7ado 4 anos\"},\"en-gb\":{\"rankingDetail\":\"Launched 4 years ago\"},\"it\":{\"rankingDetail\":\"Lanciato 4 anni\"},\"tr\":{\"rankingDetail\":\"4 y\u0131l ba\u015Flat\u0131ld\u0131\"},\"sv\":{\"rankingDetail\":\"Lanserad 4 \u00E5r sedan\"},\"fi\":{\"rankingDetail\":\"Launched 4 vuotta\"},\"is\":{\"rankingDetail\":\"Launched 4 \u00E1r\"},\"zh-hans\":{\"rankingDetail\":\"\u4E8E4 \u5E74\u524D\u4E0A\u7EBF\"},\"zh-hant\":{\"rankingDetail\":\"\u65BC 4 \u5E74\u524D \u63A8\u51FA\"},\"en\":{\"rankingDetail\":\"Launched 4 years ago\"}},\"freeSubscriberCount\":\"19,000\",\"freeSubscriberCountOrderOfMagnitude\":\"19K+\",\"author_bestseller_tier\":0,\"author_badge\":null,\"disable_monthly_subscriptions\":false,\"disable_annual_subscriptions\":false,\"hide_post_restacks\":false,\"notes_feed_enabled\":true,\"showIntroModule\":false,\"isPortraitLayout\":false,\"last_chat_post_at\":\"2025-10-31T11:34:19.438Z\",\"primary_profile_name\":\"Alex Ewerl\u00F6f\",\"primary_profile_photo_url\":\"https://substackcdn.com/image/fetch/$s_!IRtW!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg\",\"no_follow\":false,\"sponsorshipCampaigns\":{},\"paywall_chat\":\"free\",\"sections\":[{\"id\":67726,\"created_at\":\"2023-05-31T20:13:23.606Z\",\"updated_at\":\"2024-11-12T15:54:54.980Z\",\"publication_id\":1002265,\"name\":\"Technical Leadership\",\"description\":\"Technical Leadership, Staff Engineering, and impact without mandate\",\"slug\":\"tech-lead\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":0,\"port_status\":\"success\",\"logo_url\":null,\"hide_from_navbar\":false,\"email_from_name\":\"\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"unifiedPodcastSettings\":null,\"podcastSettings\":null,\"showLinks\":[],\"pageTheme\":{\"id\":51608,\"publication_id\":1002265,\"section_id\":67726,\"page\":null,\"page_hero\":\"default\",\"page_posts\":\"grid\",\"show_podcast_links\":true,\"hero_alignment\":\"center\"},\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null},{\"id\":67723,\"created_at\":\"2023-05-31T19:57:24.859Z\",\"updated_at\":\"2024-03-22T13:16:12.203Z\",\"publication_id\":1002265,\"name\":\"Reliability Engineering\",\"description\":\"Concepts that help you with building reliable systems and architecture\",\"slug\":\"sre\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":1,\"port_status\":\"success\",\"logo_url\":null,\"hide_from_navbar\":false,\"email_from_name\":\"\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"unifiedPodcastSettings\":null,\"podcastSettings\":null,\"showLinks\":[],\"pageTheme\":{\"id\":51609,\"publication_id\":1002265,\"section_id\":67723,\"page\":null,\"page_hero\":\"default\",\"page_posts\":\"grid\",\"show_podcast_links\":true,\"hero_alignment\":\"center\"},\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null},{\"id\":69986,\"created_at\":\"2023-06-17T13:00:28.926Z\",\"updated_at\":\"2023-06-17T13:00:28.926Z\",\"publication_id\":1002265,\"name\":\"Growth\",\"description\":\"Growth mindset, productivity, organizational growth, culture\",\"slug\":\"growth\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":2,\"port_status\":\"success\",\"logo_url\":null,\"hide_from_navbar\":false,\"email_from_name\":\"\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"unifiedPodcastSettings\":null,\"podcastSettings\":null,\"showLinks\":[],\"pageTheme\":{\"id\":51610,\"publication_id\":1002265,\"section_id\":69986,\"page\":null,\"page_hero\":\"default\",\"page_posts\":\"grid\",\"show_podcast_links\":true,\"hero_alignment\":\"center\"},\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null},{\"id\":65305,\"created_at\":\"2023-05-14T08:17:44.620Z\",\"updated_at\":\"2023-05-14T08:17:44.620Z\",\"publication_id\":1002265,\"name\":\"Cartoons\",\"description\":\"The cartoons and doodles I draw from time to time\",\"slug\":\"cartoons\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":3,\"port_status\":\"success\",\"logo_url\":null,\"hide_from_navbar\":false,\"email_from_name\":\"\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"unifiedPodcastSettings\":null,\"podcastSettings\":null,\"showLinks\":[],\"pageTheme\":{\"id\":51611,\"publication_id\":1002265,\"section_id\":65305,\"page\":null,\"page_hero\":\"default\",\"page_posts\":\"grid\",\"show_podcast_links\":true,\"hero_alignment\":\"center\"},\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null},{\"id\":192721,\"created_at\":\"2025-02-23T16:44:05.979Z\",\"updated_at\":\"2025-02-23T16:44:09.931Z\",\"publication_id\":1002265,\"name\":\"Code\",\"description\":\"Code and software development\",\"slug\":\"code\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":4,\"port_status\":\"success\",\"logo_url\":null,\"hide_from_navbar\":false,\"email_from_name\":\"\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"unifiedPodcastSettings\":null,\"podcastSettings\":null,\"showLinks\":[],\"pageTheme\":{\"id\":51612,\"publication_id\":1002265,\"section_id\":192721,\"page\":null,\"page_hero\":\"default\",\"page_posts\":\"grid\",\"show_podcast_links\":true,\"hero_alignment\":\"center\"},\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null}],\"podcastTabInfo\":null,\"multipub_migration\":null,\"navigationBarItems\":[{\"id\":\"693b47dd-b5e5-4dbb-b7b3-891c9b2c1835\",\"publication_id\":1002265,\"sibling_rank\":0,\"link_title\":null,\"link_url\":null,\"section_id\":null,\"post_id\":null,\"is_hidden\":false,\"standard_key\":\"about\",\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":null},{\"id\":\"52ced5af-109a-4b1e-8910-89961a72b0a3\",\"publication_id\":1002265,\"sibling_rank\":0,\"link_title\":null,\"link_url\":null,\"section_id\":null,\"post_id\":null,\"is_hidden\":true,\"standard_key\":\"leaderboard\",\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":null},{\"id\":\"5657cede-0e4c-4869-8f2d-c73ebe2b3338\",\"publication_id\":1002265,\"sibling_rank\":0,\"link_title\":null,\"link_url\":null,\"section_id\":null,\"post_id\":null,\"is_hidden\":true,\"standard_key\":\"notes\",\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":null},{\"id\":\"d020acfa-0f0b-4afb-a793-8626297a59aa\",\"publication_id\":1002265,\"sibling_rank\":1,\"link_title\":null,\"link_url\":null,\"section_id\":null,\"post_id\":null,\"is_hidden\":true,\"standard_key\":\"chat\",\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":null},{\"id\":\"fc613bc6-10ab-41ef-9d39-a6a818fbb5ba\",\"publication_id\":1002265,\"sibling_rank\":2,\"link_title\":null,\"link_url\":null,\"section_id\":67726,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":{\"id\":67726,\"slug\":\"tech-lead\",\"name\":\"Technical Leadership\",\"hide_from_navbar\":false,\"is_podcast\":false,\"logo_url\":null,\"newsletter_enabled\":null}},{\"id\":\"68d82a75-97a1-498b-b089-b9d606ee0f27\",\"publication_id\":1002265,\"sibling_rank\":3,\"link_title\":null,\"link_url\":null,\"section_id\":67723,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":{\"id\":67723,\"slug\":\"sre\",\"name\":\"Reliability Engineering\",\"hide_from_navbar\":false,\"is_podcast\":false,\"logo_url\":null,\"newsletter_enabled\":null}},{\"id\":\"8a522501-651b-45bc-b881-3e2a2497ec29\",\"publication_id\":1002265,\"sibling_rank\":4,\"link_title\":null,\"link_url\":null,\"section_id\":69986,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":{\"id\":69986,\"slug\":\"growth\",\"name\":\"Growth\",\"hide_from_navbar\":false,\"is_podcast\":false,\"logo_url\":null,\"newsletter_enabled\":null}},{\"id\":\"702039d0-d1f1-4070-adb7-6a88110ac904\",\"publication_id\":1002265,\"sibling_rank\":5,\"link_title\":null,\"link_url\":null,\"section_id\":65305,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":{\"id\":65305,\"slug\":\"cartoons\",\"name\":\"Cartoons\",\"hide_from_navbar\":false,\"is_podcast\":false,\"logo_url\":null,\"newsletter_enabled\":null}},{\"id\":\"71e27c1f-1d9d-486d-bcce-3b707aedb0f6\",\"publication_id\":1002265,\"sibling_rank\":8,\"link_title\":\"FAQ\",\"link_url\":\"\",\"section_id\":null,\"post_id\":141786627,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":{\"id\":141786627,\"slug\":\"faq\",\"title\":\"FAQ\",\"type\":\"page\"},\"section\":null},{\"id\":\"c3e7d81f-5147-498f-8317-72f318b996c3\",\"publication_id\":1002265,\"sibling_rank\":9999,\"link_title\":null,\"link_url\":null,\"section_id\":192721,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"postTag\":null,\"children\":[],\"post\":null,\"section\":{\"id\":192721,\"slug\":\"code\",\"name\":\"Code\",\"hide_from_navbar\":false,\"is_podcast\":false,\"logo_url\":null,\"newsletter_enabled\":null}}],\"has_active_perks\":false,\"contributors\":[{\"name\":\"Alex Ewerl\u00F6f\",\"handle\":\"alexewerlof\",\"role\":\"admin\",\"owner\":true,\"user_id\":87732486,\"photo_url\":\"https://substackcdn.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg\",\"bio\":\"Writes about technical leadership, growth mindset, and system reliability engineering. Senior Staff Engineer, MSc Systems Engineering from KTH, Stockholmer, dad, amateur artist. Read more here: https://www.alexewerlof.com/who\",\"status\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null}}],\"threads_v2_enabled\":true,\"viralGiftsConfig\":{\"id\":\"549946e0-1129-4592-ba5f-6bcab8c6cb5d\",\"publication_id\":1002265,\"enabled\":true,\"gifts_per_user\":5,\"gift_length_months\":1,\"send_extra_gifts\":true,\"message\":\"Technical Leadership, personal growth, software\",\"created_at\":\"2022-12-01T16:46:31.467333+00:00\",\"updated_at\":\"2022-12-01T16:46:31.467333+00:00\",\"days_til_invite\":14,\"send_emails\":true,\"show_link\":null},\"no_index\":false,\"tier\":2,\"can_set_google_site_verification\":true,\"can_have_sitemap\":true,\"iap_advanced_plans\":[{\"sku\":\"c1LuwhX14kmFN5UK5J\",\"publication_id\":\"1002265\",\"is_active\":true,\"price_base_units\":1200,\"currency_alpha3\":\"usd\",\"period\":\"month\",\"created_at\":\"2026-07-07T19:37:06.872Z\",\"updated_at\":\"2026-07-07T19:37:06.872Z\",\"id\":\"1272592\",\"payout_amount_base_units\":86,\"alternate_currencies\":{\"aud\":1800,\"brl\":6500,\"cad\":1800,\"chf\":1000,\"dkk\":8000,\"eur\":1100,\"gbp\":900,\"mxn\":21000,\"nok\":12000,\"nzd\":2200,\"pln\":4500,\"sek\":12000},\"is_founding\":false,\"display_name\":\"Alex Ewerl\u00F6f Notes (Monthly)\",\"display_price\":\"$12\"},{\"sku\":\"MC73cNbFZtu7O35fUc\",\"publication_id\":\"1002265\",\"is_active\":true,\"price_base_units\":11500,\"currency_alpha3\":\"usd\",\"period\":\"year\",\"created_at\":\"2026-07-07T19:37:06.880Z\",\"updated_at\":\"2026-07-07T19:37:06.880Z\",\"id\":\"1272593\",\"payout_amount_base_units\":847,\"alternate_currencies\":{\"aud\":17000,\"brl\":59500,\"cad\":16500,\"chf\":9500,\"dkk\":75500,\"eur\":10500,\"gbp\":9000,\"mxn\":200000,\"nok\":112500,\"nzd\":20500,\"pln\":43500,\"sek\":111000},\"is_founding\":false,\"display_name\":\"Alex Ewerl\u00F6f Notes (Yearly)\",\"display_price\":\"$115\"}],\"iap_founding_plan\":{\"base_plan_id\":\"founding20000eur\",\"name\":\"VIP supporter\",\"minimum_amount\":27500,\"suggested_amount\":27500,\"currency_alpha3\":\"usd\",\"alternate_currencies\":{\"gbp\":{\"minimum_amount\":20500,\"suggested_amount\":20500},\"eur\":{\"minimum_amount\":24000,\"suggested_amount\":24000},\"cad\":{\"minimum_amount\":38500,\"suggested_amount\":38500},\"mxn\":{\"minimum_amount\":467000,\"suggested_amount\":467000},\"aud\":{\"minimum_amount\":38500,\"suggested_amount\":38500},\"brl\":{\"minimum_amount\":141000,\"suggested_amount\":141000},\"sek\":{\"minimum_amount\":267000,\"suggested_amount\":267000},\"chf\":{\"minimum_amount\":22500,\"suggested_amount\":22500},\"dkk\":{\"minimum_amount\":177500,\"suggested_amount\":177500},\"nok\":{\"minimum_amount\":255500,\"suggested_amount\":255500},\"inr\":{\"minimum_amount\":2628500,\"suggested_amount\":2628500},\"jpy\":{\"minimum_amount\":42500,\"suggested_amount\":42500},\"nzd\":{\"minimum_amount\":47500,\"suggested_amount\":47500},\"pln\":{\"minimum_amount\":103000,\"suggested_amount\":103000}}},\"founding_plan_name_english\":\"VIP supporter\",\"base_url\":\"https://blog.alexewerlof.com\",\"hostname\":\"blog.alexewerlof.com\",\"is_on_substack\":false,\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"pageThemes\":{\"podcast\":null},\"supports_ip_content_unlock\":false,\"appTheme\":{\"colors\":{\"accent\":{\"name\":\"#ffbc52\",\"primary\":{\"r\":222,\"g\":158,\"b\":52,\"a\":1},\"primary_hover\":{\"r\":201,\"g\":140,\"b\":31,\"a\":1},\"primary_elevated\":{\"r\":201,\"g\":140,\"b\":31,\"a\":1},\"secondary\":{\"r\":222,\"g\":158,\"b\":52,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":222,\"g\":158,\"b\":52,\"a\":0.2},\"bg_hover\":{\"r\":222,\"g\":158,\"b\":52,\"a\":0.3},\"dark\":{\"primary\":{\"r\":255,\"g\":188,\"b\":82,\"a\":1},\"primary_hover\":{\"r\":255,\"g\":210,\"b\":125,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":210,\"b\":125,\"a\":1},\"secondary\":{\"r\":255,\"g\":188,\"b\":82,\"a\":0.2},\"contrast\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.8},\"bg\":{\"r\":255,\"g\":188,\"b\":82,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":188,\"b\":82,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.8},\"secondary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.6},\"tertiary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.4},\"accent\":{\"r\":151,\"g\":97,\"b\":0,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":255,\"g\":188,\"b\":82,\"a\":1}}},\"bg\":{\"name\":\"#0085e5\",\"hue\":{\"r\":0,\"g\":133,\"b\":229,\"a\":1},\"tint\":{\"r\":0,\"g\":133,\"b\":229,\"a\":0.1},\"primary\":{\"r\":229.5,\"g\":242.8,\"b\":252.39999999999998,\"a\":1},\"primary_hover\":{\"r\":225,\"g\":238.29999999999998,\"b\":247.9,\"a\":1},\"primary_elevated\":{\"r\":225,\"g\":238.29999999999998,\"b\":247.9,\"a\":1},\"secondary\":{\"r\":214.2,\"g\":227.5,\"b\":237.1,\"a\":1},\"secondary_elevated\":{\"r\":184.9454280083347,\"g\":197.91521411644928,\"b\":207.26517218866366,\"a\":1},\"tertiary\":{\"r\":197.1,\"g\":210.4,\"b\":219.99999999999997,\"a\":1},\"quaternary\":{\"r\":163.79999999999998,\"g\":177.10000000000002,\"b\":186.70000000000002,\"a\":1},\"dark\":{\"primary\":{\"r\":19.8,\"g\":34,\"b\":44.5,\"a\":1},\"primary_hover\":{\"r\":24.3,\"g\":38.50000000000001,\"b\":48.99999999999999,\"a\":1},\"primary_elevated\":{\"r\":24.3,\"g\":38.50000000000001,\"b\":48.99999999999999,\"a\":1},\"secondary\":{\"r\":31.5,\"g\":46.60000000000001,\"b\":56.2,\"a\":1},\"secondary_elevated\":{\"r\":38.73972711589789,\"g\":54.076229834048654,\"b\":63.89709451363946,\"a\":1},\"tertiary\":{\"r\":48.6,\"g\":62.8,\"b\":72.4,\"a\":1},\"quaternary\":{\"r\":81.00000000000001,\"g\":95.2,\"b\":104.80000000000001,\"a\":1}}}}},\"portalAppTheme\":{\"colors\":{\"accent\":{\"name\":\"#FFBC52\",\"primary\":{\"r\":255,\"g\":188,\"b\":82,\"a\":1},\"primary_hover\":{\"r\":255,\"g\":178,\"b\":56,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":188,\"b\":82,\"a\":1},\"secondary\":{\"r\":255,\"g\":188,\"b\":82,\"a\":1},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3},\"dark\":{\"name\":\"#FFBC52\",\"primary\":{\"r\":255,\"g\":188,\"b\":82,\"a\":1},\"primary_hover\":{\"r\":255,\"g\":178,\"b\":56,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":188,\"b\":82,\"a\":1},\"secondary\":{\"r\":255,\"g\":188,\"b\":82,\"a\":1},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":226,\"g\":232,\"b\":237,\"a\":1},\"secondary\":{\"r\":153,\"g\":161,\"b\":168,\"a\":1},\"tertiary\":{\"r\":109,\"g\":120,\"b\":130,\"a\":1},\"accent\":{\"r\":255,\"g\":188,\"b\":82,\"a\":1},\"dark\":{\"primary\":{\"r\":226,\"g\":232,\"b\":237,\"a\":1},\"secondary\":{\"r\":153,\"g\":161,\"b\":168,\"a\":1},\"tertiary\":{\"r\":109,\"g\":120,\"b\":130,\"a\":1},\"accent\":{\"r\":255,\"g\":188,\"b\":82,\"a\":1}}},\"bg\":{\"name\":\"#001426\",\"hue\":{\"r\":0,\"g\":20,\"b\":38,\"a\":1},\"tint\":{\"r\":0,\"g\":20,\"b\":38,\"a\":1},\"primary\":{\"r\":0,\"g\":20,\"b\":38,\"a\":1},\"primary_hover\":{\"r\":15,\"g\":34,\"b\":51,\"a\":1},\"primary_elevated\":{\"r\":15,\"g\":34,\"b\":51,\"a\":1},\"secondary\":{\"r\":15,\"g\":34,\"b\":51,\"a\":1},\"secondary_elevated\":{\"r\":34,\"g\":52,\"b\":67,\"a\":1},\"tertiary\":{\"r\":34,\"g\":52,\"b\":67,\"a\":1},\"quaternary\":{\"r\":72,\"g\":86,\"b\":99,\"a\":1},\"dark\":{\"name\":\"#001426\",\"hue\":{\"r\":0,\"g\":20,\"b\":38,\"a\":1},\"tint\":{\"r\":0,\"g\":20,\"b\":38,\"a\":1},\"primary\":{\"r\":0,\"g\":20,\"b\":38,\"a\":1},\"primary_hover\":{\"r\":15,\"g\":34,\"b\":51,\"a\":1},\"primary_elevated\":{\"r\":15,\"g\":34,\"b\":51,\"a\":1},\"secondary\":{\"r\":15,\"g\":34,\"b\":51,\"a\":1},\"secondary_elevated\":{\"r\":34,\"g\":52,\"b\":67,\"a\":1},\"tertiary\":{\"r\":34,\"g\":52,\"b\":67,\"a\":1},\"quaternary\":{\"r\":72,\"g\":86,\"b\":99,\"a\":1}}},\"wordmark_bg\":{\"r\":0,\"g\":20,\"b\":38,\"a\":1},\"is_dark\":true},\"fonts\":{\"heading\":\"heavy_sans\",\"body\":\"merriweather_400\"}},\"logoPalette\":{\"Vibrant\":{\"rgb\":[252,188,84],\"population\":5509},\"DarkVibrant\":{\"rgb\":[4,20,36],\"population\":556},\"LightVibrant\":{\"rgb\":[244,181,84],\"population\":7},\"Muted\":{\"rgb\":[124,100,60],\"population\":1},\"DarkMuted\":{\"rgb\":[80,71,52],\"population\":65},\"LightMuted\":{\"rgb\":[143.75274725274724,90.7912087912088,9.247252747252743],\"population\":0}},\"section\":{\"id\":67723,\"created_at\":\"2023-05-31T19:57:24.859Z\",\"updated_at\":\"2024-03-22T13:16:12.203Z\",\"publication_id\":1002265,\"name\":\"Reliability Engineering\",\"description\":\"Concepts that help you with building reliable systems and architecture\",\"slug\":\"sre\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":1,\"port_status\":\"success\",\"logo_url\":null,\"hide_from_navbar\":false,\"email_from_name\":\"\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"unifiedPodcastSettings\":null,\"podcastSettings\":null,\"showLinks\":[],\"pageTheme\":{\"id\":51609,\"publication_id\":1002265,\"section_id\":67723,\"page\":null,\"page_hero\":\"default\",\"page_posts\":\"grid\",\"show_podcast_links\":true,\"hero_alignment\":\"center\"},\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null}},\"confirmedLogin\":false,\"hide_intro_popup\":false,\"block_auto_login\":false,\"domainInfo\":{\"isSubstack\":false,\"customDomain\":\"blog.alexewerlof.com\"},\"experimentFeatures\":{},\"experimentExposures\":{},\"siteConfigs\":{\"score_upsell_email\":\"control\",\"first_chat_email_enabled\":true,\"new_commenter_approval\":false,\"pub_update_opennode_api_key\":false,\"notes_video_max_duration_minutes\":15,\"show_content_label_age_gating_in_feed\":false,\"zendesk_automation_cancellations\":false,\"mfa_action_box_enabled\":false,\"publication_max_bylines\":35,\"no_contest_charge_disputes\":false,\"scheduled_post_share_assets_enabled\":false,\"publication_tabs_reorder\":false,\"comp_expiry_email_new_copy\":\"NONE\",\"free_unlock_required\":false,\"enable_section_group_pages\":false,\"enable_post_summarization\":false,\"live_stream_host_warning_message\":\"\",\"bitcoin_enabled\":false,\"minimum_ios_os_version\":\"17.0.0\",\"show_entire_square_image\":false,\"hide_subscriber_count\":false,\"fit_in_live_stream_player\":false,\"publication_author_display_override\":\"\",\"enable_saved_segments\":true,\"generate_pdf_tax_report\":false,\"hide_post_sidebar\":false,\"show_generic_post_importer\":false,\"enable_pledges_modal\":true,\"community_automod_non_english_enabled\":true,\"audio_clips_enabled\":false,\"enable_videos_page\":false,\"exempt_from_gtm_filter\":false,\"group_sections_and_podcasts_in_menu\":false,\"boost_optin_modal_enabled\":true,\"standards_and_enforcement_features_enabled\":false,\"pub_creation_captcha_behavior\":\"risky_pubs_or_rate_limit\",\"post_blogspot_importer\":false,\"enable_high_res_background_uploading\":false,\"pub_tts_override\":\"default\",\"disable_monthly_subscriptions\":false,\"skip_welcome_email\":false,\"chat_reader_thread_notification_default\":false,\"scheduled_pinned_posts\":false,\"disable_redirect_outbound_utm_params\":false,\"reader_gift_referrals_enabled\":true,\"dont_show_guest_byline\":false,\"like_comments_enabled\":true,\"enable_theme_editor_deferred_saves\":false,\"enable_sponsorship_campaigns_advanced\":false,\"temporal_livestream_ended_draft\":true,\"enable_pangram_ai_detection\":true,\"fallback_to_archive_search_on_section_pages\":false,\"livekit_track_egress_custom_base_url\":\"http://livekit-egress-custom-recorder-participant-test.s3-website-us-east-1.amazonaws.com\",\"welcome_screen_blurb_override\":\"\",\"like_posts_enabled\":true,\"twitter_player_card_enabled\":true,\"feed_promoted_user\":false,\"show_note_stats_for_all_notes\":false,\"section_specific_csv_imports_enabled\":false,\"disable_podcast_feed_description_cta\":false,\"bypass_profile_substack_logo_detection\":false,\"use_preloaded_player_sources\":false,\"list_pruning_enabled\":false,\"facebook_connect\":false,\"opt_in_to_sections_during_subscribe\":false,\"underlined_colored_links\":false,\"enable_efficient_digest_embed\":false,\"enable_aligned_images\":false,\"max_image_upload_mb\":64,\"threads_suggested_ios_version\":null,\"pledges_disabled\":false,\"threads_minimum_ios_version\":812,\"hide_podcast_email_setup_link\":false,\"subscribe_captcha_behavior\":\"default\",\"publication_ban_sample_rate\":0,\"custom_themes_substack_subscribe_modal\":false,\"ios_post_share_assets_screenshot_trigger\":\"control\",\"opt_in_to_sections_during_subscribe_include_main_pub_newsletter\":false,\"continue_support_cta_in_newsletter_emails\":false,\"bloomberg_syndication_enabled\":false,\"allow_document_freeze\":false,\"test_age_gate_user\":false,\"podcast_main_feed_is_firehose\":false,\"pub_app_incentive_gift\":\"\",\"no_embed_redirect\":false,\"customized_email_from_name_for_new_follow_emails\":\"treatment\",\"spotify_open_access_sandbox_mode\":false,\"disable_custom_nav_menu\":false,\"enable_livestream_name_cards\":false,\"fullstory_enabled\":false,\"chat_reply_poll_interval\":3,\"thefp_enable_email_upsell_banner\":false,\"force_pub_links_to_use_subdomain\":false,\"always_show_cookie_banner\":false,\"hide_media_download_option\":false,\"hide_post_restacks\":false,\"feed_item_source_debug_mode\":false,\"twitter_embed_video_playback_enabled\":false,\"publication_homepage_title_display_override\":\"\",\"enable_email_automation_stats\":false,\"post_preview_highlight_byline\":false,\"4k_video\":false,\"enable_islands_section_intent_screen\":false,\"post_metering_enabled\":false,\"notifications_disabled\":\"\",\"cross_post_notification_threshold\":1000,\"facebook_connect_prod_app\":true,\"force_into_pymk_ranking\":false,\"minimum_android_version\":756,\"group_size_change_notification_email_enabled\":false,\"enable_transcription_translations\":false,\"use_og_image_as_twitter_image_for_post_previews\":false,\"always_use_podcast_channel_art_as_episode_art_in_rss\":false,\"seo_tier_override\":\"NONE\",\"editor_role_enabled\":false,\"no_follow_links\":false,\"publisher_api_enabled\":false,\"zendesk_support_priority\":\"default\",\"enable_post_clips_stats\":false,\"enable_subscriber_referrals_awards\":true,\"ios_profile_themes_feed_permalink_enabled\":false,\"reader_survey_during_subscribe_enabled\":true,\"use_publication_language_for_transcription\":false,\"show_substack_funded_gifts_tooltip\":true,\"disable_ai_transcription\":false,\"per_pub_notification_settings\":false,\"live_stream_creation_enabled\":true,\"disable_card_element_in_europe\":false,\"web_growth_item_promotion_threshold\":0,\"bundle_subscribe_enabled\":false,\"web_vitals_sample_rate\":0,\"allow_live_stream_auto_takedown\":\"true\",\"mobile_publication_attachments_enabled\":false,\"enable_design_templates_events_page\":false,\"enable_survey_response_crm_filters\":false,\"disable_personal_substack_initialization\":false,\"section_specific_welcome_pages\":false,\"local_payment_methods\":\"control\",\"publisher_api_cancel_comp\":false,\"posts_in_rss_feed\":20,\"publisher_dashboard_section_selector\":false,\"reader_surveys_platform_question_order\":\"36,1,4,2,3,5,6,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35\",\"login_guard_app_link_in_email\":true,\"community_moderators_enabled\":false,\"enable_custom_theme\":false,\"monthly_sub_is_one_off\":false,\"unread_notes_activity_digest\":\"control\",\"display_cookie_settings\":false,\"welcome_page_query_params\":false,\"enable_free_podcast_urls\":false,\"comp_expiry_emails_disabled\":false,\"enable_description_on_polls\":false,\"use_microlink_for_instagram_embeds\":false,\"free_signup_confirmation_behavior\":\"with_email_validation\",\"ios_post_stats_for_admins\":false,\"enable_design_templates\":false,\"use_livestream_post_media_composition\":true,\"section_specific_preambles\":false,\"pub_export_temp_disable\":false,\"show_menu_on_posts\":false,\"ios_post_subscribe_web_routing\":true,\"enable_pangram_detection_settings\":false,\"freewalls_enabled_web\":false,\"app_onboarding_survey_email\":false,\"republishing_enabled\":false,\"app_mode\":false,\"show_phone_banner\":false,\"minimum_ios_version\":2200,\"enable_author_pages\":false,\"enable_decagon_chat\":true,\"first_month_upsell\":\"control\",\"new_user_checklist_enabled\":\"use_follower_count\",\"disable_high_res_recording\":false,\"latex_upgraded_inline\":false,\"anonymous_post_metering_enabled\":false,\"rss_verification_code\":\"\",\"notification_post_emails\":\"experiment\",\"chat_suppress_contributor_push_option_enabled\":false,\"enable_subscriber_tags\":true,\"media_feed_app_prepend_inbox_limit\":0,\"export_hooks_enabled\":false,\"audio_encoding_bitrate\":null,\"extra_seats_coupon_type\":false,\"post_subdomain_universal_links\":false,\"post_import_max_file_size\":26214400,\"feed_promoted_video_publication\":false,\"livekit_reconnect_slate_url\":\"https://mux-livestream-assets.s3.us-east-1.amazonaws.com/custom-disconnect-slate-tall.png\",\"exclude_from_pymk_suggestions\":false,\"publication_ranking_variant\":\"experiment\",\"disable_annual_subscriptions\":false,\"hack_jane_manchun_wong\":true,\"android_enable_auto_gain_control\":true,\"allow_coupons_on_upgrade\":false,\"test_au_age_gate_user\":false,\"pub_auto_moderation_enabled\":false,\"disable_live_stream_ai_trimming_by_default\":false,\"disable_deletion\":false,\"livekit_egress_custom_base_url\":\"http://livekit-egress-custom-recorder.s3-website-us-east-1.amazonaws.com\",\"clip_focused_video_upload_flow\":false,\"live_stream_max_guest_users\":2,\"android_upgrade_alert_dialog_reincarnated\":true,\"enable_video_seo_data\":false,\"can_reimport_unsubscribed_users_with_2x_optin\":false,\"founding_upgrade_during_gift_disabled\":false,\"thefp_enable_section_groups\":false,\"review_incoming_email\":\"default\",\"enable_founding_gifts\":false,\"enable_sponsorship_campaigns\":false,\"thefp_forum_polls_enabled\":false,\"thefp_enable_embed_media_links\":false,\"sort_modal_search_results\":false,\"default_thumbnail_time\":10,\"load_test_unichat\":false,\"live_stream_head_alignment_guide\":false,\"free_press_combo_subscribe_flow_enabled\":false,\"enable_code_embed_toc\":false,\"enable_publication_tax_settings\":false,\"gifts_from_substack_feature_available\":true,\"disable_ai_clips\":false,\"enable_elevenlabs_voiceovers\":false,\"thefp_enable_transcripts\":false,\"show_simple_post_editor\":false,\"instacart_integration_enabled\":false,\"enable_publication_podcasts_page\":false,\"enable_direct_message_request_bypass\":false,\"enable_apple_news_sync\":false,\"live_stream_in_trending_topic_overrides\":\"\",\"free_press_newsletter_promo_enabled\":false,\"account_based_post_metering_enabled\":false,\"disable_live_stream_reactions\":false,\"instacart_partner_id\":\"\",\"enable_adhoc_email_scheduling\":true,\"media_feed_prepend_inbox_limit\":35,\"welcome_page_no_opt_out\":false,\"enable_caption_translation_tools\":false,\"section_specific_postscripts\":false,\"edit_profile_theme_colors\":false,\"disable_clipping_for_readers\":false,\"enable_apple_podcast_delivery_self_serve_pem_files\":false,\"enable_press_kit_preview_modal\":false,\"enable_rnc_page\":true,\"live_stream_video_degradation_preference\":\"maintainFramerate\",\"enable_high_follower_dm\":true,\"pause_app_badges\":false,\"profile_feed_expanded_inventory\":false,\"livekit_mux_latency_mode\":\"low\",\"feed_juiced_user\":0,\"global_iap_plan_setup_countries\":\"AE,AG,AL,AM,AT,AU,BA,BG,BH,BJ,BN,BS,BT,BW,CH,CI,CR,CY,CZ,DK,DZ,EC,EE,ET,FI,GM,GT,GY,HK,HR,HU,ID,IL,IN,IS,JM,JO,KE,KW,LC,LI,LK,LT,LU,LV,MA,MD,MG,MK,MN,MT,MU,MX,MY,MZ,NA,NO,NZ,OM,PA,PE,PH,PK,QA,RO,RS,RW,SE,SG,SI,SK,SN,SV,TH,TN,TR,TT,TW,TZ,UZ,VN,ZA\",\"enable_collapsed_content\":false,\"enable_polymarket_expandable_embeds\":true,\"publisher_dashboard_group_subs_sidebar\":false,\"use_advanced_commerce_api_for_iap\":false,\"skip_free_preview_language_in_podcast_notes\":false,\"larger_wordmark_on_publication_homepage\":false,\"enable_mobile_stats_for_admins\":false,\"enable_livestream_youtube_simulcast\":false,\"enable_persona_sandbox_environment\":false,\"allowed_email_domains\":\"mozmail.com\",\"create_nav_item_from_tag\":false,\"bypass_single_unlock_token_limit\":false,\"enable_code_embeds\":false,\"add_section_and_tag_metadata\":false,\"daily_promoted_notes_enabled\":true,\"enable_segments_ui_revamp\":true,\"enable_islands_cms\":false,\"enable_livestream_combined_stats\":false,\"redirect_to_pub_after_welcome_signup\":false,\"chartbeat_video_enabled\":false,\"enable_drip_campaigns\":false,\"enable_filtered_posts_on_homepage\":false,\"post_management_search_engine\":\"elasticsearch\",\"new_bestseller_leaderboard_feed_item_enabled\":false,\"feed_main_disabled\":false,\"enable_account_settings_revamp\":false,\"thefp_enable_fp_recirc_block\":false,\"top_search_variant\":\"control\",\"enable_debug_logs_ios\":false,\"show_pub_content_on_profile_for_pub_id\":0,\"show_pub_content_on_profile\":false,\"livekit_track_egress\":true,\"onboarding_suggestions_search\":\"experiment\",\"feed_tuner_enabled\":false,\"livekit_mux_latency_mode_rtmp\":\"low\",\"google_login_enabled\":false,\"livekit_high_quality_egress\":false,\"iap_announcement_blog_url\":\"\",\"disable_code_embed_chrome\":false,\"ios_livestream_feedback\":false,\"founding_plan_upgrade_warning\":false,\"ios_mediaplayer_reply_bar_v2\":false,\"enable_notification_email_batching\":true,\"note_author_poll_results_visible_by_default\":true,\"ios_hide_portal_tab_bar\":false,\"follow_upsell_rollout_percentage\":30,\"gift_article_enabled\":false,\"live_stream_invite_ttl_seconds\":900000,\"include_founding_plans_coupon_option\":false,\"thefp_enable_cancellation_discount_offer\":false,\"android_enable_edit_profile_theme\":false,\"disable_audio_enhancement\":false,\"pub_search_variant\":\"control\",\"ignore_video_in_notes_length_limit\":false,\"substack_gtm_enabled\":false,\"allow_long_videos\":true,\"enable_automation_filters\":false,\"use_enhanced_video_embed_player\":true,\"founding_upgrade_preview\":false,\"enable_viewing_all_livestream_viewers\":false,\"send_subscription_canceled_email\":false,\"enable_clip_prompt_variant_filtering\":true,\"chartbeat_enabled\":false,\"dpn_ranking_enabled\":true,\"enable_custom_email_css\":false,\"thefp_enable_shows_page\":false,\"enable_apple_podcast_auto_publish\":false,\"linkedin_profile_search_enabled\":false,\"publication_has_own_app\":false,\"suggested_minimum_ios_version\":0,\"thebulwark_enable_footer_actions\":false,\"enable_posts_to_segments\":true,\"trending_topics_module_long_term_experiment\":\"control\",\"phone_verification_fallback_to_twilio\":false,\"enable_suggested_searches\":true,\"enable_subscription_notification_email_batching\":true,\"feed_controls\":\"control\",\"thefp_enable_plans_modal_v2\":false,\"a24_redemption_link\":\"\",\"design_templates_launch_phase\":\"off\",\"post_custom_email_from_name_enabled\":false,\"ios_live_stream_auto_gain_enabled\":true,\"enable_publication_tts_player\":false,\"enable_group_direct_messages\":true,\"enable_notes_admins\":false,\"thefp_show_pub_app_callout_on_post\":false,\"search_ranker_variant\":\"experiment\",\"forced_featured_topic_id\":\"\",\"ios_audio_captions_disabled\":false,\"web_podcasts_tab\":false,\"related_posts_enabled\":false,\"thefp_enable_homepage_secondary_nav\":false,\"ios_live_stream_pip_dismiss_v4\":\"control\",\"reply_rate_limit_max_distinct_users_daily\":110,\"web_post_metering_enabled\":true,\"search_retrieval_variant\":\"experiment\",\"mobile_user_attachments_enabled\":false,\"web_profile_publication_recommendations_carousel\":true,\"publisher_banner\":\"\",\"enable_sponsorship_profile\":false,\"gift_article_button_enabled\":false,\"reply_rate_limit_max_distinct_users_monthly\":600,\"desktop_live_stream_screen_share_audio_enabled\":false,\"creator_match\":true,\"ai_image_generation_enabled\":true,\"ios_default_coupon_enabled\":true,\"automod_admin_hidden_comments_tab\":true,\"video_clip_suggestions_enabled\":false,\"publisher_dashboard_home_layout_experiment\":\"control\",\"android_polymarket_embed_search\":false,\"audience_section_editor_help_link\":\"https://support.substack.com/hc/en-us/articles/50558240901268-How-do-I-add-audience-specific-content-to-a-post-on-Substack\",\"portal_ranking_variant\":\"experiment\",\"permalink_reply_ranking_variant\":\"experiment\",\"allow_feed_category_filtering\":false,\"enable_dynamic_content\":true,\"enable_thumbnail_editor_media_settings_integration\":false,\"subscriber_profile_survey_responses_enabled\":true,\"private_live_streaming_enabled\":true,\"desktop_live_stream_safe_framing\":0.8,\"translate_posts_on_standalone_sites\":true,\"bundle_subscribe_default_to_regular_plan\":false,\"enable_standardized_sponsorship_formats\":false,\"enable_pencraft_signin_form\":true,\"thefp_enable_secondary_section_redesign\":false,\"bignight_page_enabled\":true,\"thefp_paywall_annual_plans_experiment\":\"control\",\"enable_launch_assistant\":false,\"enable_youtube_exploration\":true,\"publisher_dashboard_nav_refresh\":false,\"thefp_enable_bundle_post_follow\":false,\"web_notes_tiptap_v3_experiment\":\"treatment_tiptap_v3\",\"recipe_rich_text_enabled\":true,\"publisher_dashboard_nav_refresh_experiment\":\"treatment\",\"chat_channel_creation_enabled\":false,\"web_onboarding_content_type_preferences\":\"experiment\",\"web_mobile_asset_handoff\":false},\"trackingTokens\":{\"fbPixelId\":null,\"gaPixelId\":null,\"twitterPixelId\":null,\"parselyPixelId\":null,\"googleTagManagerToken\":\"\",\"googleAnalytics4Token\":\"G-G8RXW18CW7\",\"twitterPixelSignupEventId\":null,\"twitterPixelSubscribeEventId\":null},\"publicationSettings\":{\"block_ai_crawlers\":false,\"credit_token_enabled\":true,\"custom_tos_and_privacy\":false,\"did_identity\":null,\"disable_optimistic_bank_payments\":false,\"display_welcome_page_details\":true,\"payment_pledges_enabled\":false,\"enable_drop_caps\":false,\"enable_post_page_conversion\":true,\"enable_prev_next_nav\":false,\"enable_restacking\":false,\"founding_group_subscriptions_enabled\":false,\"gifts_from_substack_disabled\":false,\"google_analytics_4_token\":\"G-G8RXW18CW7\",\"group_sections_and_podcasts_in_menu_enabled\":false,\"live_stream_homepage_visibility\":\"contributorsAndAdmins\",\"live_stream_homepage_style\":\"autoPlay\",\"live_stream_replay_enabled\":true,\"medium_length_description\":\"Senior Staff Engineer distilling 2+ decades of experience in System Reliability Engineering, Technical Leadership, and growth\",\"notes_feed_enabled\":true,\"paywall_unlock_tokens\":true,\"post_preview_crop_gravity\":\"center\",\"post_preview_radius\":\"xs\",\"reader_referrals_enabled\":true,\"reader_referrals_leaderboard_enabled\":true,\"seen_coming_soon_explainer\":false,\"seen_google_analytics_migration_modal\":false,\"local_currency_modal_seen\":true,\"local_payment_methods_modal_seen\":true,\"twitter_pixel_signup_event_id\":null,\"twitter_pixel_subscribe_event_id\":null,\"use_inclusive_sales_tax\":false,\"use_local_currency\":true,\"welcome_page_opt_out_text\":\"Maybe Later\",\"cookie_settings\":\"\",\"show_restacks_below_posts\":true,\"holiday_gifting_post_header\":true,\"homepage_message_text\":\"\",\"homepage_message_link\":\"\",\"about_us_author_ids\":\"\",\"archived_section_ids\":\"\",\"column_section_ids\":\"\",\"fp_primary_column_section_ids\":\"\",\"event_section_ids\":\"\",\"podcasts_metadata\":\"\",\"video_section_ids\":\"\",\"post_metering_enabled\":false,\"account_based_post_metering_enabled\":false,\"use_custom_theme\":false,\"reply_rules\":null,\"automatic_moderation_enabled\":true,\"auto_translate_enabled\":true,\"additional_post_languages_enabled\":false,\"high_res_recording_beta\":false},\"publicationUserSettings\":null,\"userSettings\":{\"user_id\":null,\"activity_likes_enabled\":true,\"dashboard_nav_refresh_enabled\":false,\"is_guest_post_enabled\":true,\"invite_friends_nux_dismissed_at\":null,\"suggestions_feed_item_last_shown_at\":null,\"last_notification_alert_shown_at\":null,\"disable_reply_hiding\":false,\"newest_seen_chat_item_published_at\":null,\"explicitContentEnabled\":false,\"contactMatchingEnabled\":false,\"messageRequestLevel\":\"everyone\",\"liveStreamAcceptableInviteLevel\":\"everyone\",\"liveStreamAcceptableChatLevel\":\"everyone\",\"creditTokensTreatmentExposed\":false,\"appBadgeIncludesChat\":false,\"autoPlayVideo\":true,\"autoAdvanceVideo\":true,\"smart_delivery_enabled\":false,\"chatbotTermsLastAcceptedAt\":null,\"has_seen_notes_post_app_upsell\":false,\"first_note_id\":null,\"show_concurrent_live_stream_viewers\":false,\"edit_profile_feed_item_dismissed_at\":null,\"mobile_permalink_app_upsell_seen_at\":null,\"new_user_checklist_enabled\":false,\"has_seen_youtube_shorts_auto_publish_announcement\":false,\"has_seen_publish_youtube_connect_upsell\":false,\"notificationQualityFilterEnabled\":true,\"hasSeenOnboardingNewslettersScreen\":false,\"bestsellerBadgeEnabled\":true,\"hasSelfIdentifiedAsCreator\":false,\"autoTranslateEnabled\":true,\"autoTranslateBlocklist\":[],\"fineTuneFeedItemSeenAt\":null,\"fineTuneFeedItemDismissedAt\":null,\"aiGeneratedTextMode\":\"show\",\"feedVideoPreference\":5,\"feedAudioPreference\":5,\"feedLongerPostsPreference\":5,\"feedChatPreference\":5},\"mux_env_key\":\"u42pci814i6011qg3segrcpp9\",\"persona_environment_id\":\"env_o1Lbk4JhpY4PmvNkwaBdYwe5Fzkt\",\"sentry_environment\":\"production\",\"launchWelcomePage\":false,\"pendingInviteForActiveLiveStream\":null,\"isEligibleForLiveStreamCreation\":true,\"webviewPlatform\":null,\"noIndex\":false,\"post\":{\"audience\":\"everyone\",\"audience_before_archived\":null,\"canonical_url\":\"https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped\",\"default_comment_sort\":null,\"editor_v2\":false,\"exempt_from_archive_paywall\":false,\"free_unlock_required\":false,\"id\":193103401,\"podcast_art_url\":null,\"podcast_duration\":null,\"podcast_preview_upload_id\":null,\"podcast_upload_id\":null,\"podcast_url\":null,\"post_date\":\"2026-04-03T21:03:06.206Z\",\"updated_at\":\"2026-06-16T18:13:21.741Z\",\"publication_id\":1002265,\"search_engine_description\":\"Tips and tricks to work around delayed or partial observability for remote systems. Build a side page to visualize information, alert on anomalies and support automated recovery and self-service in order to reduce time to resolution and increase confidence in the software vendor.\",\"search_engine_title\":null,\"section_id\":67723,\"should_send_free_preview\":false,\"show_guest_bios\":true,\"slug\":\"reliability-engineering-for-air-gapped\",\"social_title\":null,\"subtitle\":\"Tips and tricks to work around inaccessible observability\",\"teaser_post_eligible\":true,\"title\":\"Reliability Engineering for Air-Gapped Systems\",\"type\":\"newsletter\",\"video_upload_id\":null,\"write_comment_permissions\":\"everyone\",\"meter_type\":\"none\",\"live_stream_id\":null,\"is_published\":true,\"restacks\":1,\"reactions\":{\"\u2764\":4},\"top_exclusions\":[],\"pins\":[],\"section_pins\":[],\"has_shareable_clips\":false,\"previous_post_slug\":\"multi-agent-system-reliability\",\"next_post_slug\":\"ai-reliability-engineering\",\"cover_image\":\"https://substackcdn.com/image/fetch/$s_!a6ED!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png\",\"cover_image_is_square\":true,\"cover_image_is_explicit\":false,\"videoUpload\":null,\"podcastFields\":{\"post_id\":193103401,\"podcast_episode_number\":null,\"podcast_season_number\":null,\"podcast_episode_type\":null,\"should_syndicate_to_other_feed\":null,\"syndicate_to_section_id\":null,\"hide_from_feed\":false,\"free_podcast_url\":null,\"free_podcast_duration\":null,\"preview_contains_ad\":false,\"was_imported_self_serve_sync\":false,\"draft_free_podcast_url\":null,\"draft_free_podcast_duration\":null},\"podcastUpload\":null,\"podcastPreviewUpload\":null,\"voiceover_upload_id\":null,\"voiceoverUpload\":null,\"has_voiceover\":false,\"description\":\"Tips and tricks to work around inaccessible observability\",\"body_html\":\"<p>Back in February, I helped a few teams from defense sector to measure the right thing (SLI), set reasonable expectations (SLO) and tie reliability to accountability through alerting. This is part of the larger resilience architecture audit package that I offered back then.</p><p>The details is behind an NDA (non-disclosure agreement) but there was an interesting aspect that is worth discussing:</p><p>All those systems were air-gapped, meaning the team that builds the software has no access to metrics, logs or runtime.</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!a6ED!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!a6ED!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png 424w, https://substackcdn.com/image/fetch/$s_!a6ED!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png 848w, https://substackcdn.com/image/fetch/$s_!a6ED!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png 1272w, https://substackcdn.com/image/fetch/$s_!a6ED!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!a6ED!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png\\\" width=\\\"851\\\" height=\\\"861\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/cf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:861,&quot;width&quot;:851,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:86496,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:false,&quot;topImage&quot;:true,&quot;internalRedirect&quot;:&quot;https://blog.alexewerlof.com/i/193103401?img=https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png&quot;,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!a6ED!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png 424w, https://substackcdn.com/image/fetch/$s_!a6ED!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png 848w, https://substackcdn.com/image/fetch/$s_!a6ED!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png 1272w, https://substackcdn.com/image/fetch/$s_!a6ED!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fcf150ef5-75a8-4812-a51d-2fc4b126a784_851x861.png 1456w\\\" sizes=\\\"100vw\\\" fetchpriority=\\\"high\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a><figcaption class=\\\"image-caption\\\">An air-gapped system may be operated by someone other than the developers. The observability may be absent, delayed or partial </figcaption></figure></div><p>How could they measure SLI/SLO without real time data? How could they do that without any data? How would they know something is broken? And how would they diagnose and fix the problem?</p><p>That\u2019s what this post is about. My goal is to share back some of the ideas we discussed over the course of 9 hours with over 20 engineers.</p><p><strong>Disclaimer: no AI is used in this post other than my browser\u2019s built-in spell checker. Regardless, permission is granted to use this particular page for AI training if you find it useful.</strong></p><p class=\\\"button-wrapper\\\" data-attrs=\\\"{&quot;url&quot;:&quot;https://blog.alexewerlof.com/subscribe?&quot;,&quot;text&quot;:&quot;Subscribe now&quot;,&quot;action&quot;:null,&quot;class&quot;:null}\\\" data-component-name=\\\"ButtonCreateButton\\\"><a class=\\\"button primary\\\" href=\\\"https://blog.alexewerlof.com/subscribe?\\\"><span>Subscribe now</span></a></p><h2>Strict Requirements</h2><p>I have worked at heavily regulated industries like healthcare, banking and automotive but this was the first time that I came across a system that was completely cut off from the internet for security and privacy reasons.</p><div class=\\\"digest-post-embed\\\" data-attrs=\\\"{&quot;nodeId&quot;:&quot;f7d6672f-58a6-4afe-83b9-7bf79c30c746&quot;,&quot;caption&quot;:&quot;When designing a system, both functional and non-functional requirements are important. They describe different aspects of the system behavior and have severe implications over how it is designed, implemented, maintained, and even deprecated.&quot;,&quot;cta&quot;:&quot;Read full story&quot;,&quot;showBylines&quot;:true,&quot;showDescription&quot;:true,&quot;showImage&quot;:true,&quot;size&quot;:&quot;sm&quot;,&quot;isEditorNode&quot;:true,&quot;title&quot;:&quot;NFR&quot;,&quot;publishedBylines&quot;:[{&quot;id&quot;:87732486,&quot;name&quot;:&quot;Alex Ewerl\u00F6f&quot;,&quot;bio&quot;:&quot;Writes about technical leadership, growth mindset, and system reliability engineering. Senior Staff Engineer, MSc Systems Engineering from KTH, Stockholmer, dad, amateur artist. Read more here: https://www.alexewerlof.com/who&quot;,&quot;photo_url&quot;:&quot;https://substackcdn.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg&quot;,&quot;is_guest&quot;:false,&quot;bestseller_tier&quot;:null}],&quot;post_date&quot;:&quot;2024-04-22T08:10:30.630Z&quot;,&quot;cover_image&quot;:&quot;https://substackcdn.com/image/fetch/$s_!GpLJ!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F1aac3c2c-3d39-4dca-864d-9a28f39a93e9_961x446.png&quot;,&quot;cover_image_alt&quot;:null,&quot;canonical_url&quot;:&quot;https://blog.alexewerlof.com/p/nfr&quot;,&quot;section_name&quot;:&quot;Reliability Engineering&quot;,&quot;video_upload_id&quot;:null,&quot;id&quot;:143843145,&quot;type&quot;:&quot;newsletter&quot;,&quot;reaction_count&quot;:27,&quot;comment_count&quot;:0,&quot;publication_id&quot;:1002265,&quot;publication_name&quot;:&quot;Alex Ewerl\u00F6f Notes&quot;,&quot;publication_logo_url&quot;:&quot;https://substackcdn.com/image/fetch/$s_!_Ur2!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png&quot;,&quot;belowTheFold&quot;:true,&quot;youtube_url&quot;:null,&quot;show_links&quot;:null,&quot;feed_url&quot;:null}\\\"></div><p>The NFR (non-functional requirements) dictated the system designed to:</p><ul><li><p>Be deployed on-prem (at the military facility\u2019s premises)</p></li><li><p>Operate completely without internet to minimize the attack surface but also because it was primarily used internally inside military operations.</p></li><li><p>Be deployed using an archive file that was physically handed over to the operator (on the military consumer\u2019s side) to reduce the risk of man-in-the-middle or supply chain attacks.</p></li><li><p>Any problem with the system was resolved over secure phone calls because sending diagnostics data over email or other channels was against security protocol.</p></li><li><p>In some occasions the software vendor had to physically dispatch an engineer to the client side to diagnose and fix issues onsite.</p></li></ul><p>Yeah, not your typical cloud service for sure! \uD83D\uDE04</p><p>But that was exactly the type of problem that got me involved with them. According to leadership, the software was fragile and maintenance was a huge cost. I came in after a week-long outage which put the company\u2019s reputation at risk.</p><h2>Engineers</h2><p>You might think that the system quality is a direct reflection of the talent behind it and you\u2019d be forgiven.</p><p>When I met the team, I felt their frustration but also their commitment to improve reliability and openness to new ideas.</p><p>It\u2019s not an exaggeration to say that most of what I\u2019m about to share came out of intense discussion with the team who knew their domain and constraints very well in conjunction with the service level (SLI/SLO/SLA) model that I specialize in.</p><div class=\\\"digest-post-embed\\\" data-attrs=\\\"{&quot;nodeId&quot;:&quot;42b09bd3-25ab-4a39-9ea6-46869e49d9ba&quot;,&quot;caption&quot;:&quot;In the context of service levels, there are multiple terms that are related to each other. This post serves as a glossary to define and connect:&quot;,&quot;cta&quot;:&quot;Read full story&quot;,&quot;showBylines&quot;:true,&quot;showDescription&quot;:true,&quot;showImage&quot;:true,&quot;size&quot;:&quot;sm&quot;,&quot;isEditorNode&quot;:true,&quot;title&quot;:&quot;Service Level Terminology&quot;,&quot;publishedBylines&quot;:[{&quot;id&quot;:87732486,&quot;name&quot;:&quot;Alex Ewerl\u00F6f&quot;,&quot;bio&quot;:&quot;Writes about technical leadership, growth mindset, and system reliability engineering. Senior Staff Engineer, MSc Systems Engineering from KTH, Stockholmer, dad, amateur artist. Read more here: https://www.alexewerlof.com/who&quot;,&quot;photo_url&quot;:&quot;https://substackcdn.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg&quot;,&quot;is_guest&quot;:false,&quot;bestseller_tier&quot;:null}],&quot;post_date&quot;:&quot;2024-10-25T03:16:15.538Z&quot;,&quot;cover_image&quot;:&quot;https://substackcdn.com/image/fetch/$s_!MlI7!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F5bb74595-f0d9-4d19-9a4c-23d596b721a8_1010x1122.png&quot;,&quot;cover_image_alt&quot;:null,&quot;canonical_url&quot;:&quot;https://blog.alexewerlof.com/p/service-level-terminology&quot;,&quot;section_name&quot;:&quot;Reliability Engineering&quot;,&quot;video_upload_id&quot;:null,&quot;id&quot;:150614488,&quot;type&quot;:&quot;newsletter&quot;,&quot;reaction_count&quot;:20,&quot;comment_count&quot;:4,&quot;publication_id&quot;:1002265,&quot;publication_name&quot;:&quot;Alex Ewerl\u00F6f Notes&quot;,&quot;publication_logo_url&quot;:&quot;https://substackcdn.com/image/fetch/$s_!_Ur2!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png&quot;,&quot;belowTheFold&quot;:true,&quot;youtube_url&quot;:null,&quot;show_links&quot;:null,&quot;feed_url&quot;:null}\\\"></div><h3>Self-service</h3><p>The team could not observe the system in real-time, but they could work around that limitation by helping the operator (usually the IT personnel at the military facility) act as SRE.</p><p>The idea was simple: provide a reliable page to:</p><ol><li><p><strong>Real time status:</strong> create simple dashboards to visualize CPU/Memory/Disk usage, Network connectivity, and load (e.g. request per second) and metric trends over time. This could help the operator quickly identify common problems. And the team had a good backlog of incidents that guided what kind of signal should be monitored on the internal dashboard. For example if the services couldn\u2019t find each other due to a firewal misconfiguration or network error, the operator would be able to get a high quality insight and narrow down their fix.</p></li><li><p><strong>Proactive alerting:</strong> once we have that data, creating alerts (e.g. via email) is trivial. The idea is that when there\u2019s an issue, the operator should know ASAP instead of waiting till an actual user find out about it (because then it\u2019s too late). This severely improve TTD (time to detect) and reduce down time.</p></li><li><p><strong>Mainstreamed troubleshooting:</strong> once an error was identified (e.g. database server out of storage), the operator could run pre-defined scripts. You can think of them as automated runbooks. Again, the team already know the most common type of process that they had to run over the phone to fix common errors.</p></li><li><p><strong>Auto-repair:</strong> For the most common type of error, the system would auto-diagnose and auto-repair (e.g. restarting an instance or scaling horizontally). This is the kind of repair that you get out of the box when using a cloud provider but when deploying to a wide range of on-premises setup, there\u2019s limited tooling you can count on.</p></li><li><p><strong>Intelligent Anomaly Detection:</strong> Observability is a measure of how well internal states of a system can be inferred from knowledge of its external outputs. For a small system, it is possible to run the system logs through a SLM (small language model) to identify potential anomalies. Something like <a href=\\\"https://www.liquid.ai/models\\\">Liquid AI</a> can easily run on a side card node and be tasked to analyze a fraction of logs for signals. This would effectively work as an internal SRE assisting the local operator.</p></li><li><p><strong>Anonymization:</strong> stripping the data from critical information like PII (personally identifiable information) or sensitive military planning information, it\u2019s theoretically possible to create diagnostic data that can be shared with the vendor to understand and improve the performance of the system.</p></li></ol><div class=\\\"digest-post-embed\\\" data-attrs=\\\"{&quot;nodeId&quot;:&quot;69d2a36f-a897-4c52-8bf9-ccc512bf2b49&quot;,&quot;caption&quot;:&quot;Many AI applications rely on Model-as-a-Service (MaaS) like OpenAI, Gemini, Claude, etc.&quot;,&quot;cta&quot;:&quot;Read full story&quot;,&quot;showBylines&quot;:true,&quot;showDescription&quot;:true,&quot;showImage&quot;:true,&quot;size&quot;:&quot;sm&quot;,&quot;isEditorNode&quot;:true,&quot;title&quot;:&quot;AI topology&quot;,&quot;publishedBylines&quot;:[{&quot;id&quot;:87732486,&quot;name&quot;:&quot;Alex Ewerl\u00F6f&quot;,&quot;bio&quot;:&quot;Writes about technical leadership, growth mindset, and system reliability engineering. Senior Staff Engineer, MSc Systems Engineering from KTH, Stockholmer, dad, amateur artist. Read more here: https://www.alexewerlof.com/who&quot;,&quot;photo_url&quot;:&quot;https://substackcdn.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg&quot;,&quot;is_guest&quot;:false,&quot;bestseller_tier&quot;:null}],&quot;post_date&quot;:&quot;2025-10-24T21:15:00.000Z&quot;,&quot;cover_image&quot;:&quot;https://substackcdn.com/image/fetch/$s_!4UFi!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F55317f3d-e36b-449f-b4b6-f63db8d987ad_721x689.png&quot;,&quot;cover_image_alt&quot;:null,&quot;canonical_url&quot;:&quot;https://blog.alexewerlof.com/p/ai-topology&quot;,&quot;section_name&quot;:&quot;Code&quot;,&quot;video_upload_id&quot;:null,&quot;id&quot;:181865778,&quot;type&quot;:&quot;newsletter&quot;,&quot;reaction_count&quot;:5,&quot;comment_count&quot;:0,&quot;publication_id&quot;:1002265,&quot;publication_name&quot;:&quot;Alex Ewerl\u00F6f Notes&quot;,&quot;publication_logo_url&quot;:&quot;https://substackcdn.com/image/fetch/$s_!_Ur2!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png&quot;,&quot;belowTheFold&quot;:true,&quot;youtube_url&quot;:null,&quot;show_links&quot;:null,&quot;feed_url&quot;:null}\\\"></div><p>You may have noticed that the ideas are sorted incrementally based on how hard they are to implement and how risky they are.</p><h2>Bespoke Observability?</h2><p>You may notice that some of those ideas (dashboards, alerts, logs, anomaly detection) are already addressed by observability providers. The team did not have to reinvent the wheel. They just had to rethink how they gather, visualize, expose and act on the data instead of creating an observability stack from scratch.</p><p>The goal of this exercise wasn\u2019t to create a bespoke observability stack, but rather identify the requirements and be intentional about what data to gather, how to expose it, what kind of action to support and why.</p><p>Regardless, even a simple status page would be a big win because it helped the customers answer a simple question: is it just me?</p><div class=\\\"digest-post-embed\\\" data-attrs=\\\"{&quot;nodeId&quot;:&quot;04bda80d-48fe-4abd-8a2e-d15dad01c588&quot;,&quot;caption&quot;:&quot;In the context of reliability engineering, there are three terms that are related but sometimes used incorrectly.&quot;,&quot;cta&quot;:&quot;Read full story&quot;,&quot;showBylines&quot;:true,&quot;showDescription&quot;:true,&quot;showImage&quot;:true,&quot;size&quot;:&quot;sm&quot;,&quot;isEditorNode&quot;:true,&quot;title&quot;:&quot;Degradation vs disruption&quot;,&quot;publishedBylines&quot;:[{&quot;id&quot;:87732486,&quot;name&quot;:&quot;Alex Ewerl\u00F6f&quot;,&quot;bio&quot;:&quot;Writes about technical leadership, growth mindset, and system reliability engineering. Senior Staff Engineer, MSc Systems Engineering from KTH, Stockholmer, dad, amateur artist. Read more here: https://www.alexewerlof.com/who&quot;,&quot;photo_url&quot;:&quot;https://substackcdn.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg&quot;,&quot;is_guest&quot;:false,&quot;bestseller_tier&quot;:null}],&quot;post_date&quot;:&quot;2024-07-10T13:49:59.325Z&quot;,&quot;cover_image&quot;:&quot;https://substackcdn.com/image/fetch/$s_!wL5v!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0bd88583-8b9e-4ce0-8cb6-befa958a794f_943x620.png&quot;,&quot;cover_image_alt&quot;:null,&quot;canonical_url&quot;:&quot;https://blog.alexewerlof.com/p/service-degradation-vs-disruption&quot;,&quot;section_name&quot;:&quot;Reliability Engineering&quot;,&quot;video_upload_id&quot;:null,&quot;id&quot;:146395325,&quot;type&quot;:&quot;newsletter&quot;,&quot;reaction_count&quot;:20,&quot;comment_count&quot;:0,&quot;publication_id&quot;:1002265,&quot;publication_name&quot;:&quot;Alex Ewerl\u00F6f Notes&quot;,&quot;publication_logo_url&quot;:&quot;https://substackcdn.com/image/fetch/$s_!_Ur2!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png&quot;,&quot;belowTheFold&quot;:true,&quot;youtube_url&quot;:null,&quot;show_links&quot;:null,&quot;feed_url&quot;:null}\\\"></div><p>The key idea here is to have the status page <strong>separate</strong> from the main application, ideally on a completely different release cadence. That\u2019s because if the status page and the main app are part of the same deployment and share the same dependencies, the risk of correlated downtime increases dramatically.</p><p>That is not good because the status page is most needed when the main service is experiencing degradation or disruption.</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!jfie!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!jfie!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png 424w, https://substackcdn.com/image/fetch/$s_!jfie!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png 848w, https://substackcdn.com/image/fetch/$s_!jfie!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png 1272w, https://substackcdn.com/image/fetch/$s_!jfie!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!jfie!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png\\\" width=\\\"950\\\" height=\\\"532\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:532,&quot;width&quot;:950,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:69037,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:&quot;https://blog.alexewerlof.com/i/193103401?img=https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png&quot;,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!jfie!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png 424w, https://substackcdn.com/image/fetch/$s_!jfie!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png 848w, https://substackcdn.com/image/fetch/$s_!jfie!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png 1272w, https://substackcdn.com/image/fetch/$s_!jfie!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F97b39bb5-c795-465c-b2ce-48fac96516e9_950x532.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a><figcaption class=\\\"image-caption\\\">A simple architecture showing how the status checker keeps testing the API and optionally front-end assets to populate the status DB which is in turn used to render the status page frontend</figcaption></figure></div><p>Again, the key is to treat the status page as a separate entity with its own isolation and lifecycle. Some vendors even keep it on a separate domain to decouple it from the main service DNS issues (For example: the status page for <code>github.com</code> is at <code>githubstatus.com</code>).</p><p class=\\\"button-wrapper\\\" data-attrs=\\\"{&quot;url&quot;:&quot;https://blog.alexewerlof.com/subscribe?&quot;,&quot;text&quot;:&quot;Subscribe now&quot;,&quot;action&quot;:null,&quot;class&quot;:null}\\\" data-component-name=\\\"ButtonCreateButton\\\"><a class=\\\"button primary\\\" href=\\\"https://blog.alexewerlof.com/subscribe?\\\"><span>Subscribe now</span></a></p><h2>Bonus ideas</h2><p>We also discussed a couple more ideas that work around the unique limitations of this setup:</p><ol><li><p><strong>Cryptic but specific error codes:</strong> when a support call is made without useful information, it hurts the TTR (time to resolution). The most common problem was no meaningful error, or cryptic messages like \u201COperation failed\u201D. This left both parties (the operator and vendor) wonder: what went wrong? Why? Where? How to fix that? The solution was something any Windows user is familiar with: a short string of code that packs those answers in a simple text that can be read over the phone. \u201COh you got <code>0xAF6600BB</code>? That\u2019s means the static web server has inconsistent state. Please go to the Application tab and clear storage!\u201D. Or at least that\u2019s the type of scenario that these errors were supposed to unblock. The idea was to assign a unique ID to <strong>each type of operation, module, component, and system</strong> to pinpoint exactly what failed and where. Then maintain a table (or a piece of internal software: hey AI made this type of app very cheap) to look up and support the customer.</p></li><li><p><strong>Metric batches:</strong> since we cannot rely on real-time monitoring due to security and privacy concerns, the next best thing is to create small batches of metric data (think time series, compressed) that can manually be exported by the operator, either as email attachments or via USB. </p></li><li><p><strong>Pseudonymized logs:</strong> we did mention that one of the things AI could do was to clean up the logs to be transferable to the software vendor for diagnostics but due to complexity and unpredictability, a simple regexp would do. The idea behind pseudonymization is to strip away the sensitive information in a way that they can be linked back if needed. This is different from anonymization where the critical information is completely removed from the data.</p></li></ol><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!U2b8!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!U2b8!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png 424w, https://substackcdn.com/image/fetch/$s_!U2b8!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png 848w, https://substackcdn.com/image/fetch/$s_!U2b8!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png 1272w, https://substackcdn.com/image/fetch/$s_!U2b8!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!U2b8!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png\\\" width=\\\"1155\\\" height=\\\"548\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:548,&quot;width&quot;:1155,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:75058,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:&quot;https://blog.alexewerlof.com/i/193103401?img=https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png&quot;,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!U2b8!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png 424w, https://substackcdn.com/image/fetch/$s_!U2b8!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png 848w, https://substackcdn.com/image/fetch/$s_!U2b8!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png 1272w, https://substackcdn.com/image/fetch/$s_!U2b8!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F66a77049-4e95-4736-90c5-a4f9226b7b90_1155x548.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a><figcaption class=\\\"image-caption\\\">An example of error codes from <a href=\\\"https://www.samsung.com/ph/support/home-appliances/samsung-front-load-washer-error-codes/\\\">Samsung Washing machines</a> demonstrates how the system can provide onsite help without internet connectivity.</figcaption></figure></div><h1>SLI/SLO/Alerting</h1><p>So, did we solve the SLI/SLO problem for this type of air-gaped high security setup?</p><p>Not in the sense that Google preaches in their SRE books. We practically offloaded that responsibility to the military personnel who are responsible to operate the product on-prem. This can be considered an example of the broken ownership because the people who build the system aren\u2019t the ones who run it.</p><div class=\\\"digest-post-embed\\\" data-attrs=\\\"{&quot;nodeId&quot;:&quot;aeb311d4-4b6c-4d67-9586-266ad3b285fa&quot;,&quot;caption&quot;:&quot;Have you been in any of these situations?&quot;,&quot;cta&quot;:&quot;Read full story&quot;,&quot;showBylines&quot;:true,&quot;showDescription&quot;:true,&quot;showImage&quot;:true,&quot;size&quot;:&quot;sm&quot;,&quot;isEditorNode&quot;:true,&quot;title&quot;:&quot;6 Archetypes of Broken Ownership&quot;,&quot;publishedBylines&quot;:[{&quot;id&quot;:87732486,&quot;name&quot;:&quot;Alex Ewerl\u00F6f&quot;,&quot;bio&quot;:&quot;Writes about technical leadership, growth mindset, and system reliability engineering. Senior Staff Engineer, MSc Systems Engineering from KTH, Stockholmer, dad, amateur artist. Read more here: https://www.alexewerlof.com/who&quot;,&quot;photo_url&quot;:&quot;https://substackcdn.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg&quot;,&quot;is_guest&quot;:false,&quot;bestseller_tier&quot;:null}],&quot;post_date&quot;:&quot;2023-08-01T17:10:54.401Z&quot;,&quot;cover_image&quot;:&quot;https://substackcdn.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F72091ddc-6191-4ccb-8d06-15a7edbed199_1012x964.png&quot;,&quot;cover_image_alt&quot;:null,&quot;canonical_url&quot;:&quot;https://blog.alexewerlof.com/p/broken-ownership&quot;,&quot;section_name&quot;:&quot;Reliability Engineering&quot;,&quot;video_upload_id&quot;:null,&quot;id&quot;:135623153,&quot;type&quot;:&quot;newsletter&quot;,&quot;reaction_count&quot;:84,&quot;comment_count&quot;:7,&quot;publication_id&quot;:1002265,&quot;publication_name&quot;:&quot;Alex Ewerl\u00F6f Notes&quot;,&quot;publication_logo_url&quot;:&quot;https://substackcdn.com/image/fetch/$s_!_Ur2!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png&quot;,&quot;belowTheFold&quot;:true,&quot;youtube_url&quot;:null,&quot;show_links&quot;:null,&quot;feed_url&quot;:null}\\\"></div><p>To their defense, this is primarily a side effect of the strict NFRs.</p><p>The key insight was to provide them with bespoke compliant observability and repair tools to:</p><ol><li><p><strong>Reduce false alerts:</strong> by providing them with better signals about the health of the system</p></li><li><p><strong>Reduce support calls:</strong> by enabling them to diagnose and repair common errors that happen due to discrepancy of the on-prem setup.</p></li><li><p><strong>Increase ownership:</strong> This is in a way deflecting the on-prem problems to the people who are in the best position to fix it because they own that setup (<a href=\\\"https://blog.alexewerlof.com/p/you-build-it-you-own-it\\\">know how it works, have the mandate, and are responsible</a>).</p></li><li><p><strong>Reduce TTR:</strong> from diagnostic to alerting and troubleshooting, the on-prem operators could get a lot done before even touching the support phone calls.</p></li><li><p><strong>Boost confidence:</strong> previously any time something broke, the operators blamed the software and vendor. Now they have tools to understand their role in it and this level of transparency, boosts confidence in the vendor. That\u2019s because the vendor is demonstrating that they know what they\u2019ve shipped, how it may fail, and what to do in case something goes wrong.</p></li></ol><div class=\\\"captioned-button-wrap\\\" data-attrs=\\\"{&quot;url&quot;:&quot;https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped?utm_source=substack&utm_medium=email&utm_content=share&action=share&quot;,&quot;text&quot;:&quot;Share&quot;}\\\" data-component-name=\\\"CaptionedButtonToDOM\\\"><div class=\\\"preamble\\\"><p class=\\\"cta-caption\\\">If you found this post insightful, please share it in your circles and on social media to inspire others</p></div><p class=\\\"button-wrapper\\\" data-attrs=\\\"{&quot;url&quot;:&quot;https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped?utm_source=substack&utm_medium=email&utm_content=share&action=share&quot;,&quot;text&quot;:&quot;Share&quot;}\\\" data-component-name=\\\"ButtonCreateButton\\\"><a class=\\\"button primary\\\" href=\\\"https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped?utm_source=substack&utm_medium=email&utm_content=share&action=share\\\"><span>Share</span></a></p></div>\",\"has_dynamic_content\":false,\"truncated_body_text\":\"Back in February, I helped a few teams from defense sector to measure the right thing (SLI), set reasonable expectations (SLO) and tie reliability to accountability through alerting. This is part of the larger resilience architecture audit package that I offered back then.\",\"wordcount\":1731,\"post_preview_limit\":null,\"language\":\"en\",\"postTags\":[{\"id\":\"1d165f42-7948-43e1-bcf4-aa24c765ac39\",\"publication_id\":1002265,\"name\":\"observability\",\"slug\":\"observability\",\"hidden\":false},{\"id\":\"2a106382-e2d8-4cf8-a7b3-753207bef109\",\"publication_id\":1002265,\"name\":\"non-functional requirements\",\"slug\":\"non-functional-requirements\",\"hidden\":false},{\"id\":\"4dbefd17-9a67-44ff-8c53-d537cdcaad14\",\"publication_id\":1002265,\"name\":\"system design\",\"slug\":\"system-design\",\"hidden\":false}],\"postCountryBlocks\":[],\"headlineTest\":null,\"coverImagePalette\":{\"Vibrant\":{\"rgb\":[251,187,84],\"population\":125},\"DarkVibrant\":{\"rgb\":[4,28,44],\"population\":4021},\"LightVibrant\":{\"rgb\":[180,108,236],\"population\":92},\"Muted\":{\"rgb\":[107,125,156],\"population\":35},\"DarkMuted\":{\"rgb\":[45,48,70],\"population\":181},\"LightMuted\":{\"rgb\":[83.87349397590363,17.512048192771058,135.48795180722894],\"population\":0}},\"publishedBylines\":[{\"id\":87732486,\"name\":\"Alex Ewerl\u00F6f\",\"handle\":\"alexewerlof\",\"photo_url\":\"https://substackcdn.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg\",\"bio\":\"Writes about technical leadership, growth mindset, and system reliability engineering. Senior Staff Engineer, MSc Systems Engineering from KTH, Stockholmer, dad, amateur artist. Read more here: https://www.alexewerlof.com/who\",\"profile_set_up_at\":\"2022-07-14T22:27:32.130Z\",\"reader_installed_at\":\"2022-10-11T17:24:45.190Z\",\"publicationUsers\":[{\"id\":947579,\"user_id\":87732486,\"publication_id\":1002265,\"role\":\"admin\",\"public\":true,\"is_primary\":true,\"publication\":{\"id\":1002265,\"name\":\"Alex Ewerl\u00F6f Notes\",\"subdomain\":\"alexewerlof\",\"custom_domain\":\"blog.alexewerlof.com\",\"custom_domain_optional\":false,\"hero_text\":\"Technical Leadership, Reliability Engineering, Growth\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/8c58cb07-9341-402b-bcdb-9fa767c2cdac_500x500.png\",\"author_id\":87732486,\"primary_user_id\":87732486,\"theme_var_background_pop\":\"#786CFF\",\"created_at\":\"2022-07-14T22:27:58.683Z\",\"email_from_name\":\"Alex Ewerl\u00F6f Notes\",\"copyright\":\"Alex Ewerl\u00F6f\",\"founding_plan_name\":\"VIP supporter\",\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"enabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"newspaper\",\"is_personal_mode\":false,\"logo_url_wide\":\"https://substack-post-media.s3.amazonaws.com/public/images/bd56dc7c-4b9f-43f2-99a1-ecc995284940_1200x256.png\"}},{\"id\":8561877,\"user_id\":87732486,\"publication_id\":8361889,\"role\":\"admin\",\"public\":true,\"is_primary\":false,\"publication\":{\"id\":8361889,\"name\":\"AI Reliability Engineering\",\"subdomain\":\"aireliabilityengineering\",\"custom_domain\":null,\"custom_domain_optional\":false,\"hero_text\":\"From the Author of Reliability Engineering Mindset but focused on AI systems and components.\",\"logo_url\":\"https://substackcdn.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe2713990-da82-481b-b579-01a7aaa5b85b_560x560.jpeg\",\"author_id\":87732486,\"primary_user_id\":null,\"theme_var_background_pop\":\"#FF6719\",\"created_at\":\"2026-03-18T10:58:12.396Z\",\"email_from_name\":null,\"copyright\":\"Alex Ewerl\u00F6f\",\"founding_plan_name\":null,\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"disabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"newspaper\",\"is_personal_mode\":false,\"logo_url_wide\":null}}],\"twitter_screen_name\":\"alexewerlof\",\"is_guest\":false,\"bestseller_tier\":null,\"status\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null}}],\"reaction\":null,\"reaction_count\":4,\"comment_count\":3,\"child_comment_count\":1,\"audio_items\":[{\"post_id\":193103401,\"voice_id\":\"en-US-OnyxTurboMultilingualNeural\",\"audio_url\":\"https://substack-video.s3.amazonaws.com/video_upload/post/193103401/tts/e818fda3-fa0a-4eca-92f9-feeb5cc7fb3f/en-US-OnyxTurboMultilingualNeural.mp3\",\"type\":\"tts\",\"status\":\"completed\"}],\"is_geoblocked\":false,\"hasCashtag\":false,\"section\":{\"id\":67723,\"created_at\":\"2023-05-31T19:57:24.859Z\",\"updated_at\":\"2024-03-22T13:16:12.203Z\",\"publication_id\":1002265,\"name\":\"Reliability Engineering\",\"description\":\"Concepts that help you with building reliable systems and architecture\",\"slug\":\"sre\",\"is_podcast\":false,\"is_live\":true,\"is_default_on\":true,\"sibling_rank\":1,\"port_status\":\"success\",\"logo_url\":null,\"hide_from_navbar\":false,\"email_from_name\":\"\",\"hide_posts_from_pub_listings\":false,\"email_banner_url\":null,\"cover_photo_url\":null,\"hide_intro_title\":false,\"hide_intro_subtitle\":false,\"ignore_publication_email_settings\":false,\"custom_config\":{},\"unifiedPodcastSettings\":null,\"podcastSettings\":null,\"showLinks\":[],\"pageTheme\":{\"id\":51609,\"publication_id\":1002265,\"section_id\":67723,\"page\":null,\"page_hero\":\"default\",\"page_posts\":\"grid\",\"show_podcast_links\":true,\"hero_alignment\":\"center\"},\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null}},\"comments\":[{\"id\":277164624,\"body\":\"The ideas are quite useful for cloud services as well.\\n\\nBut without real time monitoring, how could such systems be continuously improved based on actual usage?\",\"body_json\":{\"type\":\"doc\",\"attrs\":{\"schemaVersion\":\"v1\",\"title\":null},\"content\":[{\"type\":\"paragraph\",\"content\":[{\"type\":\"text\",\"text\":\"The ideas are quite useful for cloud services as well.\"}]},{\"type\":\"paragraph\",\"content\":[{\"type\":\"text\",\"text\":\"But without real time monitoring, how could such systems be continuously improved based on actual usage?\"}]}]},\"publication_id\":1002265,\"post_id\":193103401,\"user_id\":30753764,\"ancestor_path\":\"\",\"type\":\"comment\",\"deleted\":false,\"date\":\"2026-06-16T08:06:11.298Z\",\"edited_at\":null,\"status\":\"published\",\"pinned_by_user_id\":null,\"restacks\":0,\"name\":\"jps\",\"photo_url\":null,\"handle\":\"jps1384922\",\"reactor_names\":[],\"reaction\":null,\"reactions\":{\"\u2764\":0},\"reaction_count\":0,\"children\":[],\"bans\":[],\"suppressed\":false,\"user_banned\":false,\"user_banned_for_comment\":false,\"user_slug\":\"jps1384922\",\"metadata\":{\"is_author\":false,\"membership_state\":null,\"eligibleForGift\":true},\"user_bestseller_tier\":null,\"can_dm\":true,\"userStatus\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null},\"score\":1,\"children_count\":1,\"reported_by_user\":false,\"restacked\":false,\"childrenSummary\":\"2 replies by Alex Ewerl\u00F6f and others\"}],\"canonicalUrl\":\"https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped\",\"inlineComments\":false,\"readerIsSearchCrawler\":false,\"ogUrl\":\"https://blog.alexewerlof.com/p/reliability-engineering-for-air-gapped\",\"bannedFromNotes\":false,\"themeVariables\":{\"color_theme_bg_pop\":\"#FFBC52\",\"background_pop\":\"#FFBC52\",\"color_theme_bg_web\":\"#001426\",\"cover_bg_color\":\"#001426\",\"cover_bg_color_secondary\":\"#0f2233\",\"background_pop_darken\":\"#ffb238\",\"print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_darken\":\"#ffb238\",\"color_theme_print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_20\":\"rgba(255, 188, 82, 0.2)\",\"color_theme_bg_pop_30\":\"rgba(255, 188, 82, 0.3)\",\"print_pop\":\"#ffbc52\",\"color_theme_accent\":\"#ffbc52\",\"cover_print_primary\":\"#ffffff\",\"cover_print_secondary\":\"#d9d9d9\",\"cover_print_tertiary\":\"#d9d9d9\",\"cover_border_color\":\"#ffffff\",\"font_family_headings_preset\":\"'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'\",\"font_weight_headings_preset\":900,\"font_family_body_preset\":\"Merriweather,sans-serif\",\"font_weight_body_preset\":400,\"font_preset_heading\":\"sans\",\"font_preset_body\":\"fancy_serif\",\"home_hero\":\"feature-media\",\"home_posts\":\"custom\",\"home_show_top_posts\":true,\"web_bg_color\":\"#001426\",\"background_contrast_1\":\"#0f2233\",\"background_contrast_2\":\"#223443\",\"background_contrast_3\":\"#485663\",\"background_contrast_4\":\"#6d7882\",\"background_contrast_5\":\"#aeb5ba\",\"color_theme_bg_contrast_1\":\"#0f2233\",\"color_theme_bg_contrast_2\":\"#223443\",\"color_theme_bg_contrast_3\":\"#485663\",\"color_theme_bg_contrast_4\":\"#6d7882\",\"color_theme_bg_contrast_5\":\"#aeb5ba\",\"color_theme_bg_elevated\":\"#0f2233\",\"color_theme_bg_elevated_secondary\":\"#223443\",\"color_theme_bg_elevated_tertiary\":\"#485663\",\"color_theme_detail\":\"#192b3c\",\"background_contrast_pop\":\"rgba(255, 188, 82, 0.4)\",\"color_theme_bg_contrast_pop\":\"rgba(255, 188, 82, 0.4)\",\"theme_bg_is_dark\":\"1\",\"print_on_web_bg_color\":\"hsl(208.4210526315789, 25.00000000000006%, 90.74509803921569%)\",\"print_secondary_on_web_bg_color\":\"#99a1a8\",\"footer_bg_color\":\"#000C17\",\"footer_bg_color_h\":\"208.69565217391306\",\"footer_bg_color_s\":\"100%\",\"footer_bg_color_l\":\"4.509803921568627%\",\"print_on_footer_bg_color\":\"hsl(208.69565217391306, 25.000000000000057%, 90.45098039215686%)\",\"print_secondary_on_footer_bg_color\":\"#999ea2\",\"footer_bg_contrast_1\":\"#0f1b25\",\"footer_bg_contrast_2\":\"#222d36\",\"footer_bg_contrast_3\":\"#485158\",\"footer_bg_contrast_4\":\"#6d737a\",\"footer_bg_elevated\":\"#0f1b25\",\"footer_bg_elevated_secondary\":\"#222d36\",\"footer_bg_elevated_tertiary\":\"#485158\",\"footer_detail_color\":\"#19242e\",\"background_pop_rgb\":\"255, 188, 82\",\"color_theme_bg_pop_rgb\":\"255, 188, 82\",\"color_theme_accent_rgb\":\"255, 188, 82\"},\"recentEpisodes\":null,\"trackFrontendVisit\":true,\"activeLiveStream\":null,\"freeTrialCoupon\":{\"id\":\"3cce95ff\",\"trial_period_days\":7},\"defaultCoupon\":null,\"isChatActive\":false,\"publicationPageAlert\":null,\"features\":{},\"browser\":{\"name\":\"Chrome\",\"version\":\"125.0\",\"major\":\"125\"},\"showCookieBanner\":false,\"disabledCookies\":[],\"dd_env\":\"prod\",\"dd_ti\":true}")</script>
<script nonce="nQGtn4EHptwtQVZSM4a71A">window._analyticsConfig = JSON.parse("{\"properties\":{\"subdomain\":\"alexewerlof\",\"publication_id\":1002265,\"has_plans\":true,\"pub_community_enabled\":true,\"is_personal_publication\":false,\"section_name\":\"Reliability Engineering\",\"section_slug\":\"sre\",\"section_id\":67723,\"section_is_podcast\":false,\"is_subscribed\":false,\"is_free_subscribed\":false,\"is_author\":false,\"is_contributor\":false,\"is_admin\":false,\"is_founding\":false},\"adwordsAccountId\":\"AW-316245675\",\"adwordsEventSendTo\":\"Tf76CKqcyL4DEKuN5pYB\"}")</script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/lib-router.7371871e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/55188.571db0c6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/62357.a0731d37.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53754.d196682b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/33485.2686bb5d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/2237.ae8857e4.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37707.4614ce58.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9351.41564edd.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80821.b2140fcd.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44292.f3dc2b43.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70291.7aa24e85.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17499.1820773d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57975.765aca5a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/19714.2f1bdde5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48320.4e726ad5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93000.f03f24de.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44014.f579d62b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/43745.80c75c78.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/60892.92b9fc9c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/40646.258e49f2.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/87906.d493457b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/36479.ce61e623.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17706.015b0f71.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93977.f1207089.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37606.b91ab2ef.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94311.b93f6845.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89356.66161d1e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/21262.211a7337.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47925.24a01d36.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80220.03bd4c41.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71505.3dc43e33.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11100.5d148f2c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/3663.86d95e8c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18725.a5a682c3.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/86118.ff1ec843.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93519.7fe128ac.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/84828.c78aabc1.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/5075.d6e9f8ec.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/29231.a7af5911.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56725.2b32bbb6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/42336.f1c17ded.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89879.1d0af2b9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70793.951c11cb.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/39522.58e078b0.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53164.dfde36c8.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74584.94c6b3f1.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71199.a327e9f4.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63266.8bb0d59c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47358.8f5c5991.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4492.cb326b80.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/88089.680833b5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/97348.df6a856d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/52109.cf3bcc2c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56256.8a1bc5dd.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/77410.3f20fb36.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/25058.bdbc4e21.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/58483.3adddb8a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94881.0a97506f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27386.09a9082f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/10174.7f0face7.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/92610.a4fc1daf.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91214.cb10b4b8.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11244.b723f409.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99260.28b0ed6a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48445.7d5c5ba3.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/76768.9a2f2bae.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/41932.6c688f22.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/12114.979fb38e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48409.9732c6f9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/26252.4f5ae3be.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53871.2923c5c9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18494.588a06d1.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99068.bd2e1fba.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91850.79d3f415.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/49047.c7f428fb.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/16075.098bcbcf.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/28856.2eb51db5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/95396.72d1d844.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63175.bbf1c283.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74628.2e8ed950.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/32552.889dd584.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27313.c287f2d8.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/mainReactRouter.6c265246.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91507.9b9761ae.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/54770.b40bf629.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/30054.07cd629b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78118.35b48b09.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57051.f37fb930.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78191.24075434.js" charset="utf-8"></script>
<script nonce="nQGtn4EHptwtQVZSM4a71A" nomodule>
(function() {
var message = 'Your browser does not support modern JavaScript modules. Please upgrade your browser for the best experience.';
var warningDiv = document.createElement('div');
warningDiv.style.color = 'red';
warningDiv.style.padding = '10px';
warningDiv.style.margin = '10px 0';
warningDiv.style.border = '1px solid red';
warningDiv.style.backgroundColor = 'lightyellow';
warningDiv.innerText = message;
document.body.prepend(warningDiv);
})();
</script>
<!-- Datadog Analytics -->
<script nonce="nQGtn4EHptwtQVZSM4a71A">
(function(h,o,u,n,d) {
h=h[d]=h[d]||{q:[],onReady:function(c){h.q.push(c)}}
d=o.createElement(u);d.async=1;d.src=n
n=o.getElementsByTagName(u)[0];n.parentNode.insertBefore(d,n)
})(window,document,'script','https://www.datadoghq-browser-agent.com/us1/v5/datadog-rum.js','DD_RUM')
window.DD_RUM.onReady(function() {
window.DD_RUM.init({
clientToken: 'puba71073f072643721169b68f352438710',
applicationId: '2e321b35-c76b-4073-8d04-cc9a10461793',
site: 'datadoghq.com',
service: 'web',
env: window._preloads.dd_env,
version: '8b4c00716b2bc0f7ca1fded553014f7e37b5b07a',
sessionSampleRate: 1,
sessionReplaySampleRate: 10,
trackUserInteractions: window._preloads.dd_ti,
trackResources: true,
trackLongTasks: true,
defaultPrivacyLevel: 'mask-user-input',
allowedTracingUrls: [/https?:\/\/(.+\/.)?substack(cdn)?\.com/]
});
})
</script>
<!-- End Datadog Analytics -->
<!-- Cloudflare Web Analytics -->
<script defer src="https://static.cloudflareinsights.com/beacon.min.js" data-cf-beacon='{"token": "216309cffb464db4b0e02daf0b8e8060"}'></script>
<!-- End Cloudflare Web Analytics -->
<!-- Fallback tracking pixels -->
<noscript>
<style>
#nojs-banner {
position: fixed;
bottom: 0;
left: 0;
padding: 16px 16px 16px 32px;
width: 100%;
box-sizing: border-box;
background: red;
color: white;
font-family: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
font-size: 13px;
line-height: 13px;
}
#nojs-banner a {
color: inherit;
text-decoration: underline;
}
</style>
<div id="nojs-banner">
This site requires JavaScript to run correctly. Please <a href="https://enable-javascript.com/" target="_blank">turn on JavaScript</a> or unblock scripts
</div>
</noscript>
<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" nonce="nQGtn4EHptwtQVZSM4a71A" data-cf-beacon='{"version":"2024.11.0","token":"68cfe66b5c4749e2ba64d4d9640c04c0","spa":2}' crossorigin="anonymous"></script>
</body>
</html>