483 lines
144 KiB
HTML
483 lines
144 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en" dir="ltr">
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="norton-safeweb-site-verification" content="24usqpep0ejc5w6hod3dulxwciwp0djs6c6ufp96av3t4whuxovj72wfkdjxu82yacb7430qjm8adbd5ezlt4592dq4zrvadcn9j9n-0btgdzpiojfzno16-fnsnu7xd" />
|
||
|
||
|
||
<link rel="preconnect" href="https://substackcdn.com" />
|
||
|
||
|
||
|
||
<title data-rh="true">AWS Networking, Environments and You - by Charity Majors</title>
|
||
|
||
<meta data-rh="true" name="theme-color" content="#ffffff"/><meta data-rh="true" property="og:type" content="article"/><meta data-rh="true" property="og:title" content="AWS Networking, Environments and You"/><meta data-rh="true" name="twitter:title" content="AWS Networking, Environments and You"/><meta data-rh="true" name="description" content="Last week we hit an important milestone in the life of our baby startup: a functional production environment, with real data flowing from ingestion to storage to serving queries out!"/><meta data-rh="true" property="og:description" content="Last week we hit an important milestone in the life of our baby startup: a functional production environment, with real data flowing from ingestion to storage to serving queries out!"/><meta data-rh="true" name="twitter:description" content="Last week we hit an important milestone in the life of our baby startup: a functional production environment, with real data flowing from ingestion to storage to serving queries out!"/><meta data-rh="true" property="og:image" content="https://substackcdn.com/image/fetch/$s_!khu1!,f_auto,q_auto:best,fl_progressive:steep/https%3A%2F%2Fcharitydotwtf.substack.com%2Ftwitter%2Fsubscribe-card.jpg%3Fv%3D-281824914%26version%3D9"/><meta data-rh="true" name="twitter:image" content="https://substackcdn.com/image/fetch/$s_!khu1!,f_auto,q_auto:best,fl_progressive:steep/https%3A%2F%2Fcharitydotwtf.substack.com%2Ftwitter%2Fsubscribe-card.jpg%3Fv%3D-281824914%26version%3D9"/><meta data-rh="true" name="twitter:card" content="summary"/><meta data-rh="true" property="article:modified_time" content="2025-12-16T01:00:58.582Z"/>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<style>
|
||
@layer legacy, pencraftReset, pencraft;
|
||
</style>
|
||
|
||
|
||
<link rel="preload" as="style" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
|
||
|
||
|
||
|
||
<link rel="preload" as="font" href="https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M5knjsS_ul.woff2" crossorigin />
|
||
|
||
|
||
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/11100.9900854a.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/11909.73f9f9c6.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/93519.6640a5b2.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/42118.cba90b72.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/39522.25128f14.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/47358.668d8a3c.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/94881.cb76fe04.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27386.72f00430.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/49234.f15fc605.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/50629.97d2663c.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/48409.0588981c.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/11983.972021ca.css" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27313.92c631cb.css" />
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=0, viewport-fit=cover" />
|
||
<meta name="author" content="Charity Majors" />
|
||
<meta property="og:url" content="https://charity.wtf/p/aws-networking-environments-and-you" />
|
||
|
||
|
||
<link rel="canonical" href="https://charity.wtf/p/aws-networking-environments-and-you" />
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="shortcut icon" href="https://substackcdn.com/image/fetch/$s_!jvlR!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Ffavicon.ico">
|
||
|
||
|
||
|
||
<link rel="icon" type="image/png" sizes="16x16" href="https://substackcdn.com/image/fetch/$s_!N42a!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Ffavicon-16x16.png">
|
||
|
||
|
||
|
||
<link rel="icon" type="image/png" sizes="32x32" href="https://substackcdn.com/image/fetch/$s_!hIAJ!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Ffavicon-32x32.png">
|
||
|
||
|
||
|
||
<link rel="icon" type="image/png" sizes="48x48" href="https://substackcdn.com/image/fetch/$s_!-tuJ!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Ffavicon-48x48.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="57x57" href="https://substackcdn.com/image/fetch/$s_!_ZAb!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-57x57.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="60x60" href="https://substackcdn.com/image/fetch/$s_!garv!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-60x60.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="72x72" href="https://substackcdn.com/image/fetch/$s_!YuQM!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-72x72.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="76x76" href="https://substackcdn.com/image/fetch/$s_!vaZL!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-76x76.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="114x114" href="https://substackcdn.com/image/fetch/$s_!Hvv-!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-114x114.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="120x120" href="https://substackcdn.com/image/fetch/$s_!DfUU!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-120x120.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="144x144" href="https://substackcdn.com/image/fetch/$s_!TIIj!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-144x144.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="152x152" href="https://substackcdn.com/image/fetch/$s_!Genc!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-152x152.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="167x167" href="https://substackcdn.com/image/fetch/$s_!3nCK!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-167x167.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="180x180" href="https://substackcdn.com/image/fetch/$s_!I9xu!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-180x180.png">
|
||
|
||
|
||
|
||
<link rel="apple-touch-icon" sizes="1024x1024" href="https://substackcdn.com/image/fetch/$s_!R7My!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F481e6ddf-7887-42bb-8be9-5e08437c0550%2Fapple-touch-icon-1024x1024.png">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="alternate" type="application/rss+xml" href="/feed" title="charity.wtf"/>
|
||
|
||
|
||
|
||
|
||
<style>
|
||
@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8on7mTNmnUHowCw.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8onXmTNmnUHowCw.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8onTmTNmnUHowCw.woff2) format('woff2');unicode-range:U+0100-02AF,U+0304,U+0308,U+0329,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20CF,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Spectral';font-style:italic;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCt-xNNww_2s0amA9M8onrmTNmnUHo.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+2074,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M9knjsS_ulYHs.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M2knjsS_ulYHs.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M3knjsS_ulYHs.woff2) format('woff2');unicode-range:U+0100-02AF,U+0304,U+0308,U+0329,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20CF,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Spectral';font-style:normal;font-weight:400;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCr-xNNww_2s0amA9M5knjsS_ul.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+2074,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3FafaPWnIIMrY.woff2) format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3OafaPWnIIMrY.woff2) format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3PafaPWnIIMrY.woff2) format('woff2');unicode-range:U+0100-02AF,U+0304,U+0308,U+0329,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20CF,U+2113,U+2C60-2C7F,U+A720-A7FF}@font-face{font-family:'Spectral';font-style:normal;font-weight:600;font-display:fallback;src:url(https://fonts.gstatic.com/s/spectral/v13/rnCs-xNNww_2s0amA9vmtm3BafaPWnII.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+2074,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
|
||
|
||
</style>
|
||
|
||
|
||
|
||
|
||
|
||
<style data-pub-theme-prefix>:root{--background_pop:#ff6719;--cover_bg_color:#FFFFFF;--cover_bg_color_secondary:#f0f0f0;--background_pop_darken:#ff5600;--print_on_pop:#ffffff;--print_pop:#ff6719;--cover_print_primary:#363737;--cover_print_secondary:#757575;--cover_print_tertiary:#b6b6b6;--cover_border_color:#ff6719;--background_contrast_1:#f0f0f0;--background_contrast_2:#dddddd;--background_contrast_3:#b7b7b7;--background_contrast_4:#929292;--background_contrast_5:#515151;--color_theme_detail:#e6e6e6;--background_contrast_pop:rgba(255, 103, 25, 0.4);--theme_bg_is_dark:0;--background_pop_rgb:255, 103, 25;}:root:where(:has(> body:not(.app-theme))){--color_theme_bg_pop:#FF6719;--background_pop:#FF6719;--cover_bg_color:#FFFFFF;--cover_bg_color_secondary:#f0f0f0;--background_pop_darken:#ff5600;--print_on_pop:#ffffff;--color_theme_bg_pop_darken:#ff5600;--color_theme_print_on_pop:#ffffff;--color_theme_bg_pop_20:rgba(255, 103, 25, 0.2);--color_theme_bg_pop_30:rgba(255, 103, 25, 0.3);--print_pop:#ff6719;--color_theme_accent:#ff6719;--cover_print_primary:#363737;--cover_print_secondary:#757575;--cover_print_tertiary:#b6b6b6;--cover_border_color:#ff6719;--home_hero:newspaper;--home_posts:list;--web_bg_color:#ffffff;--background_contrast_1:#f0f0f0;--background_contrast_2:#dddddd;--background_contrast_3:#b7b7b7;--background_contrast_4:#929292;--background_contrast_5:#515151;--color_theme_bg_contrast_1:#f0f0f0;--color_theme_bg_contrast_2:#dddddd;--color_theme_bg_contrast_3:#b7b7b7;--color_theme_bg_contrast_4:#929292;--color_theme_bg_contrast_5:#515151;--color_theme_bg_elevated:#ffffff;--color_theme_bg_elevated_secondary:#f0f0f0;--color_theme_bg_elevated_tertiary:#dddddd;--color_theme_detail:#e6e6e6;--background_contrast_pop:rgba(255, 103, 25, 0.4);--color_theme_bg_contrast_pop:rgba(255, 103, 25, 0.4);--theme_bg_is_dark:0;--print_on_web_bg_color:#363737;--print_secondary_on_web_bg_color:#868787;--background_pop_rgb:255, 103, 25;--color_theme_bg_pop_rgb:255, 103, 25;--color_theme_accent_rgb:255, 103, 25;}</style>
|
||
|
||
|
||
<link rel="stylesheet" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
|
||
|
||
|
||
<style data-pub-theme-postfix></style>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</head>
|
||
|
||
<body class="">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div id="entry">
|
||
<div id="main" style="--post-preview-border-radius:var(--border-radius-xs);" class="main typography use-theme-bg"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div data-testid="navbar" class="main-menu"><div class="mainMenuContent-DME8DR"><div style="position:relative;height:71px;" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-20 pc-paddingRight-20 pc-justifyContent-space-between pc-alignItems-center pc-reset border-bottom-detail-k1F6C4 topBar-pIF0J1"><div style="flex-basis:0px;flex-grow:1;" class="logoContainer-p12gJb"><div class="pencraft pc-display-flex pc-gap-40 pc-alignItems-center pc-reset"><a tabindex="-1" href="https://substack.com?utm_content=island-substack-logo" class="pencraft pc-display-flex pc-width-20 pc-height-20 pc-justifyContent-center pc-alignItems-center pc-reset"><svg aria-hidden="true" style="transform:scale(1.4);" width="16" height="18" viewBox="0 0 16 18" fill="#ff6719" stroke-width="1.8" stroke="none" xmlns="http://www.w3.org/2000/svg"><g><path d="M16 4H0V6H16V4Z"></path><path d="M0 8V18L7.9993 13.534L16 18V8H0Z"></path><path d="M16 0H0V2H16V0Z"></path></g></svg></a></div></div><div style="flex-grow:0;" class="titleContainer-DJYq5v"><h1 class="pencraft pc-reset font-pub-headings-FE5byy reset-IxiVJZ title-oOnUGd"><a href="/" class="pencraft pc-display-contents pc-reset">Charity Majors</a></h1></div><div style="flex-basis:0px;flex-grow:1;" class="pencraft pc-display-flex pc-justifyContent-flex-end pc-alignItems-center pc-reset buttonsContainerContainer-ThaN_w"><div class="buttonsContainer-SJBuep"><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-flex-end pc-alignItems-center pc-reset navbar-buttons"><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><span data-state="closed"><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></span><button tabindex="0" type="button" aria-label="Share Publication" id="radix-P0-4" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg></button></div><button tabindex="0" type="button" data-testid="noncontributor-cta-button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_primary-RfbeYt size_md-gCDS3o">Subscribe</button><button tabindex="0" data-native="true" type="button" data-href="https://substack.com/sign-in?redirect=%2Fp%2Faws-networking-environments-and-you&for_pub=charitydotwtf" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_tertiary-rlke8z size_md-gCDS3o">Sign in</button></div></div></div></div></div><div style="height:72px;"></div></div></div><div><script nonce="uxOuplozQoxmNQbU1vPabg" type="application/ld+json">{"@context":"https://schema.org","@type":"NewsArticle","url":"https://charity.wtf/p/aws-networking-environments-and-you","mainEntityOfPage":"https://charity.wtf/p/aws-networking-environments-and-you","headline":"AWS Networking, Environments and You","description":"Last week we hit an important milestone in the life of our baby startup: a functional production environment, with real data flowing from ingestion to storage to serving queries out!","image":[],"datePublished":"2016-03-23T18:20:31+00:00","dateModified":"2016-03-23T18:20:31+00:00","isAccessibleForFree":true,"author":[{"@type":"Person","name":"Charity Majors","url":"https://substack.com/@charitydotwtf","description":"cofounder and CTO of honeycomb.io; pioneered modern observability. co-author of O'Reilly books \"Database Reliability Engineering\" and \"Observability Engineering\", now wrapping up the 2nd ed. loves free software, free speech, and peaty single malts.","identifier":"user:32306597","image":{"@type":"ImageObject","contentUrl":"https://substackcdn.com/image/fetch/$s_!EAp-!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!EAp-!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg"}}],"publisher":{"@type":"Organization","name":"charity.wtf","url":"https://charity.wtf","description":"observability, tech advice, honeycomb.io, etc","interactionStatistic":{"@type":"InteractionCounter","name":"Subscribers","interactionType":"https://schema.org/SubscribeAction","userInteractionCount":1000},"identifier":"pub:2935724","logo":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!Tg4H!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F194fc089-9e91-4a15-85cc-5da3fca53eed_1280x1280.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!Tg4H!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F194fc089-9e91-4a15-85cc-5da3fca53eed_1280x1280.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!Tg4H!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F194fc089-9e91-4a15-85cc-5da3fca53eed_1280x1280.png"},"image":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!Tg4H!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F194fc089-9e91-4a15-85cc-5da3fca53eed_1280x1280.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!Tg4H!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F194fc089-9e91-4a15-85cc-5da3fca53eed_1280x1280.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!Tg4H!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F194fc089-9e91-4a15-85cc-5da3fca53eed_1280x1280.png"}},"interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":0},{"@type":"InteractionCounter","interactionType":"https://schema.org/ShareAction","userInteractionCount":0},{"@type":"InteractionCounter","interactionType":"https://schema.org/CommentAction","userInteractionCount":0}]}</script><script nonce="uxOuplozQoxmNQbU1vPabg" type="application/ld+json">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"charity.wtf","item":"https://charity.wtf"},{"@type":"ListItem","position":2,"name":"AWS Networking, Environments and You","item":"https://charity.wtf/p/aws-networking-environments-and-you"}]}</script><div aria-label="Post" role="main" class="single-post-container"><div class="container"><div class="single-post"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><article class="typography newsletter-post post"><div role="region" aria-label="Post header" class="post-header"><h1 dir="auto" class="post-title published title-X77sOw">AWS Networking, Environments and You</h1><div aria-label="Post UFI" role="region" class="pencraft pc-display-flex pc-flexDirection-column pc-paddingBottom-16 pc-reset"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-gap-12 pc-alignItems-center pc-reset byline-wrapper"><div class="pencraft pc-display-flex pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-row pc-gap-8 pc-alignItems-center pc-justifyContent-flex-start pc-reset"><div style="--scale:36px;--offset:9px;--border-width:4.5px;" class="pencraft pc-display-flex pc-flexDirection-row pc-alignItems-center pc-justifyContent-flex-start pc-reset ltr-qDBmby"><a href="https://substack.com/@charitydotwtf" aria-label="View Charity Majors's profile" class="pencraft pc-display-contents pc-reset"><div style="--scale:36px;" tabindex="0" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA animate-XFJxE4 outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 pressable-sm-YIJFKJ showFocus-sk_vEm container-TAtrWj interactive-UkK0V6 avatar-u8q6xB last-JfNEJ_"><div style="--scale:36px;" title="Charity Majors" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!EAp-!,w_36,h_36,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg 1x, https://substackcdn.com/image/fetch/$s_!EAp-!,w_72,h_72,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg 2x, https://substackcdn.com/image/fetch/$s_!EAp-!,w_108,h_108,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg 3x"/><img src="https://substackcdn.com/image/fetch/$s_!EAp-!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg" alt="Charity Majors's avatar" srcset="https://substackcdn.com/image/fetch/$s_!EAp-!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg 1x, https://substackcdn.com/image/fetch/$s_!EAp-!,w_72,h_72,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg 2x, https://substackcdn.com/image/fetch/$s_!EAp-!,w_108,h_108,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg 3x" width="36" height="36" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div></a></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><div class="pencraft pc-reset color-primary-zABazT line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><span style="min-width:0;" data-state="closed"><a href="https://substack.com/@charitydotwtf" class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ">Charity Majors</a></span></div><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><div class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><time datetime="2016-03-23T18:20:31.000Z" title="2016-03-23T18:20:31.000Z">Mar 23, 2016</time></div></div></div></div></div><div class="pencraft pc-display-flex pc-gap-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg></button></div><button tabindex="0" type="button" aria-label="View comments (0)" data-href="https://charity.wtf/p/aws-networking-environments-and-you/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg></button><button tabindex="0" type="button" id="radix-P0-27" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div><div class="visibility-check"></div><div><div class="dt-post-body"><div class="available-content"><div dir="auto" class="body markup"><p><span>Last week we hit an important milestone in the life of our baby </span><a href="http://twitter.com/houndexplorer">startup</a><span>: a functional production environment, with real data flowing from ingestion to storage to serving queries out! (Of course everything promptly exploded , but whatever.)</span></p><p>This got me started thinking about how to safely stage and promote terraform changes, in order to isolate the blast radius of certain destructive changes. I tweeted some random thing about it,</p><blockquote><p>hey, anyone have a strong argument for one VPC per env {staging,prod} vs single VPC with subnets for envs, managing isolation w SGs/routes?</p><p><span>— Charity Majors (@mipsytipsy) </span><a href="https://twitter.com/mipsytipsy/status/710224093027524608?ref_src=twsrc%5Etfw">March 16, 2016</a></p></blockquote><p><span>… and two surprisingly fun things happened. I learned some new things about AWS that I didn’t know (after YEARS of using it). And hilariously, I learned that I and lots of other people still believe things about AWS that </span><strong>aren’t actually true </strong><span>anymore.</span></p><p>That’s why I decided to write this. I’m not gonna get super religious on you, but I want to recap the current set of common and/or best practices for managing environments, network segmentation, and security automation in AWS, and I also want to remind you that this post will be out of date as soon as I publish it, because things are changing pretty crazy fast.</p><h2 class="header-anchor-post">Multiple Environments: Why I Should Care?<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§multiple-environments-why-i-should-care" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181561603/multiple-environments-why-i-should-care" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>An environment exists to provide resource isolation. You might want this for security reasons, or testability, or performance, or just so you can give your developers a place to fuck around and not hurt anything.</p><p>Maybe you run a bunch of similar production environments so you can give your customers security guarantees or avoid messy performance cotenancy stuff. Then you need a template for stamping out lots of these environments, and you need to be *really* sure they can’t leak into each other.</p><p>Or maybe you are more concerned about the vast oceans of things you need to care about beyond whatever unit tests or functional tests are running on your laptop. Like: capacity planning, load testing, validating storage changes against production workloads, exercising failovers, etc. Any scary changes you have, you need a production-like env to practice in.</p><p><span>Bottom line: If</span><strong> you can’t spin up a full copy of your infra and test it, you don’t actually have “infrastructure as code”.</strong><span> You just have … some code, and duct tape.</span></p><p><a href="https://twitter.com/phinze/status/710227204349624321">https://twitter.com/phinze/status/710227204349624321</a></p><p>The basics are simple:</p><ul><li><p>Non-production environments must be walled off from production as strongly as possible. You should NEVER be able to accidentally to connect to a prod db from staging (or from one prod env to another).</p></li><li><p>Production and non-production environments (or all other prod envs) should share as much of the same tooling and code paths as possible. Like, some amount asymptotically approaching 100%. Any gaps there will inevitably, eventually bite you in the ass.</p></li></ul><h2 class="header-anchor-post">Managing Multiple Environments in AWS<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§managing-multiple-environments-in-aws" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181561603/managing-multiple-environments-in-aws" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>There are baaaasically three patterns that people use to manage multiple environments in AWS these days:</p><ol><li><p>One AWS billing account and one flat network (VPC or Classic), with isolation performed by routes or security groups.</p></li><li><p>Many AWS accounts with consolidated billing. Each environment is a separate account (often maps to one acct per customer).</p></li><li><p>One AWS billing account and many VPCs, where each environment ~= its own VPC.</p></li></ol><p>Let’s start old school with a flat network.</p><h3 class="header-anchor-post">1: One Account, One Flattish Network<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§1-one-account-one-flattish-network" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181561603/1-one-account-one-flattish-network" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>This is what basically everyone did before VPC. (And ummm let’s be honest, lots of us kept it up for a while because GOD networking is such a pain.)</p><p>In EC2 Classic the best you got was security groups. And — unlike VPC security groups — you couldn’t stack them, or change the security group of an instance type without destroying it, and there was a crazy low hard cap on the (# of secgroup rules * # of secgroups). You could kind of gently “suggest” environments with things like DNS subdomains and config management, and sometimes you would see people literally just roll over and resort to $ENV variables.</p><p>Most people either a) gave up and just had a flat network, or b) this happened.</p><blockquote><p><a href="https://twitter.com/mipsytipsy?ref_src=twsrc%5Etfw">@mipsytipsy</a><span> we use a single vpc and more sgs than instances, it's a tangled legacy mess. I'd rather go for vpc per env/team whatever</span></p><p><span>— Jan B. (@bracki) </span><a href="https://twitter.com/bracki/status/710230597977956353?ref_src=twsrc%5Etfw">March 16, 2016</a></p></blockquote><p>At Parse we did a bunch of complicated security groups plus chef environments that let us spin up staging clusters and the occasional horrible silo’d production stack for exceptional customer requirements. Awful.</p><p><em>VPC has made this better.</em><span> Even if you’re still using a flat network model. You can now manage your route tables, stack security groups and IAM rules and reapply to existing nodes without destroying the node or dropping your internet connections. You can define private network subnets with NAT gateways, etc.</span></p><p><a href="https://twitter.com/sudarkoff/status/710264656367996928">https://twitter.com/sudarkoff/status/710264656367996928</a></p><p>Some people tell me they are using a single VPC with network ACLs to separate environments for ease of use, because you can reuse security groups across environments. Honestly this seems a bit more like a bug than a feature to me, because a) you give up isolation and b) I don’t see how that helps you have a versioned, tested stack.</p><p><a href="https://twitter.com/tomtheguvnor/status/710226892633333762">https://twitter.com/tomtheguvnor/status/710226892633333762</a></p><p>Ok moving on to the opposite of the spectrum, the crazy kids who are doing tens (or TENS OF THOUSANDS) of linked accounts.</p><h3 class="header-anchor-post">2: One AWS Account Per Environment<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§2-one-aws-account-per-environment" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181561603/2-one-aws-account-per-environment" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p><span>A surprising number of people adopted this model in the bad old EC2 Classic days, not because they necessarily wanted it but because they need a stronger security model and looser resource caps. This is why AWS released </span><a href="http://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/consolidated-billing.html">Consolidated Billing</a><span> way back in 2010.</span></p><p><span>I actually learned a lot this week about the multi-account model! Like that you can create </span><a href="http://docs.aws.amazon.com/IAM/latest/UserGuide/tutorial-cross-account-with-roles.html">IAM roles that span accounts</a><span>, or that you can </span><a href="http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/sharingamis-explicit.html">share AMIs between accounts.</a><span>. This model is complicated, but there are some real benefits. Like real, actual, hardcore security/perf isolation. And you will run into fewer resource limits than jamming everything into a single account, and revoking/managing IAM creds is clearer.</span></p><p>Security nerds love this model, but it’s not clear that …. literally anyone else does.</p><blockquote><p><a href="https://twitter.com/mipsytipsy?ref_src=twsrc%5Etfw">@mipsytipsy</a><span> i did one full AWS account per ENV as a way to fully segment (using consolidated billing to manage). felt "safer"</span></p><p><span>— petecheslock@hachyderm.io (@petecheslock) </span><a href="https://twitter.com/petecheslock/status/710277140764295168?ref_src=twsrc%5Etfw">March 17, 2016</a></p></blockquote><p>Some things that make me despise it without even trying it:</p><ul><li><p>AWS billing is ALREADY a horrendous hairball coughed up by the world’s ugliest cat, I can’t even imagine trying to wrangle multiple accounts.</p></li><li><p>It’s more expensive, you incur extra billing costs.</p></li><li><p>Having to explicitly list any resource you want to share between any accounts just makes me want to tear my hair out strand by strand while screaming on a street corner</p></li><li><p>Account creation API still has manual steps, like getting certs/keypairs.</p></li><li><p>You cannot make bulk changes to accounts, and AWS doesn’t like you having thousands of linked accounts. Also limits your flexibility with Reserved Instances.</p></li></ul><p><span>Here is a pretty reasonable </span><a href="https://segment.com/blog/rebuilding-our-infrastructure/">blog p0st</a><span> laying out some of the benefits though, and as you can see, there are plenty of other crazy people who like it. Mostly security nerds.</span></p><h3 class="header-anchor-post">3: One AWS Account, One VPC Per Environment<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§3-one-aws-account-one-vpc-per-environment" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181561603/3-one-aws-account-one-vpc-per-environment" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>I have saved the best for last. I think this is the best model, and the one I am adopting. You spin up a production VPC, a staging VPC, dev VPC, Travis-CI VPC. EVERYBODY GETS A VPC!#@!</p><p><span>One of those things that everybody seems to “know” but isn’t true is that you can’t have lots of VPCs. Yes, it is capped at 5 by default, and many people have stories about how they couldn’t get it raised, and that used to be true. But </span><strong>the hard cap is now 200</strong><span>, not 10, so VPC awayyyyyyyy my pretties!</span></p><blockquote><p><a href="https://twitter.com/mipsytipsy?ref_src=twsrc%5Etfw">@mipsytipsy</a><span> 1 account per env, many VPCs. IAM/VPC peer makes shit easy... we run many accounts, 200+ VPCs in our prod acc</span></p><p><span>— Ben Bromhead (@BenBromhead) </span><a href="https://twitter.com/BenBromhead/status/710239464530923520?ref_src=twsrc%5Etfw">March 16, 2016</a></p></blockquote><p>Here’s another reason to love VPC <-> env mapping: orchestration is finally coming around to the party. Even recently people were still trying to make chef-metal a thing, or developing their own coordination software from scratch with Boto, or just using the console and diffing and committing to git.</p><p><span>Dude, </span><em>stop. </em><span>We are past the point where you should default to using Terraform or CloudFormation for the bones of your infrastructure, for the things that rarely change. And once you’ve done that you’re most of the way to a reusable, testable stack.</span></p><p>Most of the cotenancy problems that account-per-env solved are a lot less compelling to me now that VPCs exist.</p><blockquote><p><a href="https://twitter.com/mipsytipsy?ref_src=twsrc%5Etfw">@mipsytipsy</a><span> if you're doing automation, sep VPC/accounts let you fully test that automation rather than "just" app code</span></p><p><span>— Adam (@uptill_3) </span><a href="https://twitter.com/uptill_3/status/710397938065866752?ref_src=twsrc%5Etfw">March 17, 2016</a></p></blockquote><p>VPCs are here to help you think about infrastructure like regular old code. Lots of VPCs are approximately as easy to manage as as one VPC. Unlike lots of accounts, which are there to give you headaches and one-offs and custom scripts and pointy-clicky shit and complicated horrible things to work around.</p><blockquote><p><a href="https://twitter.com/mipsytipsy?ref_src=twsrc%5Etfw">@mipsytipsy</a><span> We just have CloudFormation templates that take IP prefix and VPC ID as arguments; seems to have worked pretty well.</span></p><p><span>— Alex Rasmussen (@alexras) </span><a href="https://twitter.com/alexras/status/710230250504912896?ref_src=twsrc%5Etfw">March 16, 2016</a></p></blockquote><p>VPCs have some caveats of their own. Like, you can only assign a /16 to any VPC. If you’re using 4 availability zones and public subnets + natted private subnets, that’s only ~8k IPs per subnet/AZ pair. Shrug.</p><p><a href="https://aws.amazon.com/about-aws/whats-new/2016/03/announcing-support-for-security-group-references-in-a-peered-vpc/">You can peer security groups across different VPCs</a><span>, but not across regions (yet). Also, if you’re a terraform user, be aware that it handles VPC peering fine but doesn’t handle multiple accounts very well.</span></p><p>Lots of people seem to have had issues with security group limits per VPC, even though the limit is 500 and says it can be raised by request. I’m …. not sure what to think of that. I *feel* like if you’re building a thing with > 500 sec group rules on a single VPC, you’re probably doing something wrong.</p><h2 class="header-anchor-post">Test my code you piece of shit I dare you<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§test-my-code-you-piece-of-shit-i-dare-you" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181561603/test-my-code-you-piece-of-shit-i-dare-you" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p><span>Here’s the thing that got me excited about this from the start though, which is having the ability to do things like test terraform modules on a staging VPC from a branch before promoting the clean changes to master. If you plan on doing things like running bleeding-edge software in production *cough* you need allllll the guard rails and test coverage you can possibly scare up. </span><strong>VPCs help you get this</strong><span> in a massive way.</span></p><p>Super quick example, say you’re using adding a NAT gateway to your staging cluster, you would use the remote git source with your changes:</p><p><span>[code language=”javascript”]</span><br/><span>// staging</span><br/><span>module "aws_vpc" {</span><br/><span>source = "git::ssh://git@github.com/houndsh/infra.git//terraform/modules/aws_vpc?ref=charity.adding-nat-gw"</span><br/><span>env = ${var.env}</span><br/><span>…</span><br/><span>}</span><br/><span>[/code]</span></p><p>And then once you’ve validated the change, you simply merge your branch to master and run terraform plan/apply to production.</p><p><span>[code language=”javascript”]</span><br/><span>// production</span><br/><span>module "aws_vpc" {</span><br/><span>source = "github.com/houndsh/infra/terraform/modules/aws_vpc"</span><br/><span>env = "${var.env}"</span><br/><span>…</span><br/><span>}</span><br/><span>[/code]</span></p><p>And for GOD’S SAKE USE DIFFERENT STATE FILES FOR EACH VPC / ENVIRONMENT okayyyyyy but that is a different rant, not an AWS rant, so let’s move along.</p><h2 class="header-anchor-post">In Conclusion<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§in-conclusion" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://charity.wtf/i/181561603/in-conclusion" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>There are legit reasons to use all three of these models, and infinite variations upon them, and your use case is not my use case, blah blah.</p><p><span>But moving from one VPC to multiple VPCs is really not a big deal. I know a lot of us bear scars, but it is </span><em>nothing</em><span> like the horror that was trying to move from EC2 Classic to VPC.</span></p><p><a href="https://twitter.com/mistofvongola/status/710239882912747524">https://twitter.com/mistofvongola/status/710239882912747524</a></p><p>VPC has a steeper learning curve than Classic, but it is sooooo worth it. Every day I’m rollin up on some new toy you get with VPC (ICMP for ELBs! composable IAM host roles! new VPC NAT gateway!!!). The rate at which they’re releasing new shit for VPC is staggering, I can barely keep up.</p><blockquote><p><a href="https://twitter.com/mipsytipsy?ref_src=twsrc%5Etfw">@mipsytipsy</a><span> we have multiple aws accounts with one vpc per. Private and public subnets in that vpc. Most non ELB aws resources in private</span></p><p><span>— Alfonso (@alfonso__c) </span><a href="https://twitter.com/alfonso__c/status/710232944162504704?ref_src=twsrc%5Etfw">March 16, 2016</a></p></blockquote><p>Alright, said I wasn’t gonna get religious but I totally lied.</p><p><strong>VPC is the future and it is awesome, and unless you have some VERY SPECIFIC AND CONVINCING reasons to do otherwise, you should be spinning up a VPC per environment with orchestration and prob doing it from CI on every code commit, almost like it’s just like, you know, code</strong><span>.</span></p><p>That you need to test.</p><p>Cause it is.</p><p><em>(thank you to everybody who chatted with me and taught me things and gave awesome feedback!! knuckle tatts credit to @mosheroperandi)</em></p></div></div><div class="visibility-check"></div></div><div class="post-footer"><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG border-top-detail-themed-k9TZAY border-bottom-detail-themed-Ua9186 post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg></button></div><button tabindex="0" type="button" aria-label="View comments (0)" data-href="https://charity.wtf/p/aws-networking-environments-and-you/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg></button><button tabindex="0" type="button" id="radix-P0-56" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div></article></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="visibility-check"></div><div id="discussion" class="pencraft pc-display-flex pc-flexDirection-column pc-gap-16 pc-paddingTop-32 pc-paddingBottom-32 pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-32 pc-reset container"><h4 class="pencraft pc-reset line-height-24-jnGwiv font-display-nhmvtD size-20-P_cSRT weight-bold-DmI9lw reset-IxiVJZ">Discussion about this post</h4><div class="pencraft pc-alignSelf-flex-start pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Select discussion type" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-70-content-comments" data-state="active" id="radix-P0-70-trigger-comments" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Comments</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-70-content-restacks" data-state="inactive" id="radix-P0-70-trigger-restacks" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Restacks</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div></div></div><div id="substack-comments" class="single-post-section comments-section"><div class="container"><div class="visibility-check"></div><div data-test-id="comment-input" class="pencraft pc-display-flex pc-reset flex-grow-rzmknG"><form class="comment-input-form form-CkZ7Kt"><div style="--scale:32px;" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><div style="--scale:32px;" title="User" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png" alt="User's avatar" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x" width="32" height="32" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-8 pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-flexDirection-column pc-boxShadow-xs pc-reset bg-primary-zk6FDl flex-grow-rzmknG border-detail-EGrm7T pc-borderRadius-sm"><textarea name="body" placeholder="Write a comment..." aria-label="Write a comment..." rows="4" class="pencraft input-qHk4bN autogrowing-_ipn9Y textarea-GbEjRX inputText-pV_yWb unstyled-IJLUJU"></textarea></div><div data-state="closed" class="pencraft pc-display-flex pc-flexDirection-column pc-reset overflow-hidden-WdpwT6"></div></div></form></div></div></div></div><div class="single-post-section"><div class="container"><div class="visibility-check"></div><div style="margin-left:-8px;margin-right:-8px;" aria-label="Top Posts Footer" role="region" class="pencraft pc-paddingTop-24 pc-paddingBottom-24 pc-reset"><div class="portable-archive empty-list"><div aria-label="Archive sort tabs" role="navigation" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-8 pc-paddingRight-8 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Tabs" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-83-content-top" data-state="active" id="radix-P0-83-trigger-top" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Top</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-83-content-new" data-state="inactive" id="radix-P0-83-trigger-new" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Latest</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-83-content-community" data-state="inactive" id="radix-P0-83-trigger-community" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Discussions</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></div><div class="portable-archive-list"><p class="portable-archive-empty dt-archive-empty-state">No posts</p></div></div></div></div></div><div class="visibility-check"></div><div class="pencraft pc-display-contents pc-reset pubInvertedTheme-U483dz"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-justifyContent-center pc-padding-48 pc-mobile-padding-24 pc-reset bg-primary-zk6FDl container-jsOc9L"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-gap-24 pc-flexWrap-wrap pc-reset content-jLbYeh"><h3 class="pencraft pc-reset color-primary-zABazT align-center-y7ZD4w line-height-28-s562kJ font-display-nhmvtD size-24-lFU3ly weight-semibold-uqA4FV reset-IxiVJZ">Ready for more?</h3><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe_footer"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB emailInputOnAccentBackground-TfaCGr input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonOnAccentBackground-vmEt94 buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div></div></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset container-K5XPov"></div><div style="--color-app-light-accent:rgb(255 103 25 / 1);--color-app-light-accent-hover:rgb(255 86 0 / 1);--color-app-light-accent-inverse:rgb(255 255 255 / 1);--color-app-light-accent-text:rgb(255 103 25 / 1);--color-app-dark-accent:rgb(255 103 25 / 1);--color-app-dark-accent-hover:rgb(255 132 73 / 1);--color-app-dark-accent-inverse:rgb(255 255 255 / 1);--color-app-dark-accent-text:rgb(255 103 25 / 1);--color-app-light-bg-accent:rgb(255 103 25 / 0.2);--color-app-light-bg-accent-hover:rgb(255 103 25 / 0.3);--color-app-dark-bg-accent:rgb(255 103 25 / 0.2);--color-app-dark-bg-accent-hover:rgb(255 103 25 / 0.3);--color-app-light-fg-primary:rgb(54 55 55 / 1);--color-app-light-fg-secondary:rgb(134 135 135 / 1);--color-app-light-fg-tertiary:rgb(146 146 146 / 1);--color-app-dark-fg-primary:rgb(255 255 255 / 0.9);--color-app-dark-fg-secondary:rgb(255 255 255 / 0.6);--color-app-dark-fg-tertiary:rgb(255 255 255 / 0.4);--color-app-light-bg-primary:rgb(255 255 255 / 1);--color-app-light-bg-primary-hover:rgb(240 240 240 / 1);--color-app-light-bg-secondary:rgb(240 240 240 / 1);--color-app-light-bg-tertiary:rgb(221 221 221 / 1);--color-app-light-bg-quaternary:rgb(183 183 183 / 1);--color-app-dark-bg-primary:rgb(22 23 24 / 1);--color-app-dark-bg-primary-hover:rgb(27 28 29 / 1);--color-app-dark-bg-secondary:rgb(35 37 37 / 1);--color-app-dark-bg-tertiary:rgb(54 55 55 / 1);--color-app-dark-bg-quaternary:rgb(90 91 91 / 1);" class="pencraft pc-display-contents pc-reset appTheme-xUAu9p"><div class="footer-wrap publication-footer"><div class="visibility-check"></div><div class="footer profile-themed-background"><div class="container"><div class="footer-terms"><span>© 2026 Charity Majors</span><span> · </span><a href="https://substack.com/privacy" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Privacy</a><span> ∙ </span><a href="https://substack.com/tos" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Terms</a><span> ∙ </span><a href="https://substack.com/ccpa#personal-data-collected" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Collection notice</a></div><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-center pc-alignItems-center pc-reset footerButtons-ap9Sk7"><a data-native="true" href="https://substack.com/signup?utm_source=substack&utm_medium=web&utm_content=footer" class="footerSubstackCta-v5HWfj"><svg aria-hidden="true" width="1000" height="1000" viewBox="0 0 1000 1000" fill="#ff6719" stroke-width="1.8" stroke="none" xmlns="http://www.w3.org/2000/svg"><g><path d="M764.166 348.371H236.319V419.402H764.166V348.371Z"></path><path d="M236.319 483.752V813.999L500.231 666.512L764.19 813.999V483.752H236.319Z"></path><path d="M764.166 213H236.319V284.019H764.166V213Z"></path></g></svg> Start your Substack</a><a data-native="true" href="https://substack.com/app/app-store-redirect?utm_campaign=app-marketing&utm_content=web-footer-button" class="footerSubstackCta-v5HWfj getTheApp-Yk3w1O noIcon-z7v9D8">Get the app</a></div><div translated class="pencraft pc-reset reset-IxiVJZ footer-slogan-blurb"><a href="https://substack.com" data-native="true">Substack</a> is the home for great culture</div></div></div></div></div></div><div role="region" aria-label="Notifications (F8)" tabindex="-1" style="pointer-events:none;"><ol tabindex="-1" style="--offset:0px;z-index:1000;" class="viewport-_BM4Bg"></ol></div><div></div><script suppressHydrationWarning nonce="uxOuplozQoxmNQbU1vPabg">window.__staticRouterHydrationData = JSON.parse("{\"loaderData\":{},\"actionData\":null,\"errors\":null}");</script>
|
||
</div>
|
||
|
||
|
||
<script src="https://js.sentry-cdn.com/6c2ff3e3828e4017b7faf7b63e24cdf8.min.js" crossorigin="anonymous"></script>
|
||
<script nonce="uxOuplozQoxmNQbU1vPabg">
|
||
window.Sentry && window.Sentry.onLoad(function() {
|
||
window.Sentry.init({
|
||
environment: window._preloads.sentry_environment,
|
||
dsn: window._preloads.sentry_dsn,
|
||
})
|
||
})
|
||
</script>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script nonce="uxOuplozQoxmNQbU1vPabg">window._preloads = JSON.parse("{\"cspNonce\":\"uxOuplozQoxmNQbU1vPabg\",\"isEU\":false,\"language\":\"en\",\"country\":\"US\",\"leaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"enabledLeaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"userLocale\":{\"language\":\"en\",\"region\":\"US\",\"source\":\"accept-language\"},\"base_url\":\"https://charity.wtf\",\"stripe_publishable_key\":\"pk_live_51QfnARLDSWi1i85FBpvw6YxfQHljOpWXw8IKi5qFWEzvW8HvoD8cqTulR9UWguYbYweLvA16P7LN6WZsGdZKrNkE00uGbFaOE3\",\"captcha_site_key\":\"6LeI15YsAAAAAPXyDcvuVqipba_jEFQCjz1PFQoz\",\"pub\":{\"apple_pay_disabled\":false,\"apex_domain\":null,\"author_id\":32306597,\"byline_images_enabled\":true,\"bylines_enabled\":true,\"chartable_token\":null,\"community_enabled\":true,\"copyright\":\"Charity Majors\",\"cover_photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/6e483de5-cf92-41be-beb1-b5af853e75f9_555x555.png\",\"created_at\":\"2024-08-26T05:48:54.313Z\",\"custom_domain_optional\":false,\"custom_domain\":\"charity.wtf\",\"default_comment_sort\":\"best_first\",\"default_coupon\":null,\"default_group_coupon\":null,\"default_show_guest_bios\":true,\"email_banner_url\":null,\"email_from_name\":\"Charity Majors\",\"email_from\":null,\"embed_tracking_disabled\":false,\"explicit\":false,\"expose_paywall_content_to_search_engines\":true,\"fb_pixel_id\":null,\"fb_site_verification_token\":null,\"flagged_as_spam\":false,\"founding_subscription_benefits\":null,\"free_subscription_benefits\":null,\"ga_pixel_id\":null,\"google_site_verification_token\":null,\"google_tag_manager_token\":null,\"hero_image\":null,\"hero_text\":\"observability, tech advice, honeycomb.io, etc\",\"hide_intro_subtitle\":null,\"hide_intro_title\":null,\"hide_podcast_feed_link\":false,\"homepage_type\":\"profile\",\"id\":2935724,\"image_thumbnails_always_enabled\":false,\"invite_only\":false,\"hide_podcast_from_pub_listings\":false,\"hide_podcast_from_pub_listings_prior_to\":null,\"language\":\"en\",\"logo_url_wide\":null,\"logo_url\":\"https://substackcdn.com/image/fetch/$s_!Tg4H!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F194fc089-9e91-4a15-85cc-5da3fca53eed_1280x1280.png\",\"minimum_group_size\":2,\"moderation_enabled\":true,\"name\":\"charity.wtf\",\"paid_subscription_benefits\":null,\"parsely_pixel_id\":null,\"chartbeat_domain\":null,\"payments_state\":\"disabled\",\"paywall_free_trial_enabled\":false,\"podcast_art_url\":null,\"paid_podcast_episode_art_url\":null,\"podcast_byline\":null,\"podcast_description\":null,\"podcast_enabled\":false,\"podcast_feed_url\":null,\"podcast_title\":null,\"post_preview_limit\":null,\"primary_user_id\":32306597,\"require_clickthrough\":false,\"show_pub_podcast_tab\":false,\"show_recs_on_homepage\":true,\"subdomain\":\"charitydotwtf\",\"subscriber_invites\":0,\"support_email\":null,\"theme_var_background_pop\":\"#FF6719\",\"theme_var_color_links\":false,\"theme_var_cover_bg_color\":null,\"trial_end_override\":null,\"twitter_pixel_id\":null,\"type\":\"newsletter\",\"post_reaction_faces_enabled\":true,\"is_personal_mode\":false,\"plans\":null,\"stripe_user_id\":null,\"stripe_country\":null,\"stripe_publishable_key\":null,\"stripe_platform_account\":null,\"automatic_tax_enabled\":null,\"author_name\":\"Charity Majors\",\"author_handle\":\"charitydotwtf\",\"author_photo_url\":\"https://substackcdn.com/image/fetch/$s_!EAp-!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg\",\"author_bio\":\"cofounder and CTO of honeycomb.io; pioneered modern observability. co-author of O'Reilly books \\\"Database Reliability Engineering\\\" and \\\"Observability Engineering\\\", now wrapping up the 2nd ed. loves free software, free speech, and peaty single malts.\",\"has_custom_tos\":false,\"has_custom_privacy\":false,\"theme\":{\"background_pop_color\":null,\"web_bg_color\":\"#ffffff\",\"cover_bg_color\":null,\"publication_id\":2935724,\"color_links\":null,\"font_preset_heading\":null,\"font_preset_body\":null,\"font_family_headings\":null,\"font_family_body\":null,\"font_family_ui\":null,\"font_size_body_desktop\":null,\"print_secondary\":null,\"custom_css_web\":null,\"custom_css_email\":null,\"home_hero\":\"newspaper\",\"home_posts\":\"list\",\"home_show_top_posts\":false,\"hide_images_from_list\":false,\"home_hero_alignment\":\"left\",\"home_hero_show_podcast_links\":true,\"default_post_header_variant\":null,\"custom_header\":null,\"custom_footer\":null,\"social_media_links\":null,\"font_options\":null,\"section_template\":null,\"custom_subscribe\":null,\"design_template\":null,\"design_template_options\":null},\"threads_v2_settings\":{\"first_thread_email_sent_at\":null,\"activated_at\":null,\"reader_thread_notifications_enabled\":false,\"create_thread_minimum_role\":\"contributor\",\"photo_replies_enabled\":true,\"boost_free_subscriber_chat_preview_enabled\":false,\"push_suppression_enabled\":false},\"default_group_coupon_percent_off\":null,\"default_group_coupon_include_founding\":null,\"pause_return_date\":null,\"has_posts\":true,\"has_recommendations\":true,\"first_post_date\":\"2015-12-27T01:57:32.000Z\",\"has_podcast\":false,\"has_free_podcast\":false,\"has_subscriber_only_podcast\":false,\"has_community_content\":true,\"rankingDetail\":\"Launched 2 years ago\",\"rankingDetailFreeIncluded\":\"Thousands of subscribers\",\"rankingDetailOrderOfMagnitude\":0,\"rankingDetailFreeIncludedOrderOfMagnitude\":1000,\"rankingDetailFreeSubscriberCount\":\"Over 6,000 subscribers\",\"rankingDetailByLanguage\":{\"ar\":{\"rankingDetail\":\"\u062A\u0645 \u0627\u0644\u0625\u0637\u0644\u0627\u0642 2 years ago\"},\"ca\":{\"rankingDetail\":\"S\u2019ha llan\u00E7at fa 2 anys\"},\"da\":{\"rankingDetail\":\"Lancering 2 \u00E5r\"},\"de\":{\"rankingDetail\":\"vor 2 Jahren gelauncht\"},\"es\":{\"rankingDetail\":\"Lanzado hace 2 a\u00F1os\"},\"fr\":{\"rankingDetail\":\"Lanc\u00E9 il y a 2 ann\u00E9es\"},\"id\":{\"rankingDetail\":\"Diluncurkan 2 tahun yang lalu\"},\"ja\":{\"rankingDetail\":\"\u958B\u59CB\u65E5 2\u5E74\u524D\"},\"ko\":{\"rankingDetail\":\"2\uB144 \uC804\uC5D0 \uCD9C\uC2DC\"},\"nb\":{\"rankingDetail\":\"Lansert 2 \u00E5r\"},\"nl\":{\"rankingDetail\":\"Gelanceerd 2 jaar geleden\"},\"pl\":{\"rankingDetail\":\"Uruchomiono 2 lat temu\"},\"pt\":{\"rankingDetail\":\"Lan\u00E7ado 2 anos\"},\"pt-br\":{\"rankingDetail\":\"Lan\u00E7ado 2 anos\"},\"en-gb\":{\"rankingDetail\":\"Launched 2 years ago\"},\"it\":{\"rankingDetail\":\"Lanciato 2 anni\"},\"tr\":{\"rankingDetail\":\"2 y\u0131l ba\u015Flat\u0131ld\u0131\"},\"sv\":{\"rankingDetail\":\"Lanserad 2 \u00E5r sedan\"},\"fi\":{\"rankingDetail\":\"Launched 2 vuotta\"},\"is\":{\"rankingDetail\":\"Launched 2 \u00E1r\"},\"zh-hans\":{\"rankingDetail\":\"\u4E8E2 \u5E74\u524D\u4E0A\u7EBF\"},\"zh-hant\":{\"rankingDetail\":\"\u65BC 2 \u5E74\u524D \u63A8\u51FA\"},\"en\":{\"rankingDetail\":\"Launched 2 years ago\"}},\"freeSubscriberCount\":\"6,000\",\"freeSubscriberCountOrderOfMagnitude\":\"6.3K+\",\"author_bestseller_tier\":0,\"author_badge\":{\"type\":\"subscriber\",\"tier\":5,\"accent_colors\":null},\"disable_monthly_subscriptions\":false,\"disable_annual_subscriptions\":false,\"hide_post_restacks\":false,\"notes_feed_enabled\":false,\"showIntroModule\":false,\"isPortraitLayout\":false,\"last_chat_post_at\":null,\"primary_profile_name\":\"Charity Majors\",\"primary_profile_photo_url\":\"https://substackcdn.com/image/fetch/$s_!EAp-!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg\",\"no_follow\":false,\"sponsorshipCampaigns\":{},\"paywall_chat\":\"free\",\"sections\":[],\"podcastTabInfo\":null,\"multipub_migration\":null,\"navigationBarItems\":[],\"has_active_perks\":false,\"contributors\":[{\"name\":\"Charity Majors\",\"handle\":\"charitydotwtf\",\"role\":\"admin\",\"owner\":true,\"user_id\":32306597,\"photo_url\":\"https://substackcdn.com/image/fetch/$s_!EAp-!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg\",\"bio\":\"cofounder and CTO of honeycomb.io; pioneered modern observability. co-author of O'Reilly books \\\"Database Reliability Engineering\\\" and \\\"Observability Engineering\\\", now wrapping up the 2nd ed. loves free software, free speech, and peaty single malts.\",\"status\":{\"bestsellerTier\":null,\"subscriberTier\":5,\"leaderboard\":null,\"vip\":false,\"badge\":{\"type\":\"subscriber\",\"tier\":5,\"accent_colors\":null},\"subscriber\":null}}],\"threads_v2_enabled\":true,\"viralGiftsConfig\":null,\"no_index\":false,\"tier\":2,\"can_set_google_site_verification\":true,\"can_have_sitemap\":true,\"founding_plan_name_english\":\"Founding Member\",\"base_url\":\"https://charity.wtf\",\"hostname\":\"charity.wtf\",\"is_on_substack\":false,\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"pageThemes\":{\"podcast\":null},\"supports_ip_content_unlock\":false,\"appTheme\":{\"colors\":{\"accent\":{\"name\":\"#ff6719\",\"primary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"primary_hover\":{\"r\":232,\"g\":83,\"b\":0,\"a\":1},\"primary_elevated\":{\"r\":232,\"g\":83,\"b\":0,\"a\":1},\"secondary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3},\"dark\":{\"primary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"primary_hover\":{\"r\":255,\"g\":132,\"b\":73,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":132,\"b\":73,\"a\":1},\"secondary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.8},\"secondary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.6},\"tertiary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.4},\"accent\":{\"r\":213,\"g\":66,\"b\":0,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"primary_elevated\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"secondary\":{\"r\":238,\"g\":238,\"b\":238,\"a\":1},\"secondary_elevated\":{\"r\":206.90096477355226,\"g\":206.90096477355175,\"b\":206.9009647735519,\"a\":1},\"tertiary\":{\"r\":219,\"g\":219,\"b\":219,\"a\":1},\"quaternary\":{\"r\":182,\"g\":182,\"b\":182,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}}}},\"portalAppTheme\":{\"colors\":{\"accent\":{\"name\":\"#FF6719\",\"primary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"primary_hover\":{\"r\":255,\"g\":86,\"b\":0,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"secondary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3},\"dark\":{\"primary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"primary_hover\":{\"r\":255,\"g\":132,\"b\":73,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":132,\"b\":73,\"a\":1},\"secondary\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"secondary\":{\"r\":134,\"g\":135,\"b\":135,\"a\":1},\"tertiary\":{\"r\":146,\"g\":146,\"b\":146,\"a\":1},\"accent\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":255,\"g\":103,\"b\":25,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"secondary\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"secondary_elevated\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"tertiary\":{\"r\":221,\"g\":221,\"b\":221,\"a\":1},\"quaternary\":{\"r\":183,\"g\":183,\"b\":183,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}},\"wordmark_bg\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"is_dark\":false}},\"logoPalette\":{\"Vibrant\":{\"rgb\":[180,92,88],\"population\":2},\"DarkVibrant\":{\"rgb\":[16,117,82],\"population\":12},\"LightVibrant\":{\"rgb\":[220,177,157],\"population\":9},\"Muted\":{\"rgb\":[158,133,126],\"population\":555},\"DarkMuted\":{\"rgb\":[58,63,68],\"population\":331},\"LightMuted\":{\"rgb\":[210,190,169],\"population\":99}}},\"confirmedLogin\":false,\"hide_intro_popup\":false,\"block_auto_login\":false,\"domainInfo\":{\"isSubstack\":false,\"customDomain\":\"charity.wtf\"},\"experimentFeatures\":{},\"experimentExposures\":{},\"siteConfigs\":{\"score_upsell_email\":\"control\",\"first_chat_email_enabled\":true,\"new_commenter_approval\":false,\"pub_update_opennode_api_key\":false,\"notes_video_max_duration_minutes\":15,\"show_content_label_age_gating_in_feed\":false,\"zendesk_automation_cancellations\":false,\"mfa_action_box_enabled\":false,\"publication_max_bylines\":35,\"no_contest_charge_disputes\":false,\"scheduled_post_share_assets_enabled\":false,\"publication_tabs_reorder\":false,\"comp_expiry_email_new_copy\":\"NONE\",\"free_unlock_required\":false,\"enable_section_group_pages\":false,\"enable_post_summarization\":false,\"live_stream_host_warning_message\":\"\",\"bitcoin_enabled\":false,\"minimum_ios_os_version\":\"17.0.0\",\"show_entire_square_image\":false,\"hide_subscriber_count\":false,\"fit_in_live_stream_player\":false,\"publication_author_display_override\":\"\",\"enable_saved_segments\":true,\"generate_pdf_tax_report\":false,\"hide_post_sidebar\":false,\"show_generic_post_importer\":false,\"enable_pledges_modal\":true,\"community_automod_non_english_enabled\":true,\"audio_clips_enabled\":false,\"enable_videos_page\":false,\"exempt_from_gtm_filter\":false,\"group_sections_and_podcasts_in_menu\":false,\"boost_optin_modal_enabled\":true,\"standards_and_enforcement_features_enabled\":false,\"pub_creation_captcha_behavior\":\"risky_pubs_or_rate_limit\",\"post_blogspot_importer\":false,\"enable_high_res_background_uploading\":false,\"pub_tts_override\":\"default\",\"disable_monthly_subscriptions\":false,\"skip_welcome_email\":false,\"chat_reader_thread_notification_default\":false,\"scheduled_pinned_posts\":false,\"disable_redirect_outbound_utm_params\":false,\"reader_gift_referrals_enabled\":true,\"dont_show_guest_byline\":false,\"like_comments_enabled\":true,\"enable_theme_editor_deferred_saves\":false,\"enable_sponsorship_campaigns_advanced\":false,\"temporal_livestream_ended_draft\":true,\"enable_pangram_ai_detection\":true,\"fallback_to_archive_search_on_section_pages\":false,\"livekit_track_egress_custom_base_url\":\"http://livekit-egress-custom-recorder-participant-test.s3-website-us-east-1.amazonaws.com\",\"welcome_screen_blurb_override\":\"\",\"like_posts_enabled\":true,\"twitter_player_card_enabled\":true,\"feed_promoted_user\":false,\"show_note_stats_for_all_notes\":false,\"section_specific_csv_imports_enabled\":false,\"disable_podcast_feed_description_cta\":false,\"bypass_profile_substack_logo_detection\":false,\"use_preloaded_player_sources\":false,\"list_pruning_enabled\":false,\"facebook_connect\":false,\"opt_in_to_sections_during_subscribe\":false,\"underlined_colored_links\":false,\"enable_efficient_digest_embed\":false,\"enable_aligned_images\":false,\"max_image_upload_mb\":64,\"threads_suggested_ios_version\":null,\"pledges_disabled\":false,\"threads_minimum_ios_version\":812,\"hide_podcast_email_setup_link\":false,\"subscribe_captcha_behavior\":\"default\",\"publication_ban_sample_rate\":0,\"custom_themes_substack_subscribe_modal\":false,\"ios_post_share_assets_screenshot_trigger\":\"control\",\"opt_in_to_sections_during_subscribe_include_main_pub_newsletter\":false,\"continue_support_cta_in_newsletter_emails\":false,\"bloomberg_syndication_enabled\":false,\"allow_document_freeze\":false,\"test_age_gate_user\":false,\"podcast_main_feed_is_firehose\":false,\"pub_app_incentive_gift\":\"\",\"no_embed_redirect\":false,\"customized_email_from_name_for_new_follow_emails\":\"treatment\",\"spotify_open_access_sandbox_mode\":false,\"disable_custom_nav_menu\":false,\"enable_livestream_name_cards\":false,\"fullstory_enabled\":false,\"chat_reply_poll_interval\":3,\"thefp_enable_email_upsell_banner\":false,\"force_pub_links_to_use_subdomain\":false,\"always_show_cookie_banner\":false,\"hide_media_download_option\":false,\"hide_post_restacks\":false,\"feed_item_source_debug_mode\":false,\"twitter_embed_video_playback_enabled\":false,\"publication_homepage_title_display_override\":\"\",\"enable_email_automation_stats\":false,\"post_preview_highlight_byline\":false,\"4k_video\":false,\"enable_islands_section_intent_screen\":false,\"post_metering_enabled\":false,\"notifications_disabled\":\"\",\"cross_post_notification_threshold\":1000,\"facebook_connect_prod_app\":true,\"force_into_pymk_ranking\":false,\"minimum_android_version\":756,\"group_size_change_notification_email_enabled\":false,\"enable_transcription_translations\":false,\"use_og_image_as_twitter_image_for_post_previews\":false,\"always_use_podcast_channel_art_as_episode_art_in_rss\":false,\"seo_tier_override\":\"NONE\",\"editor_role_enabled\":false,\"no_follow_links\":false,\"publisher_api_enabled\":false,\"zendesk_support_priority\":\"default\",\"enable_post_clips_stats\":false,\"enable_subscriber_referrals_awards\":true,\"ios_profile_themes_feed_permalink_enabled\":false,\"reader_survey_during_subscribe_enabled\":true,\"use_publication_language_for_transcription\":false,\"show_substack_funded_gifts_tooltip\":true,\"disable_ai_transcription\":false,\"per_pub_notification_settings\":false,\"live_stream_creation_enabled\":true,\"disable_card_element_in_europe\":false,\"web_growth_item_promotion_threshold\":0,\"bundle_subscribe_enabled\":false,\"web_vitals_sample_rate\":0,\"allow_live_stream_auto_takedown\":\"true\",\"mobile_publication_attachments_enabled\":false,\"enable_design_templates_events_page\":false,\"enable_survey_response_crm_filters\":false,\"disable_personal_substack_initialization\":false,\"section_specific_welcome_pages\":false,\"local_payment_methods\":\"control\",\"publisher_api_cancel_comp\":false,\"posts_in_rss_feed\":20,\"publisher_dashboard_section_selector\":false,\"reader_surveys_platform_question_order\":\"36,1,4,2,3,5,6,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35\",\"login_guard_app_link_in_email\":true,\"community_moderators_enabled\":false,\"enable_custom_theme\":false,\"monthly_sub_is_one_off\":false,\"unread_notes_activity_digest\":\"control\",\"display_cookie_settings\":false,\"welcome_page_query_params\":false,\"enable_free_podcast_urls\":false,\"comp_expiry_emails_disabled\":false,\"enable_description_on_polls\":false,\"use_microlink_for_instagram_embeds\":false,\"free_signup_confirmation_behavior\":\"with_email_validation\",\"ios_post_stats_for_admins\":false,\"enable_design_templates\":false,\"use_livestream_post_media_composition\":true,\"section_specific_preambles\":false,\"pub_export_temp_disable\":false,\"show_menu_on_posts\":false,\"ios_post_subscribe_web_routing\":true,\"enable_pangram_detection_settings\":false,\"freewalls_enabled_web\":false,\"app_onboarding_survey_email\":false,\"republishing_enabled\":false,\"app_mode\":false,\"show_phone_banner\":false,\"minimum_ios_version\":2200,\"enable_author_pages\":false,\"enable_decagon_chat\":true,\"first_month_upsell\":\"control\",\"new_user_checklist_enabled\":\"use_follower_count\",\"disable_high_res_recording\":false,\"latex_upgraded_inline\":false,\"anonymous_post_metering_enabled\":false,\"rss_verification_code\":\"\",\"notification_post_emails\":\"experiment\",\"chat_suppress_contributor_push_option_enabled\":false,\"enable_subscriber_tags\":true,\"media_feed_app_prepend_inbox_limit\":0,\"export_hooks_enabled\":false,\"audio_encoding_bitrate\":null,\"extra_seats_coupon_type\":false,\"post_subdomain_universal_links\":false,\"post_import_max_file_size\":26214400,\"feed_promoted_video_publication\":false,\"livekit_reconnect_slate_url\":\"https://mux-livestream-assets.s3.us-east-1.amazonaws.com/custom-disconnect-slate-tall.png\",\"exclude_from_pymk_suggestions\":false,\"publication_ranking_variant\":\"experiment\",\"disable_annual_subscriptions\":false,\"hack_jane_manchun_wong\":true,\"android_enable_auto_gain_control\":true,\"allow_coupons_on_upgrade\":false,\"test_au_age_gate_user\":false,\"pub_auto_moderation_enabled\":false,\"disable_live_stream_ai_trimming_by_default\":false,\"disable_deletion\":false,\"livekit_egress_custom_base_url\":\"http://livekit-egress-custom-recorder.s3-website-us-east-1.amazonaws.com\",\"clip_focused_video_upload_flow\":false,\"live_stream_max_guest_users\":2,\"android_upgrade_alert_dialog_reincarnated\":true,\"enable_video_seo_data\":false,\"can_reimport_unsubscribed_users_with_2x_optin\":false,\"founding_upgrade_during_gift_disabled\":false,\"thefp_enable_section_groups\":false,\"review_incoming_email\":\"default\",\"enable_founding_gifts\":false,\"enable_sponsorship_campaigns\":false,\"thefp_forum_polls_enabled\":false,\"thefp_enable_embed_media_links\":false,\"sort_modal_search_results\":false,\"default_thumbnail_time\":10,\"load_test_unichat\":false,\"live_stream_head_alignment_guide\":false,\"free_press_combo_subscribe_flow_enabled\":false,\"enable_code_embed_toc\":false,\"enable_publication_tax_settings\":false,\"gifts_from_substack_feature_available\":true,\"disable_ai_clips\":false,\"enable_elevenlabs_voiceovers\":false,\"thefp_enable_transcripts\":false,\"show_simple_post_editor\":false,\"instacart_integration_enabled\":false,\"enable_publication_podcasts_page\":false,\"enable_direct_message_request_bypass\":false,\"enable_apple_news_sync\":false,\"live_stream_in_trending_topic_overrides\":\"\",\"free_press_newsletter_promo_enabled\":false,\"account_based_post_metering_enabled\":false,\"disable_live_stream_reactions\":false,\"instacart_partner_id\":\"\",\"enable_adhoc_email_scheduling\":true,\"media_feed_prepend_inbox_limit\":35,\"welcome_page_no_opt_out\":false,\"enable_caption_translation_tools\":false,\"section_specific_postscripts\":false,\"edit_profile_theme_colors\":false,\"disable_clipping_for_readers\":false,\"enable_apple_podcast_delivery_self_serve_pem_files\":false,\"enable_press_kit_preview_modal\":false,\"enable_rnc_page\":true,\"live_stream_video_degradation_preference\":\"maintainFramerate\",\"enable_high_follower_dm\":true,\"pause_app_badges\":false,\"profile_feed_expanded_inventory\":false,\"livekit_mux_latency_mode\":\"low\",\"feed_juiced_user\":0,\"global_iap_plan_setup_countries\":\"AE,AG,AL,AM,AT,AU,BA,BG,BH,BJ,BN,BS,BT,BW,CH,CI,CR,CY,CZ,DK,DZ,EC,EE,ET,FI,GM,GT,GY,HK,HR,HU,ID,IL,IN,IS,JM,JO,KE,KW,LC,LI,LK,LT,LU,LV,MA,MD,MG,MK,MN,MT,MU,MX,MY,MZ,NA,NO,NZ,OM,PA,PE,PH,PK,QA,RO,RS,RW,SE,SG,SI,SK,SN,SV,TH,TN,TR,TT,TW,TZ,UZ,VN,ZA\",\"enable_polymarket_expandable_embeds\":true,\"publisher_dashboard_group_subs_sidebar\":false,\"use_advanced_commerce_api_for_iap\":false,\"skip_free_preview_language_in_podcast_notes\":false,\"larger_wordmark_on_publication_homepage\":false,\"enable_mobile_stats_for_admins\":false,\"enable_livestream_youtube_simulcast\":false,\"enable_persona_sandbox_environment\":false,\"allowed_email_domains\":\"mozmail.com\",\"create_nav_item_from_tag\":false,\"bypass_single_unlock_token_limit\":false,\"enable_code_embeds\":false,\"add_section_and_tag_metadata\":false,\"daily_promoted_notes_enabled\":true,\"enable_segments_ui_revamp\":true,\"enable_islands_cms\":false,\"enable_livestream_combined_stats\":false,\"redirect_to_pub_after_welcome_signup\":false,\"chartbeat_video_enabled\":false,\"enable_drip_campaigns\":false,\"enable_filtered_posts_on_homepage\":false,\"post_management_search_engine\":\"elasticsearch\",\"new_bestseller_leaderboard_feed_item_enabled\":false,\"feed_main_disabled\":false,\"enable_account_settings_revamp\":false,\"thefp_enable_fp_recirc_block\":false,\"top_search_variant\":\"control\",\"enable_debug_logs_ios\":false,\"show_pub_content_on_profile_for_pub_id\":0,\"show_pub_content_on_profile\":false,\"livekit_track_egress\":true,\"onboarding_suggestions_search\":\"experiment\",\"feed_tuner_enabled\":false,\"livekit_mux_latency_mode_rtmp\":\"low\",\"google_login_enabled\":false,\"livekit_high_quality_egress\":false,\"iap_announcement_blog_url\":\"\",\"disable_code_embed_chrome\":false,\"ios_livestream_feedback\":false,\"founding_plan_upgrade_warning\":false,\"ios_mediaplayer_reply_bar_v2\":false,\"enable_notification_email_batching\":true,\"note_author_poll_results_visible_by_default\":true,\"ios_hide_portal_tab_bar\":false,\"follow_upsell_rollout_percentage\":30,\"gift_article_enabled\":false,\"live_stream_invite_ttl_seconds\":900000,\"include_founding_plans_coupon_option\":false,\"thefp_enable_cancellation_discount_offer\":false,\"android_enable_edit_profile_theme\":false,\"disable_audio_enhancement\":false,\"pub_search_variant\":\"control\",\"ignore_video_in_notes_length_limit\":false,\"substack_gtm_enabled\":false,\"allow_long_videos\":true,\"enable_automation_filters\":false,\"use_enhanced_video_embed_player\":true,\"founding_upgrade_preview\":false,\"enable_viewing_all_livestream_viewers\":false,\"send_subscription_canceled_email\":false,\"enable_clip_prompt_variant_filtering\":true,\"chartbeat_enabled\":false,\"dpn_ranking_enabled\":true,\"enable_custom_email_css\":false,\"thefp_enable_shows_page\":false,\"enable_apple_podcast_auto_publish\":false,\"linkedin_profile_search_enabled\":false,\"publication_has_own_app\":false,\"suggested_minimum_ios_version\":0,\"thebulwark_enable_footer_actions\":false,\"enable_posts_to_segments\":true,\"trending_topics_module_long_term_experiment\":\"control\",\"phone_verification_fallback_to_twilio\":false,\"enable_suggested_searches\":true,\"enable_subscription_notification_email_batching\":true,\"feed_controls\":\"control\",\"thefp_enable_plans_modal_v2\":false,\"a24_redemption_link\":\"\",\"design_templates_launch_phase\":\"off\",\"post_custom_email_from_name_enabled\":false,\"ios_live_stream_auto_gain_enabled\":true,\"enable_publication_tts_player\":false,\"enable_group_direct_messages\":true,\"enable_notes_admins\":false,\"thefp_show_pub_app_callout_on_post\":false,\"search_ranker_variant\":\"experiment\",\"forced_featured_topic_id\":\"\",\"ios_audio_captions_disabled\":false,\"web_podcasts_tab\":false,\"related_posts_enabled\":false,\"thefp_enable_homepage_secondary_nav\":false,\"ios_live_stream_pip_dismiss_v4\":\"control\",\"reply_rate_limit_max_distinct_users_daily\":110,\"web_post_metering_enabled\":true,\"search_retrieval_variant\":\"experiment\",\"mobile_user_attachments_enabled\":false,\"web_profile_publication_recommendations_carousel\":true,\"publisher_banner\":\"\",\"enable_sponsorship_profile\":false,\"gift_article_button_enabled\":false,\"reply_rate_limit_max_distinct_users_monthly\":600,\"desktop_live_stream_screen_share_audio_enabled\":false,\"creator_match\":true,\"ai_image_generation_enabled\":true,\"ios_default_coupon_enabled\":true,\"automod_admin_hidden_comments_tab\":true,\"video_clip_suggestions_enabled\":false,\"publisher_dashboard_home_layout_experiment\":\"control\",\"android_polymarket_embed_search\":false,\"audience_section_editor_help_link\":\"https://support.substack.com/hc/en-us/articles/50558240901268-How-do-I-add-audience-specific-content-to-a-post-on-Substack\",\"portal_ranking_variant\":\"experiment\",\"permalink_reply_ranking_variant\":\"experiment\",\"allow_feed_category_filtering\":false,\"enable_dynamic_content\":true,\"enable_thumbnail_editor_media_settings_integration\":false,\"subscriber_profile_survey_responses_enabled\":true,\"private_live_streaming_enabled\":true,\"desktop_live_stream_safe_framing\":0.8,\"translate_posts_on_standalone_sites\":true,\"bundle_subscribe_default_to_regular_plan\":false,\"enable_standardized_sponsorship_formats\":false,\"enable_pencraft_signin_form\":true,\"thefp_enable_secondary_section_redesign\":false,\"bignight_page_enabled\":true,\"thefp_paywall_annual_plans_experiment\":\"control\",\"enable_launch_assistant\":false,\"enable_youtube_exploration\":true,\"publisher_dashboard_nav_refresh\":false,\"thefp_enable_bundle_post_follow\":false,\"web_notes_tiptap_v3_experiment\":\"treatment_tiptap_v3\",\"recipe_rich_text_enabled\":true,\"publisher_dashboard_nav_refresh_experiment\":\"treatment\",\"chat_channel_creation_enabled\":false,\"web_onboarding_content_type_preferences\":\"experiment\",\"web_mobile_asset_handoff\":false},\"trackingTokens\":{\"fbPixelId\":null,\"gaPixelId\":null,\"twitterPixelId\":null,\"parselyPixelId\":null,\"googleTagManagerToken\":null,\"googleAnalytics4Token\":null,\"twitterPixelSignupEventId\":null,\"twitterPixelSubscribeEventId\":null},\"publicationSettings\":{\"block_ai_crawlers\":false,\"credit_token_enabled\":false,\"custom_tos_and_privacy\":false,\"did_identity\":null,\"disable_optimistic_bank_payments\":false,\"display_welcome_page_details\":true,\"payment_pledges_enabled\":false,\"enable_drop_caps\":false,\"enable_post_page_conversion\":true,\"enable_prev_next_nav\":false,\"enable_restacking\":true,\"founding_group_subscriptions_enabled\":false,\"gifts_from_substack_disabled\":false,\"google_analytics_4_token\":null,\"group_sections_and_podcasts_in_menu_enabled\":false,\"live_stream_homepage_visibility\":\"contributorsAndAdmins\",\"live_stream_homepage_style\":\"autoPlay\",\"live_stream_replay_enabled\":true,\"medium_length_description\":\"\",\"notes_feed_enabled\":false,\"paywall_unlock_tokens\":false,\"post_preview_crop_gravity\":\"auto\",\"post_preview_radius\":\"xs\",\"reader_referrals_enabled\":false,\"reader_referrals_leaderboard_enabled\":false,\"seen_coming_soon_explainer\":false,\"seen_google_analytics_migration_modal\":false,\"local_currency_modal_seen\":false,\"local_payment_methods_modal_seen\":false,\"twitter_pixel_signup_event_id\":null,\"twitter_pixel_subscribe_event_id\":null,\"use_inclusive_sales_tax\":false,\"use_local_currency\":false,\"welcome_page_opt_out_text\":\"No thanks\",\"cookie_settings\":\"\",\"show_restacks_below_posts\":true,\"holiday_gifting_post_header\":false,\"homepage_message_text\":\"\",\"homepage_message_link\":\"\",\"about_us_author_ids\":\"\",\"archived_section_ids\":\"\",\"column_section_ids\":\"\",\"fp_primary_column_section_ids\":\"\",\"event_section_ids\":\"\",\"podcasts_metadata\":\"\",\"video_section_ids\":\"\",\"post_metering_enabled\":false,\"account_based_post_metering_enabled\":false,\"use_custom_theme\":false,\"reply_rules\":null,\"automatic_moderation_enabled\":true,\"auto_translate_enabled\":true,\"additional_post_languages_enabled\":false,\"high_res_recording_beta\":false},\"publicationUserSettings\":null,\"userSettings\":{\"user_id\":null,\"activity_likes_enabled\":true,\"dashboard_nav_refresh_enabled\":false,\"is_guest_post_enabled\":true,\"invite_friends_nux_dismissed_at\":null,\"suggestions_feed_item_last_shown_at\":null,\"last_notification_alert_shown_at\":null,\"disable_reply_hiding\":false,\"newest_seen_chat_item_published_at\":null,\"explicitContentEnabled\":false,\"contactMatchingEnabled\":false,\"messageRequestLevel\":\"everyone\",\"liveStreamAcceptableInviteLevel\":\"everyone\",\"liveStreamAcceptableChatLevel\":\"everyone\",\"creditTokensTreatmentExposed\":false,\"appBadgeIncludesChat\":false,\"autoPlayVideo\":true,\"autoAdvanceVideo\":true,\"smart_delivery_enabled\":false,\"chatbotTermsLastAcceptedAt\":null,\"has_seen_notes_post_app_upsell\":false,\"first_note_id\":null,\"show_concurrent_live_stream_viewers\":false,\"edit_profile_feed_item_dismissed_at\":null,\"mobile_permalink_app_upsell_seen_at\":null,\"new_user_checklist_enabled\":false,\"has_seen_youtube_shorts_auto_publish_announcement\":false,\"has_seen_publish_youtube_connect_upsell\":false,\"notificationQualityFilterEnabled\":true,\"hasSeenOnboardingNewslettersScreen\":false,\"bestsellerBadgeEnabled\":true,\"hasSelfIdentifiedAsCreator\":false,\"autoTranslateEnabled\":true,\"autoTranslateBlocklist\":[],\"fineTuneFeedItemSeenAt\":null,\"fineTuneFeedItemDismissedAt\":null,\"aiGeneratedTextMode\":\"show\",\"feedVideoPreference\":5,\"feedAudioPreference\":5,\"feedLongerPostsPreference\":5,\"feedChatPreference\":5},\"mux_env_key\":\"u42pci814i6011qg3segrcpp9\",\"persona_environment_id\":\"env_o1Lbk4JhpY4PmvNkwaBdYwe5Fzkt\",\"sentry_environment\":\"production\",\"launchWelcomePage\":false,\"pendingInviteForActiveLiveStream\":null,\"isEligibleForLiveStreamCreation\":true,\"webviewPlatform\":null,\"noIndex\":false,\"post\":{\"audience\":\"everyone\",\"audience_before_archived\":null,\"canonical_url\":\"https://charity.wtf/p/aws-networking-environments-and-you\",\"default_comment_sort\":null,\"editor_v2\":false,\"exempt_from_archive_paywall\":false,\"free_unlock_required\":false,\"id\":181561603,\"podcast_art_url\":null,\"podcast_duration\":null,\"podcast_preview_upload_id\":null,\"podcast_upload_id\":null,\"podcast_url\":null,\"post_date\":\"2016-03-23T18:20:31.000Z\",\"updated_at\":\"2025-12-16T01:00:58.582Z\",\"publication_id\":2935724,\"search_engine_description\":null,\"search_engine_title\":null,\"section_id\":null,\"should_send_free_preview\":false,\"show_guest_bios\":true,\"slug\":\"aws-networking-environments-and-you\",\"social_title\":null,\"subtitle\":\"\",\"teaser_post_eligible\":true,\"title\":\"AWS Networking, Environments and You\",\"type\":\"newsletter\",\"video_upload_id\":null,\"write_comment_permissions\":\"everyone\",\"meter_type\":\"none\",\"live_stream_id\":null,\"is_published\":true,\"restacks\":0,\"reactions\":{\"\u2764\":0},\"top_exclusions\":[],\"pins\":[],\"section_pins\":[],\"has_shareable_clips\":false,\"previous_post_slug\":\"two-weeks-with-terraform\",\"next_post_slug\":\"terraform-vpc-and-why-you-want-a-tfstate-file-per-env\",\"cover_image\":null,\"cover_image_is_square\":false,\"cover_image_is_explicit\":false,\"videoUpload\":null,\"podcastFields\":null,\"podcastUpload\":null,\"podcastPreviewUpload\":null,\"voiceover_upload_id\":null,\"voiceoverUpload\":null,\"has_voiceover\":false,\"description\":\"Last week we hit an important milestone in the life of our baby startup: a functional production environment, with real data flowing from ingestion to storage to serving queries out!\",\"body_html\":\"<p>Last week we hit an important milestone in the life of our baby <a href=\\\"http://twitter.com/houndexplorer\\\">startup</a>: a functional production environment, with real data flowing from ingestion to storage to serving queries out! (Of course everything promptly exploded , but whatever.)</p><p>This got me started thinking about how to safely stage and promote terraform changes, in order to isolate the blast radius of certain destructive changes. I tweeted some random thing about it,</p><blockquote><p>hey, anyone have a strong argument for one VPC per env {staging,prod} vs single VPC with subnets for envs, managing isolation w SGs/routes?</p><p>\u2014 Charity Majors (@mipsytipsy) <a href=\\\"https://twitter.com/mipsytipsy/status/710224093027524608?ref_src=twsrc%5Etfw\\\">March 16, 2016</a></p></blockquote><p>\u2026 and two surprisingly fun things happened. I learned some new things about AWS that I didn\u2019t know (after YEARS of using it). And hilariously, I learned that I and lots of other people still believe things about AWS that <strong>aren\u2019t actually true </strong>anymore.</p><p>That\u2019s why I decided to write this. I\u2019m not gonna get super religious on you, but I want to recap the current set of common and/or best practices for managing environments, network segmentation, and security automation in AWS, and I also want to remind you that this post will be out of date as soon as I publish it, because things are changing pretty crazy fast.</p><h2>Multiple Environments: Why I Should Care?</h2><p>An environment exists to provide resource isolation. You might want this for security reasons, or testability, or performance, or just so you can give your developers a place to fuck around and not hurt anything.</p><p>Maybe you run a bunch of similar production environments so you can give your customers security guarantees or avoid messy performance cotenancy stuff. Then you need a template for stamping out lots of these environments, and you need to be *really* sure they can\u2019t leak into each other.</p><p>Or maybe you are more concerned about the vast oceans of things you need to care about beyond whatever unit tests or functional tests are running on your laptop. Like: capacity planning, load testing, validating storage changes against production workloads, exercising failovers, etc. Any scary changes you have, you need a production-like env to practice in.</p><p>Bottom line: If<strong> you can\u2019t spin up a full copy of your infra and test it, you don\u2019t actually have \u201Cinfrastructure as code\u201D.</strong> You just have \u2026 some code, and duct tape.</p><p><a href=\\\"https://twitter.com/phinze/status/710227204349624321\\\">https://twitter.com/phinze/status/710227204349624321</a></p><p>The basics are simple:</p><ul><li><p>Non-production environments must be walled off from production as strongly as possible. You should NEVER be able to accidentally to connect to a prod db from staging (or from one prod env to another).</p></li><li><p>Production and non-production environments (or all other prod envs) should share as much of the same tooling and code paths as possible. Like, some amount asymptotically approaching 100%. Any gaps there will inevitably, eventually bite you in the ass.</p></li></ul><h2>Managing Multiple Environments in AWS</h2><p>There are baaaasically three patterns that people use to manage multiple environments in AWS these days:</p><ol><li><p>One AWS billing account and one flat network (VPC or Classic), with isolation performed by routes or security groups.</p></li><li><p>Many AWS accounts with consolidated billing. Each environment is a separate account (often maps to one acct per customer).</p></li><li><p>One AWS billing account and many VPCs, where each environment ~= its own VPC.</p></li></ol><p>Let\u2019s start old school with a flat network.</p><h3>1: One Account, One Flattish Network</h3><p>This is what basically everyone did before VPC. (And ummm let\u2019s be honest, lots of us kept it up for a while because GOD networking is such a pain.)</p><p>In EC2 Classic the best you got was security groups. And \u2014 unlike VPC security groups \u2014 you couldn\u2019t stack them, or change the security group of an instance type without destroying it, and there was a crazy low hard cap on the (# of secgroup rules * # of secgroups). You could kind of gently \u201Csuggest\u201D environments with things like DNS subdomains and config management, and sometimes you would see people literally just roll over and resort to $ENV variables.</p><p>Most people either a) gave up and just had a flat network, or b) this happened.</p><blockquote><p><a href=\\\"https://twitter.com/mipsytipsy?ref_src=twsrc%5Etfw\\\">@mipsytipsy</a> we use a single vpc and more sgs than instances, it's a tangled legacy mess. I'd rather go for vpc per env/team whatever</p><p>\u2014 Jan B. (@bracki) <a href=\\\"https://twitter.com/bracki/status/710230597977956353?ref_src=twsrc%5Etfw\\\">March 16, 2016</a></p></blockquote><p>At Parse we did a bunch of complicated security groups plus chef environments that let us spin up staging clusters and the occasional horrible silo\u2019d production stack for exceptional customer requirements. Awful.</p><p><em>VPC has made this better.</em> Even if you\u2019re still using a flat network model. You can now manage your route tables, stack security groups and IAM rules and reapply to existing nodes without destroying the node or dropping your internet connections. You can define private network subnets with NAT gateways, etc.</p><p><a href=\\\"https://twitter.com/sudarkoff/status/710264656367996928\\\">https://twitter.com/sudarkoff/status/710264656367996928</a></p><p>Some people tell me they are using a single VPC with network ACLs to separate environments for ease of use, because you can reuse security groups across environments. Honestly this seems a bit more like a bug than a feature to me, because a) you give up isolation and b) I don\u2019t see how that helps you have a versioned, tested stack.</p><p><a href=\\\"https://twitter.com/tomtheguvnor/status/710226892633333762\\\">https://twitter.com/tomtheguvnor/status/710226892633333762</a></p><p>Ok moving on to the opposite of the spectrum, the crazy kids who are doing tens (or TENS OF THOUSANDS) of linked accounts.</p><h3>2: One AWS Account Per Environment</h3><p>A surprising number of people adopted this model in the bad old EC2 Classic days, not because they necessarily wanted it but because they need a stronger security model and looser resource caps. This is why AWS released <a href=\\\"http://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/consolidated-billing.html\\\">Consolidated Billing</a> way back in 2010.</p><p>I actually learned a lot this week about the multi-account model! Like that you can create <a href=\\\"http://docs.aws.amazon.com/IAM/latest/UserGuide/tutorial-cross-account-with-roles.html\\\">IAM roles that span accounts</a>, or that you can <a href=\\\"http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/sharingamis-explicit.html\\\">share AMIs between accounts.</a>. This model is complicated, but there are some real benefits. Like real, actual, hardcore security/perf isolation. And you will run into fewer resource limits than jamming everything into a single account, and revoking/managing IAM creds is clearer.</p><p>Security nerds love this model, but it\u2019s not clear that \u2026. literally anyone else does.</p><blockquote><p><a href=\\\"https://twitter.com/mipsytipsy?ref_src=twsrc%5Etfw\\\">@mipsytipsy</a> i did one full AWS account per ENV as a way to fully segment (using consolidated billing to manage). felt \\\"safer\\\"</p><p>\u2014 petecheslock@hachyderm.io (@petecheslock) <a href=\\\"https://twitter.com/petecheslock/status/710277140764295168?ref_src=twsrc%5Etfw\\\">March 17, 2016</a></p></blockquote><p>Some things that make me despise it without even trying it:</p><ul><li><p>AWS billing is ALREADY a horrendous hairball coughed up by the world\u2019s ugliest cat, I can\u2019t even imagine trying to wrangle multiple accounts.</p></li><li><p>It\u2019s more expensive, you incur extra billing costs.</p></li><li><p>Having to explicitly list any resource you want to share between any accounts just makes me want to tear my hair out strand by strand while screaming on a street corner</p></li><li><p>Account creation API still has manual steps, like getting certs/keypairs.</p></li><li><p>You cannot make bulk changes to accounts, and AWS doesn\u2019t like you having thousands of linked accounts. Also limits your flexibility with Reserved Instances.</p></li></ul><p>Here is a pretty reasonable <a href=\\\"https://segment.com/blog/rebuilding-our-infrastructure/\\\">blog p0st</a> laying out some of the benefits though, and as you can see, there are plenty of other crazy people who like it. Mostly security nerds.</p><h3>3: One AWS Account, One VPC Per Environment</h3><p>I have saved the best for last. I think this is the best model, and the one I am adopting. You spin up a production VPC, a staging VPC, dev VPC, Travis-CI VPC. EVERYBODY GETS A VPC!#@!</p><p>One of those things that everybody seems to \u201Cknow\u201D but isn\u2019t true is that you can\u2019t have lots of VPCs. Yes, it is capped at 5 by default, and many people have stories about how they couldn\u2019t get it raised, and that used to be true. But <strong>the hard cap is now 200</strong>, not 10, so VPC awayyyyyyyy my pretties!</p><blockquote><p><a href=\\\"https://twitter.com/mipsytipsy?ref_src=twsrc%5Etfw\\\">@mipsytipsy</a> 1 account per env, many VPCs. IAM/VPC peer makes shit easy... we run many accounts, 200+ VPCs in our prod acc</p><p>\u2014 Ben Bromhead (@BenBromhead) <a href=\\\"https://twitter.com/BenBromhead/status/710239464530923520?ref_src=twsrc%5Etfw\\\">March 16, 2016</a></p></blockquote><p>Here\u2019s another reason to love VPC <-> env mapping: orchestration is finally coming around to the party. Even recently people were still trying to make chef-metal a thing, or developing their own coordination software from scratch with Boto, or just using the console and diffing and committing to git.</p><p>Dude, <em>stop. </em>We are past the point where you should default to using Terraform or CloudFormation for the bones of your infrastructure, for the things that rarely change. And once you\u2019ve done that you\u2019re most of the way to a reusable, testable stack.</p><p>Most of the cotenancy problems that account-per-env solved are a lot less compelling to me now that VPCs exist.</p><blockquote><p><a href=\\\"https://twitter.com/mipsytipsy?ref_src=twsrc%5Etfw\\\">@mipsytipsy</a> if you're doing automation, sep VPC/accounts let you fully test that automation rather than \\\"just\\\" app code</p><p>\u2014 Adam (@uptill_3) <a href=\\\"https://twitter.com/uptill_3/status/710397938065866752?ref_src=twsrc%5Etfw\\\">March 17, 2016</a></p></blockquote><p>VPCs are here to help you think about infrastructure like regular old code. Lots of VPCs are approximately as easy to manage as as one VPC. Unlike lots of accounts, which are there to give you headaches and one-offs and custom scripts and pointy-clicky shit and complicated horrible things to work around.</p><blockquote><p><a href=\\\"https://twitter.com/mipsytipsy?ref_src=twsrc%5Etfw\\\">@mipsytipsy</a> We just have CloudFormation templates that take IP prefix and VPC ID as arguments; seems to have worked pretty well.</p><p>\u2014 Alex Rasmussen (@alexras) <a href=\\\"https://twitter.com/alexras/status/710230250504912896?ref_src=twsrc%5Etfw\\\">March 16, 2016</a></p></blockquote><p>VPCs have some caveats of their own. Like, you can only assign a /16 to any VPC. If you\u2019re using 4 availability zones and public subnets + natted private subnets, that\u2019s only ~8k IPs per subnet/AZ pair. Shrug.</p><p><a href=\\\"https://aws.amazon.com/about-aws/whats-new/2016/03/announcing-support-for-security-group-references-in-a-peered-vpc/\\\">You can peer security groups across different VPCs</a>, but not across regions (yet). Also, if you\u2019re a terraform user, be aware that it handles VPC peering fine but doesn\u2019t handle multiple accounts very well.</p><p>Lots of people seem to have had issues with security group limits per VPC, even though the limit is 500 and says it can be raised by request. I\u2019m \u2026. not sure what to think of that. I *feel* like if you\u2019re building a thing with > 500 sec group rules on a single VPC, you\u2019re probably doing something wrong.</p><h2>Test my code you piece of shit I dare you</h2><p>Here\u2019s the thing that got me excited about this from the start though, which is having the ability to do things like test terraform modules on a staging VPC from a branch before promoting the clean changes to master. If you plan on doing things like running bleeding-edge software in production *cough* you need allllll the guard rails and test coverage you can possibly scare up. <strong>VPCs help you get this</strong> in a massive way.</p><p>Super quick example, say you\u2019re using adding a NAT gateway to your staging cluster, you would use the remote git source with your changes:</p><p>[code language=\u201Djavascript\u201D]<br>// staging<br>module \\\"aws_vpc\\\" {<br>source = \\\"git::ssh://git@github.com/houndsh/infra.git//terraform/modules/aws_vpc?ref=charity.adding-nat-gw\\\"<br>env = ${var.env}<br>\u2026<br>}<br>[/code]</p><p>And then once you\u2019ve validated the change, you simply merge your branch to master and run terraform plan/apply to production.</p><p>[code language=\u201Djavascript\u201D]<br>// production<br>module \\\"aws_vpc\\\" {<br>source = \\\"github.com/houndsh/infra/terraform/modules/aws_vpc\\\"<br>env = \\\"${var.env}\\\"<br>\u2026<br>}<br>[/code]</p><p>And for GOD\u2019S SAKE USE DIFFERENT STATE FILES FOR EACH VPC / ENVIRONMENT okayyyyyy but that is a different rant, not an AWS rant, so let\u2019s move along.</p><h2>In Conclusion</h2><p>There are legit reasons to use all three of these models, and infinite variations upon them, and your use case is not my use case, blah blah.</p><p>But moving from one VPC to multiple VPCs is really not a big deal. I know a lot of us bear scars, but it is <em>nothing</em> like the horror that was trying to move from EC2 Classic to VPC.</p><p><a href=\\\"https://twitter.com/mistofvongola/status/710239882912747524\\\">https://twitter.com/mistofvongola/status/710239882912747524</a></p><p>VPC has a steeper learning curve than Classic, but it is sooooo worth it. Every day I\u2019m rollin up on some new toy you get with VPC (ICMP for ELBs! composable IAM host roles! new VPC NAT gateway!!!). The rate at which they\u2019re releasing new shit for VPC is staggering, I can barely keep up.</p><blockquote><p><a href=\\\"https://twitter.com/mipsytipsy?ref_src=twsrc%5Etfw\\\">@mipsytipsy</a> we have multiple aws accounts with one vpc per. Private and public subnets in that vpc. Most non ELB aws resources in private</p><p>\u2014 Alfonso (@alfonso__c) <a href=\\\"https://twitter.com/alfonso__c/status/710232944162504704?ref_src=twsrc%5Etfw\\\">March 16, 2016</a></p></blockquote><p>Alright, said I wasn\u2019t gonna get religious but I totally lied.</p><p><strong>VPC is the future and it is awesome, and unless you have some VERY SPECIFIC AND CONVINCING reasons to do otherwise, you should be spinning up a VPC per environment with orchestration and prob doing it from CI on every code commit, almost like it\u2019s just like, you know, code</strong>.</p><p>That you need to test.</p><p>Cause it is.</p><p><em>(thank you to everybody who chatted with me and taught me things and gave awesome feedback!! knuckle tatts credit to @mosheroperandi)</em></p>\",\"has_dynamic_content\":false,\"truncated_body_text\":\"Last week we hit an important milestone in the life of our baby startup: a functional production environment, with real data flowing from ingestion to storage to serving queries out! (Of course everything promptly exploded , but whatever.)\",\"wordcount\":2115,\"post_preview_limit\":null,\"postTags\":[],\"postCountryBlocks\":[],\"headlineTest\":null,\"publishedBylines\":[{\"id\":32306597,\"name\":\"Charity Majors\",\"handle\":\"charitydotwtf\",\"photo_url\":\"https://substackcdn.com/image/fetch/$s_!EAp-!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff7a54851-0549-41da-b041-3cfc959ec0ba_3088x2316.jpeg\",\"bio\":\"cofounder and CTO of honeycomb.io; pioneered modern observability. co-author of O'Reilly books \\\"Database Reliability Engineering\\\" and \\\"Observability Engineering\\\", now wrapping up the 2nd ed. loves free software, free speech, and peaty single malts.\",\"profile_set_up_at\":\"2022-11-24T22:50:45.155Z\",\"reader_installed_at\":\"2023-08-30T02:09:51.674Z\",\"publicationUsers\":[{\"id\":2985119,\"user_id\":32306597,\"publication_id\":2935724,\"role\":\"admin\",\"public\":true,\"is_primary\":true,\"publication\":{\"id\":2935724,\"name\":\"charity.wtf\",\"subdomain\":\"charitydotwtf\",\"custom_domain\":\"charity.wtf\",\"custom_domain_optional\":false,\"hero_text\":\"observability, tech advice, honeycomb.io, etc\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/194fc089-9e91-4a15-85cc-5da3fca53eed_1280x1280.png\",\"author_id\":32306597,\"primary_user_id\":32306597,\"theme_var_background_pop\":\"#FF6719\",\"created_at\":\"2024-08-26T05:48:54.313Z\",\"email_from_name\":\"Charity Majors\",\"copyright\":\"Charity Majors\",\"founding_plan_name\":null,\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"disabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"profile\",\"is_personal_mode\":false,\"logo_url_wide\":null}}],\"is_guest\":false,\"bestseller_tier\":null,\"status\":{\"bestsellerTier\":null,\"subscriberTier\":5,\"leaderboard\":null,\"vip\":false,\"badge\":{\"type\":\"subscriber\",\"tier\":5,\"accent_colors\":null},\"subscriber\":null}}],\"reaction\":null,\"reaction_count\":0,\"comment_count\":0,\"child_comment_count\":0,\"is_geoblocked\":false,\"hasCashtag\":false},\"comments\":[],\"canonicalUrl\":\"https://charity.wtf/p/aws-networking-environments-and-you\",\"inlineComments\":false,\"readerIsSearchCrawler\":false,\"ogUrl\":\"https://charity.wtf/p/aws-networking-environments-and-you\",\"bannedFromNotes\":false,\"themeVariables\":{\"color_theme_bg_pop\":null,\"background_pop\":\"#ff6719\",\"color_theme_bg_web\":null,\"cover_bg_color\":\"#FFFFFF\",\"cover_bg_color_secondary\":\"#f0f0f0\",\"background_pop_darken\":\"#ff5600\",\"print_on_pop\":\"#ffffff\",\"print_pop\":\"#ff6719\",\"color_theme_accent\":null,\"cover_print_primary\":\"#363737\",\"cover_print_secondary\":\"#757575\",\"cover_print_tertiary\":\"#b6b6b6\",\"cover_border_color\":\"#ff6719\",\"background_contrast_1\":\"#f0f0f0\",\"background_contrast_2\":\"#dddddd\",\"background_contrast_3\":\"#b7b7b7\",\"background_contrast_4\":\"#929292\",\"background_contrast_5\":\"#515151\",\"color_theme_detail\":\"#e6e6e6\",\"background_contrast_pop\":\"rgba(255, 103, 25, 0.4)\",\"theme_bg_is_dark\":\"0\",\"background_pop_rgb\":\"255, 103, 25\"},\"recentEpisodes\":null,\"trackFrontendVisit\":true,\"activeLiveStream\":null,\"freeTrialCoupon\":null,\"defaultCoupon\":null,\"isChatActive\":false,\"publicationPageAlert\":null,\"features\":{},\"browser\":{\"name\":\"Chrome\",\"version\":\"125.0\",\"major\":\"125\"},\"showCookieBanner\":false,\"disabledCookies\":[],\"dd_env\":\"prod\",\"dd_ti\":true}")</script>
|
||
<script nonce="uxOuplozQoxmNQbU1vPabg">window._analyticsConfig = JSON.parse("{\"properties\":{\"subdomain\":\"charitydotwtf\",\"publication_id\":2935724,\"has_plans\":false,\"pub_community_enabled\":true,\"is_personal_publication\":false,\"is_subscribed\":false,\"is_free_subscribed\":false,\"is_author\":false,\"is_contributor\":false,\"is_admin\":false,\"is_founding\":false},\"adwordsAccountId\":\"AW-316245675\",\"adwordsEventSendTo\":\"Tf76CKqcyL4DEKuN5pYB\"}")</script>
|
||
|
||
|
||
|
||
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/lib-router.d5564214.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/55188.8c7a3da4.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/62357.eb9c2ec5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53754.305166fb.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/33485.d885963a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/2237.c201511d.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37707.89196941.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9351.24a8b3eb.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80821.48106133.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44292.197bbb84.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70291.83407523.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17499.2fdf012b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57975.120ac36b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/19714.8a794144.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48320.2e9250be.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93000.d46d6eb8.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44014.aba48b36.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/43745.b6692f96.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/60892.85436d63.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/40646.9b13ea71.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/87906.dadcd2af.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/36479.40ac7839.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17706.0bbae692.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93977.4cad198a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37606.507d23bb.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94311.3544d4b6.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/22012.53715f4b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/21262.bdf05692.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80220.ee65f827.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/58865.6a185680.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11100.f2ef7ae9.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/3663.01aab681.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18725.87552a3a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11909.58b156c3.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93519.d296e647.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/84828.c140eedb.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/5075.ae3a195a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/54758.658f8e4a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56725.91edd1cf.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/42336.0f4d38fa.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89879.fa47877a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70793.966f7240.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/39522.ed55b69a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53164.ee98f041.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74584.e0c3f012.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71199.c04af3e2.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63266.5548174c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47358.e1246005.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4492.331e4104.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/88089.96bbb910.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/97348.bfd56d5d.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/52109.79f815c3.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56256.7bef5b33.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/77410.fc5e367b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/25058.ccdd552c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56562.dc4f92e5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94881.e2bcff2c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27386.91a0566a.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/10174.85d1a072.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63743.0268e568.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91214.ed37df83.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/75919.15a57364.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48445.e99acabd.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/51271.616b66d2.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/49234.28d3e23c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/50629.2d19b4da.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/88015.62e502f2.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48409.4e94bf26.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/26252.8fc2e1de.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18494.bca04444.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/7192.66cb85ed.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/31249.7a90b7de.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91850.7af3926c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/49047.999e5f80.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/16075.37c0e7f5.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/28856.eb6b6b43.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/95396.afe9e479.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63175.af4fe7dc.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74628.03acd2b1.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11983.9c226605.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27313.92778837.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/mainReactRouter.cdc41f5c.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91507.d02d8c94.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/54770.2a5db23b.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/30054.d8db895d.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78118.5e65876d.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57051.c36d0ac6.js" charset="utf-8"></script>
|
||
|
||
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78191.da318204.js" charset="utf-8"></script>
|
||
|
||
|
||
<script nonce="uxOuplozQoxmNQbU1vPabg" nomodule>
|
||
(function() {
|
||
var message = 'Your browser does not support modern JavaScript modules. Please upgrade your browser for the best experience.';
|
||
var warningDiv = document.createElement('div');
|
||
warningDiv.style.color = 'red';
|
||
warningDiv.style.padding = '10px';
|
||
warningDiv.style.margin = '10px 0';
|
||
warningDiv.style.border = '1px solid red';
|
||
warningDiv.style.backgroundColor = 'lightyellow';
|
||
warningDiv.innerText = message;
|
||
document.body.prepend(warningDiv);
|
||
})();
|
||
</script>
|
||
|
||
|
||
<!-- Datadog Analytics -->
|
||
<script nonce="uxOuplozQoxmNQbU1vPabg">
|
||
(function(h,o,u,n,d) {
|
||
h=h[d]=h[d]||{q:[],onReady:function(c){h.q.push(c)}}
|
||
d=o.createElement(u);d.async=1;d.src=n
|
||
n=o.getElementsByTagName(u)[0];n.parentNode.insertBefore(d,n)
|
||
})(window,document,'script','https://www.datadoghq-browser-agent.com/us1/v5/datadog-rum.js','DD_RUM')
|
||
window.DD_RUM.onReady(function() {
|
||
window.DD_RUM.init({
|
||
clientToken: 'puba71073f072643721169b68f352438710',
|
||
applicationId: '2e321b35-c76b-4073-8d04-cc9a10461793',
|
||
site: 'datadoghq.com',
|
||
service: 'web',
|
||
env: window._preloads.dd_env,
|
||
version: '069dfdb26e4d65848712c36d6d96dc51cbd4f10c',
|
||
sessionSampleRate: 1,
|
||
sessionReplaySampleRate: 10,
|
||
trackUserInteractions: window._preloads.dd_ti,
|
||
trackResources: true,
|
||
trackLongTasks: true,
|
||
defaultPrivacyLevel: 'mask-user-input',
|
||
allowedTracingUrls: [/https?:\/\/(.+\/.)?substack(cdn)?\.com/]
|
||
});
|
||
})
|
||
</script>
|
||
<!-- End Datadog Analytics -->
|
||
|
||
<!-- Cloudflare Web Analytics -->
|
||
<script defer src="https://static.cloudflareinsights.com/beacon.min.js" data-cf-beacon='{"token": "216309cffb464db4b0e02daf0b8e8060"}'></script>
|
||
<!-- End Cloudflare Web Analytics -->
|
||
|
||
|
||
<!-- Fallback tracking pixels -->
|
||
|
||
|
||
|
||
|
||
<noscript>
|
||
<style>
|
||
#nojs-banner {
|
||
position: fixed;
|
||
bottom: 0;
|
||
left: 0;
|
||
padding: 16px 16px 16px 32px;
|
||
width: 100%;
|
||
box-sizing: border-box;
|
||
background: red;
|
||
color: white;
|
||
font-family: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
|
||
font-size: 13px;
|
||
line-height: 13px;
|
||
}
|
||
#nojs-banner a {
|
||
color: inherit;
|
||
text-decoration: underline;
|
||
}
|
||
</style>
|
||
|
||
<div id="nojs-banner">
|
||
This site requires JavaScript to run correctly. Please <a href="https://enable-javascript.com/" target="_blank">turn on JavaScript</a> or unblock scripts
|
||
</div>
|
||
</noscript>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" nonce="uxOuplozQoxmNQbU1vPabg" data-cf-beacon='{"version":"2024.11.0","token":"68cfe66b5c4749e2ba64d4d9640c04c0","spa":2}' crossorigin="anonymous"></script>
|
||
</body>
|
||
</html>
|