Files
nexus/sreweekly/articles/341/08-thoughts-on-api-reliability.html
2026-09-12 17:23:01 +08:00

491 lines
207 KiB
HTML
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en" dir="ltr">
<head>
<meta charset="utf-8" />
<meta name="norton-safeweb-site-verification" content="24usqpep0ejc5w6hod3dulxwciwp0djs6c6ufp96av3t4whuxovj72wfkdjxu82yacb7430qjm8adbd5ezlt4592dq4zrvadcn9j9n-0btgdzpiojfzno16-fnsnu7xd" />
<link rel="preconnect" href="https://substackcdn.com" />
<title data-rh="true">Thoughts on API Reliability - by Utsav Shah</title>
<meta data-rh="true" name="theme-color" content="#fafafa"/><meta data-rh="true" property="og:type" content="article"/><meta data-rh="true" property="og:title" content="Thoughts on API Reliability"/><meta data-rh="true" name="twitter:title" content="Thoughts on API Reliability"/><meta data-rh="true" name="description" content="Building and maintaining a public API is a unique software engineering challenge."/><meta data-rh="true" property="og:description" content="Building and maintaining a public API is a unique software engineering challenge."/><meta data-rh="true" name="twitter:description" content="Building and maintaining a public API is a unique software engineering challenge."/><meta data-rh="true" property="og:image" content="https://substackcdn.com/image/fetch/$s_!MwBD!,w_1200,h_675,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png"/><meta data-rh="true" name="twitter:image" content="https://substackcdn.com/image/fetch/$s_!7xi0!,f_auto,q_auto:best,fl_progressive:steep/https%3A%2F%2Freliability.substack.com%2Fapi%2Fv1%2Fpost_preview%2F68658421%2Ftwitter.jpg%3Fversion%3D4"/><meta data-rh="true" name="twitter:card" content="summary_large_image"/><meta data-rh="true" property="article:modified_time" content="2023-02-12T16:48:27.093Z"/>
<style>
@layer legacy, pencraftReset, pencraft;
</style>
<link rel="preload" as="style" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/11100.cee9fbee.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/86118.c25b9f67.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/93519.6640a5b2.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/42118.cba90b72.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/39522.25128f14.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/47358.668d8a3c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/94881.cb76fe04.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27386.72f00430.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/76768.f15fc605.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/41932.97d2663c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/48409.0588981c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/32552.313415d5.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27313.92c631cb.css" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=0, viewport-fit=cover" />
<meta name="author" content="Utsav Shah" />
<meta property="og:url" content="https://www.softwareatscale.dev/p/thoughts-on-api-reliability" />
<link rel="canonical" href="https://www.softwareatscale.dev/p/thoughts-on-api-reliability" />
<meta name="google-site-verification" content="IORiP7_fax0ESrUYObrVUA0QF8qgq2BzPsISmpFzgfg" />
<link rel="shortcut icon" href="https://substackcdn.com/image/fetch/$s_!HSk1!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Ffavicon.ico">
<link rel="icon" type="image/png" sizes="16x16" href="https://substackcdn.com/image/fetch/$s_!GXS9!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Ffavicon-16x16.png">
<link rel="icon" type="image/png" sizes="32x32" href="https://substackcdn.com/image/fetch/$s_!Wf00!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Ffavicon-32x32.png">
<link rel="apple-touch-icon" sizes="57x57" href="https://substackcdn.com/image/fetch/$s_!brnx!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="https://substackcdn.com/image/fetch/$s_!KKA0!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-60x60.png">
<link rel="apple-touch-icon" sizes="72x72" href="https://substackcdn.com/image/fetch/$s_!DlLP!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-72x72.png">
<link rel="apple-touch-icon" sizes="76x76" href="https://substackcdn.com/image/fetch/$s_!QdaO!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-76x76.png">
<link rel="apple-touch-icon" sizes="114x114" href="https://substackcdn.com/image/fetch/$s_!pc-G!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-114x114.png">
<link rel="apple-touch-icon" sizes="120x120" href="https://substackcdn.com/image/fetch/$s_!ItDq!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-120x120.png">
<link rel="apple-touch-icon" sizes="144x144" href="https://substackcdn.com/image/fetch/$s_!UsOA!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-144x144.png">
<link rel="apple-touch-icon" sizes="152x152" href="https://substackcdn.com/image/fetch/$s_!kiLr!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-152x152.png">
<link rel="apple-touch-icon" sizes="167x167" href="https://substackcdn.com/image/fetch/$s_!nSUQ!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-167x167.png">
<link rel="apple-touch-icon" sizes="180x180" href="https://substackcdn.com/image/fetch/$s_!vQUt!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-180x180.png">
<link rel="apple-touch-icon" sizes="1024x1024" href="https://substackcdn.com/image/fetch/$s_!K39t!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Ff90464ed-4003-4ec0-ae83-b5da286d3063%2Fapple-touch-icon-1024x1024.png">
<link rel="alternate" type="application/rss+xml" href="/feed" title="Software at Scale"/>
<style data-pub-theme-prefix>:root{--color_theme_bg_pop:#1a2e05;--background_pop:#1a2e05;--color_theme_bg_web:#fafafa;--cover_bg_color:#fafafa;--cover_bg_color_secondary:#ebebeb;--background_pop_darken:#0d1703;--print_on_pop:#ffffff;--color_theme_bg_pop_darken:#0d1703;--color_theme_print_on_pop:#ffffff;--color_theme_bg_pop_20:rgba(26, 46, 5, 0.2);--color_theme_bg_pop_30:rgba(26, 46, 5, 0.3);--print_pop:#1a2e05;--color_theme_accent:#1a2e05;--cover_print_primary:#363737;--cover_print_secondary:#757575;--cover_print_tertiary:#b6b6b6;--cover_border_color:#1a2e05;--font_family_headings_preset:'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';--font_weight_headings_preset:900;--font_family_body_preset:'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';--font_weight_body_preset:400;--font_preset_heading:heavy_sans;--font_preset_body:sans;--home_hero:magaziney;--home_posts:list;--web_bg_color:#fafafa;--background_contrast_1:#ebebeb;--background_contrast_2:#d8d8d8;--background_contrast_3:#b3b3b3;--background_contrast_4:#909090;--background_contrast_5:#4f4f4f;--color_theme_bg_contrast_1:#ebebeb;--color_theme_bg_contrast_2:#d8d8d8;--color_theme_bg_contrast_3:#b3b3b3;--color_theme_bg_contrast_4:#909090;--color_theme_bg_contrast_5:#4f4f4f;--color_theme_bg_elevated:#fafafa;--color_theme_bg_elevated_secondary:#ebebeb;--color_theme_bg_elevated_tertiary:#d8d8d8;--color_theme_detail:#e1e1e1;--background_contrast_pop:rgba(26, 46, 5, 0.4);--color_theme_bg_contrast_pop:rgba(26, 46, 5, 0.4);--theme_bg_is_dark:0;--print_on_web_bg_color:#363737;--print_secondary_on_web_bg_color:#848585;--background_pop_rgb:26, 46, 5;--color_theme_bg_pop_rgb:26, 46, 5;--color_theme_accent_rgb:26, 46, 5;}</style>
<link rel="stylesheet" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
<style data-pub-theme-postfix></style>
</head>
<body class="">
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=G-HR35NWP3CE" height="0"
width="0" style="display:none;visibility:hidden"></iframe></noscript>
<div id="entry">
<div id="main" style="--post-preview-border-radius:var(--border-radius-xs);" class="main typography use-theme-bg"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div data-testid="navbar" class="main-menu"><div class="mainMenuContent-DME8DR"><div style="position:relative;height:71px;" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-20 pc-paddingRight-20 pc-justifyContent-space-between pc-alignItems-center pc-reset border-bottom-detail-k1F6C4 topBar-pIF0J1"><div style="flex-basis:0px;flex-grow:1;" class="logoContainer-p12gJb"><a href="/" native class="pencraft pc-display-contents pc-reset"><div draggable="false" class="pencraft pc-display-flex pc-position-relative pc-reset"><div style="width:40px;height:40px;" class="pencraft pc-display-flex pc-reset bg-white-ZBV5av pc-borderRadius-sm overflow-hidden-WdpwT6 sizing-border-box-DggLA4"><picture class="pencraft pc-display-contents pc-reset"><source type="image/webp" sizes="40px" srcset="https://substackcdn.com/image/fetch/$s_!NNuR!,w_40,h_40,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png 40w, https://substackcdn.com/image/fetch/$s_!NNuR!,w_80,h_80,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png 80w, https://substackcdn.com/image/fetch/$s_!NNuR!,w_120,h_120,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png 120w" style="display:none;"/><img style="object-fit:cover;width:40px;height:40px;" src="https://substackcdn.com/image/fetch/$s_!NNuR!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png" srcset="https://substackcdn.com/image/fetch/$s_!NNuR!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png 40w, https://substackcdn.com/image/fetch/$s_!NNuR!,w_80,h_80,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png 80w, https://substackcdn.com/image/fetch/$s_!NNuR!,w_120,h_120,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png 120w" sizes="40px" alt="Software at Scale" draggable="false" class="pencraft pc-reset"/></picture></div></div></a></div><div style="flex-grow:0;" class="titleContainer-DJYq5v"><h1 class="pencraft pc-reset font-pub-headings-FE5byy reset-IxiVJZ title-oOnUGd"><a href="/" class="pencraft pc-display-contents pc-reset">Software at Scale</a></h1></div><div style="flex-basis:0px;flex-grow:1;" class="pencraft pc-display-flex pc-justifyContent-flex-end pc-alignItems-center pc-reset buttonsContainerContainer-ThaN_w"><div class="buttonsContainer-SJBuep"><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-flex-end pc-alignItems-center pc-reset navbar-buttons"><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><span data-state="closed"><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></span><button tabindex="0" type="button" aria-label="Share Publication" id="radix-P0-4" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg></button></div><button tabindex="0" type="button" data-testid="noncontributor-cta-button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_primary-RfbeYt size_md-gCDS3o">Subscribe</button><button tabindex="0" data-native="true" type="button" data-href="https://substack.com/sign-in?redirect=%2Fp%2Fthoughts-on-api-reliability&amp;for_pub=reliability" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_tertiary-rlke8z size_md-gCDS3o">Sign in</button></div></div></div></div></div><div style="height:72px;"></div></div></div><div><script nonce="VqkFSleQStUMFdS2hc29nQ" type="application/ld+json">{"@context":"https://schema.org","@type":"NewsArticle","url":"https://www.softwareatscale.dev/p/thoughts-on-api-reliability","mainEntityOfPage":"https://www.softwareatscale.dev/p/thoughts-on-api-reliability","headline":"Thoughts on API Reliability","description":"Building and maintaining a public API is a unique software engineering challenge.","image":[{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!MwBD!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png"}],"datePublished":"2022-09-30T17:17:52+00:00","dateModified":"2022-09-30T17:17:52+00:00","isAccessibleForFree":true,"author":[{"@type":"Person","name":"Utsav Shah","url":"https://substack.com/@reliability","description":"Software Engineer interested in building simple, self correcting and reliable systems. Twitter: @utsav_sha","identifier":"user:6567678","sameAs":["https://twitter.com/utsav_sha"],"image":{"@type":"ImageObject","contentUrl":"https://substackcdn.com/image/fetch/$s_!Ta5D!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F0f431e7e-fd06-4ab7-a5df-cda6f979e27a_1536x1536.jpeg","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!Ta5D!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F0f431e7e-fd06-4ab7-a5df-cda6f979e27a_1536x1536.jpeg"}}],"publisher":{"@type":"Organization","name":"Software at Scale","url":"https://www.softwareatscale.dev","description":"Discussions on running software systems, teams, and products.","interactionStatistic":{"@type":"InteractionCounter","name":"Subscribers","interactionType":"https://schema.org/SubscribeAction","userInteractionCount":1000},"identifier":"pub:55725","logo":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!NNuR!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!NNuR!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!NNuR!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png"},"image":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!NNuR!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!NNuR!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!NNuR!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png"},"sameAs":["https://twitter.com/utsav_sha"]},"interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":5},{"@type":"InteractionCounter","interactionType":"https://schema.org/ShareAction","userInteractionCount":0},{"@type":"InteractionCounter","interactionType":"https://schema.org/CommentAction","userInteractionCount":0}]}</script><script nonce="VqkFSleQStUMFdS2hc29nQ" type="application/ld+json">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Software at Scale","item":"https://www.softwareatscale.dev"},{"@type":"ListItem","position":2,"name":"Thoughts on API Reliability","item":"https://www.softwareatscale.dev/p/thoughts-on-api-reliability"}]}</script><div aria-label="Post" role="main" class="single-post-container"><div class="container"><div class="single-post"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><article class="typography newsletter-post post"><div role="region" aria-label="Post header" class="post-header"><h1 dir="auto" class="post-title published title-X77sOw">Thoughts on API Reliability</h1><div aria-label="Post UFI" role="region" class="pencraft pc-display-flex pc-flexDirection-column pc-paddingBottom-16 pc-reset"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-gap-12 pc-alignItems-center pc-reset byline-wrapper"><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><div class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><time datetime="2022-09-30T17:17:52.150Z" title="2022-09-30T17:17:52.150Z">Sep 30, 2022</time></div></div></div></div></div><div class="pencraft pc-display-flex pc-gap-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (5)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">5</div></button></div><button tabindex="0" type="button" aria-label="View comments (0)" data-href="https://www.softwareatscale.dev/p/thoughts-on-api-reliability/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg></button><button tabindex="0" type="button" id="radix-P0-24" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div><div class="visibility-check"></div><div><div class="dt-post-body"><div class="available-content"><div dir="auto" class="body markup"><p>Building and maintaining a public API is a unique software engineering challenge. We have to consider (roughly prioritized):</p><ul><li><p>Security - exposing a service on the public internet has a variety of security implications - we want authentication/authorization as airtight as possible.</p></li><li><p>Reliability - the API is not useful if it’s down often.</p></li><li><p>Ease of use - it should be as frictionless as possible for developers to work with the API. Authentication flows, pagination, documentation should be as smooth and consistent as possible.</p></li><li><p>Stability - public API callers cannot change in lockstep with the internal codebase, so long term stability guarantees should be thought through.</p></li><li><p>Internal Developer Velocity - it should be simple for other developers in the organization to add new endpoints.</p></li></ul><p><span>This is a vast and complex area, so I want to focus on one topic - API reliability. Once we want to consider our API</span><a href="https://en.wikipedia.org/wiki/Software_release_life_cycle#General_availability_(GA)"> generally available</a><span> for a non-trivial set of users, we need to ensure it will stay reliable. In this post, we discuss common API reliability risks, mitigations, and a framework to keep our APIs reliable for the long term.</span></p><div class="subscription-widget-wrap"><div class="subscription-widget show-subscribe"><div class="preamble"><p>Thanks for reading Software at Scale! Subscribe for free to receive new posts and support my work.</p></div><div data-component-name="SubscribeWidget" class="subscribe-widget"><div class="pencraft pc-display-flex pc-justifyContent-center pc-reset"><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe-widget-preamble"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><input type="hidden" name="source_post_id" value="68658421"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div></div></div><h1 class="header-anchor-post"><strong>Reliability Risk Model</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§reliability-risk-model" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/reliability-risk-model" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h1><p><span>Just like </span><a href="https://slsa.dev/spec/v0.1/threats">security threat models</a><span>, it’s useful to think about all the risks and potential scenarios that our API should be able to defend against. </span></p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!MwBD!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!MwBD!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png 424w, https://substackcdn.com/image/fetch/$s_!MwBD!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png 848w, https://substackcdn.com/image/fetch/$s_!MwBD!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png 1272w, https://substackcdn.com/image/fetch/$s_!MwBD!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!MwBD!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png" width="1456" height="771" data-attrs="{&quot;src&quot;:&quot;https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:771,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:null,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:null,&quot;href&quot;:null,&quot;belowTheFold&quot;:false,&quot;topImage&quot;:true,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!MwBD!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png 424w, https://substackcdn.com/image/fetch/$s_!MwBD!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png 848w, https://substackcdn.com/image/fetch/$s_!MwBD!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png 1272w, https://substackcdn.com/image/fetch/$s_!MwBD!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png 1456w" sizes="100vw" fetchpriority="high" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><p>Breaking down risks into four categories:</p><p><strong>Development risks</strong><span>: New faulty application code or infrastructure as code changes can lead to reliability issues. Additionally, application architecture and engineering practices also dictate ongoing reliability. For example, the right RPC library will enforce request timeouts and prevent overloads.</span></p><p><strong>Deployment risks</strong><span>: The set of issues that prevent safe deployments. For example, the time it takes for a team to roll back a bad code change, and the likelihood that an on-call is empowered and trained to take a quick set of mitigations. There’s also the risk impact around faulty changes that can be tuned. For example, a faulty change might lead to a small availability impact to a canary environment, or a global outage, depending on our infrastructure and processes.</span></p><p><strong>Abuse and load risks</strong><span>: APIs can go down even if there’s no changes going out due to changes in client behavior. Bad actors, and good but misconfigured clients are almost indistinguishable in most cases - and there’s a set of risks around abusive request patterns that have to be considered.</span></p><p><strong>Infrastructure/dependency/operator risks: </strong><span>APIs are hosted somewhere, whether it may be a single machine in an office, or on thousands of cloud instances, and there’s physical infrastructure that might have issues. Alternatively, software infrastructure that our API depends on, like a caching layer, might fail and cause cascading issues.</span></p><p>Examples from my past experience/conversations:</p><ul><li><p>Bad code deployments - new bugs/misconfigurations that take our API down.</p></li><li><p>Scanners that check for known vulnerabilities of common deployments. For example, Wordpress scanners that check if our admin site is publicly available. These might cause a large request volume.</p></li><li><p>Security scanners deployed by the company’s security team that perform the same operations as above, but are less concerned about getting rate limited/banned. So the request throughput may even be higher.</p></li><li><p>Cron jobs that make large bursts of requests on the hour that a service isn’t provisioned for.</p></li><li><p><span>Large bursts of requests when a particular resource goes viral (</span><a href="https://news.ycombinator.com/item?id=20147951">the HackerNews hug of death</a><span>).</span></p></li><li><p><span>Large customers attempting a programmatic data-export/migration that cause database load due to</span><a href="https://stackoverflow.com/questions/49857507/understanding-hot-keys-in-dynamo-db"> hot keys</a><span>/contention.</span></p></li><li><p>Extremely large customers performing unbounded API operations.</p><ul><li><p>At one of my previous jobs, we had a concept of users and teams, and teams were generally not more than hundreds of users. But we signed up with a university, and they added forty thousand students to a one team. None of our features and APIs were designed for that sized team.</p></li></ul></li><li><p>Internal services or batch jobs that overload a service that serves external requests.</p></li><li><p>A single bad host in a small deployment that causes a large availability hit due to error amplification. </p><ul><li><p>For example, successful RPCs to the service generally take hundreds of milliseconds, but failures return almost instantaneously. Therefore, more requests get routed to the bad host since it has more capacity.</p></li></ul></li><li><p>The website is down but the on-call engineer does not feel comfortable rolling back and is waiting for a 90 minute CI build for a forward fix to go out.</p></li></ul><p>It’s worth coming up with a set of risks for our API, as it helps guide a set of principles to operate on and prioritize mitigations.</p><h1 class="header-anchor-post"><strong>Mitigations</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§mitigations" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/mitigations" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h1><p>These are some common mitigations for various reliability risks.</p><h2 class="header-anchor-post">Development Risks<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§development-risks" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/development-risks" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><h3 class="header-anchor-post"><strong>Bounded operations</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§bounded-operations" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/bounded-operations" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Our first observation is that most API defense measures like rate limiting, request quotas, and timeouts take place per operation. For example, rate limits are most often set up on requests per second. Therefore, we never want a single API operation to do unbounded work. Examples:</p><ul><li><p>List operations should always be paginated.</p><ul><li><p>Let’s say we have a ListUsers API call - this might be very cheap for small customers, but expensive for large customers - there’s increased database load, network bandwidth and serialization cost for the same operation. We have to tune the rate limit for our largest customers, which increases our developer friction.</p></li><li><p>In addition, the number of returned items have to be capped. Otherwise, pagination is not useful.</p></li></ul></li><li><p>Set operations that need to operate on a list of items should always take a list of IDs.</p><ul><li><p>And limit the number of items that can be operated on at a time.</p></li></ul></li></ul><p>This is likely the highest ROI principle to keep in mind - it’s easy to add more rate limiting infrastructure after the fact, but it’s hard to convert an unpaginated operation into a paginated one, because users will depend on older behavior.</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!Tna_!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!Tna_!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png 424w, https://substackcdn.com/image/fetch/$s_!Tna_!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png 848w, https://substackcdn.com/image/fetch/$s_!Tna_!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png 1272w, https://substackcdn.com/image/fetch/$s_!Tna_!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!Tna_!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png" width="1456" height="296" data-attrs="{&quot;src&quot;:&quot;https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/c7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:296,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:null,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:null,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!Tna_!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png 424w, https://substackcdn.com/image/fetch/$s_!Tna_!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png 848w, https://substackcdn.com/image/fetch/$s_!Tna_!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png 1272w, https://substackcdn.com/image/fetch/$s_!Tna_!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a><figcaption class="image-caption">An example email from a vendor adding pagination to their APIs</figcaption></figure></div><h3 class="header-anchor-post"><strong>Request timeouts</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§request-timeouts" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/request-timeouts" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p><span>Low request timeouts are primarily an expectation setting mechanism and guardrail - if our API operations take too long, timeouts inform users that the operations have failed. If this happens sufficiently often, ideally, the API operator finds out about it through observability tools or support tickets, and can change the operation to do less work. It forces us to time-bound API operations. Ideally, the timeout also aborts the request so that the operation does not do any more work, but that’s often hard to implement correctly. For example, it’s really tricky to abort an</span><a href="https://stackoverflow.com/questions/69545993/express-js-abort-request-on-timeout?rq=1"> ongoing request in</a><span> Node. These little things are deeply considered in</span><a href="https://grpc.io/blog/deadlines/"> RPC libraries</a><span> and implemented best by them - don’t roll your own request timeouts!</span></p><p>It’s very easy to bump up request timeouts as a cheap mitigation to a user perceived bug, but resist this temptation/schedule work to clean this up.</p><h2 class="header-anchor-post">Deployment Risks<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§deployment-risks" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/deployment-risks" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><h3 class="header-anchor-post">Change management <div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§change-management" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/change-management" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Once a faulty change has been rolled out, there needs to be a clear set of owners or DRIs who are trained or feel empowered to mitigate the issue. For business critical services, consider having an on-call rotation of engineers who are trained and equipped to drive mitigations.</p><h4 class="header-anchor-post">Emergency rollbacks<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§emergency-rollbacks" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/emergency-rollbacks" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h4><p>For stateless services, a rollback is almost always the right approach to mitigate code change related outages. Rollbacks don’t need to go through the full test suite, as long as we can guarantee that service is being deployed at a change that was previously tested and deployed - which greatly helps with mitigation time.</p><h3 class="header-anchor-post">API validation tests<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§api-validation-tests" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/api-validation-tests" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p><span>A staging deployment that has no reliability bearings on the production deployment is a great test bed for risky changes and continuous </span><a href="https://docs.datadoghq.com/synthetics/">API validation tests</a><span>.</span></p><h3 class="header-anchor-post">Canary deployments / Staged rollouts<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§canary-deployments-staged-rollouts" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/canary-deployments-staged-rollouts" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p><span>After a certain scale, it’s worth considering a separate canary deployment which receives code changes before the rest of the fleet. This helps prevent large outages if a faulty code change hasn’t been tested well enough in other environments. A canary deployment has the advantage of being most similar to the production environment, so it helps testing changes in a more realistic scenario. More sophisticated setups may benefit from </span><a href="https://netflixtechblog.com/automated-canary-analysis-at-netflix-with-kayenta-3260bc7acc69">Canary Analysis</a><span>.</span></p><p><span>A canary deployment has an unexpected benefit of being a load request</span><a href="https://www.softwareatscale.dev/p/why-you-should-frequently-turn-down"> playground</a><span> and can help us understand our load limits. For example, by introducing a load balancer that automatically retries on 500s and spreads requests across canary and production deployments, we can afford to have a few requests to our canaries fail without impacting global availability. This lets us run experiments, like tuning down the number of canary replicas and see how our instances operate under load. This area is worth investing in if we run a high throughput, business critical service.</span></p><p>Additionally, larger, multi-region or stateful services should be deployed in stages to reduce the blast radius of faulty changes.</p><h3 class="header-anchor-post">Separating live and batch traffic<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§separating-live-and-batch-traffic" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/separating-live-and-batch-traffic" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>With a service oriented architecture, it’s very simple to overload an externally facing, critical service with an internal batch job that needs to RPC to the service. Even worse, I’ve seen cases where mobile applications run end to end tests against the production service. The major downside here is that completely disabling such a workflow becomes infeasible over time, since other business critical parts of the system depend on the API.</p><p><span>It’s worth trying to completely separate traffic tiers, or even deployments, to prevent non-critical traffic from taking down our system. The Google </span><a href="https://sre.google/sre-book/handling-overload/">SRE book has a good notion on criticality</a><span>.</span></p><h2 class="header-anchor-post">Abuse and load risks<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§abuse-and-load-risks" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/abuse-and-load-risks" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><h3 class="header-anchor-post">Token limits<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§token-limits" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/token-limits" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Simple limits can go a long way in helping with API reliability. For example, a limit on the number of API tokens created per user can prevent a malicious bot-farm from working around disabled/rate-limited API tokens.</p><h3 class="header-anchor-post"><strong>Admin tools</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§admin-tools" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/admin-tools" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Consider having an option to disable a misbehaving API key in an admin portal. This significantly reduces the amount of time it will take to mitigate an outage, and the on-call engineer will thank us for it. Having this ability to disable misbehaving API keys relies on having enough observability to know which clients are being the most troublesome. The simplest path is to have sampled logs on request counts, broken down by customer ID/user ID/hashed API token/token ID so we can find the offender easily.</p><h3 class="header-anchor-post"><strong>Web Application Firewalls (WAFs)</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§web-application-firewalls-wafs" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/web-application-firewalls-wafs" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p><span>WAFs (</span><a href="https://aws.amazon.com/waf/">AWS</a><span>,</span><a href="https://www.cloudflare.com/learning/ddos/glossary/web-application-firewall-waf/"> Cloudflare</a><span>) are multi-purpose, and help with basic security posture as well as coarse rate limiting.</span></p><p><span>WAFs often let us set up rate limiting rules per IP address, and some even let us set up rules on</span><a href="https://cloud.google.com/armor/docs/rules-language-reference"> request headers/bodies</a><span>. They’re good at preventing misconfigured clients, scanners, and other tools from hitting our service at all, so that more sophisticated rate limiting only needs to be applied at another layer with lower request volumes. Another advantage is that they’re very cheap -</span><a href="https://aws.amazon.com/waf/pricing/"> AWS charges 60 cents per million requests</a><span> + a flat fee, and in terms of engineering time - it’s very easy to add it in a cloud console/</span><a href="https://en.wikipedia.org/wiki/Infrastructure_as_code">IaC</a><span>. Finally, WAFs often have an option to manually block requests from an I.P. address, which can come in very handy during emergencies. Open source proxies like Envoy have enough configurability to enable WAF-like functionality, and may</span><a href="https://github.com/envoyproxy/envoy/issues/7918"> support holistic in-built ones</a><span> some day.</span></p><p><span>A limitation of WAFs is that they only apply on small time frames, like “request count over the past five minutes”. So they won’t really help with cron-jobs with a greater than five minute cadence, and they’re completely useless for the first few minutes of an outage. Another issue is that many universities and corporate entities have a </span><a href="https://en.wikipedia.org/wiki/Network_address_translation">NAT gateway</a><span>, which all requests through a single I.P. address. So over-aggressive WAF rules are certain to block legitimate traffic.</span></p><h3 class="header-anchor-post"><strong>API Gateways</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§api-gateways" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/api-gateways" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>API Gateways are a nebulous topic. They provide a central point for our internal services with the external internet, for example, you can serve requests through different services for different routes on the same domain. For the purposes of reliability, they can be thought of as a layer of abstraction between the chaotic outside internet and internal systems. For example, if our setup will ever have more than a single service available over the external internet, we will have to think about the same measures - WAF, observability, timeouts, rate limits, for all such services. Whereas if all our services are protected by the gateway, then these defense measures only need to be set up once. And over time, as we might want to centralize other aspects, like authentication/authorization, and the gateway’s routing logic can be updated accordingly and without needing to duplicate this logic across several services.</p><p><span>In practice,</span><a href="https://aws.amazon.com/api-gateway/"> API Gateways</a><span> often provide</span><a href="https://zuplo.com/"> rate limiting</a><span> as well, so we don’t have to re-implement it in-house. They offer capabilities like</span><a href="https://docs.aws.amazon.com/apigateway/latest/developerguide/canary-release.html"> canary deployments</a><span> to reduce the blast radius of code deployments. So it’s worth considering whether to start an API architecture with a gateway.</span></p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!VVKN!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!VVKN!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png 424w, https://substackcdn.com/image/fetch/$s_!VVKN!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png 848w, https://substackcdn.com/image/fetch/$s_!VVKN!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png 1272w, https://substackcdn.com/image/fetch/$s_!VVKN!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!VVKN!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png" width="1456" height="1037" data-attrs="{&quot;src&quot;:&quot;https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:1037,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:null,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:null,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!VVKN!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png 424w, https://substackcdn.com/image/fetch/$s_!VVKN!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png 848w, https://substackcdn.com/image/fetch/$s_!VVKN!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png 1272w, https://substackcdn.com/image/fetch/$s_!VVKN!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><h3 class="header-anchor-post"><strong>Rate limiting</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§rate-limiting" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/rate-limiting" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>The value proposition of rate limiting is fairly straightforward. The main question of rate limiting is always whether to fail open or fail closed.</p><ul><li><p>If the rate limiter fails open - we have to consider the case where an abusive client causes the rate limiter to go down. We need to be vigilant about monitoring operational errors in the rate limiter, since it can silently be disabled, which might lead to an outage.</p></li><li><p>If it fails closed - we have to trust another infrastructure component with not degrading the availability of our API. For example, if our API has an availability SLA, then our rate limiter needs to meet or exceed that.</p><ul><li><p>See the discussion on load testing below.</p></li></ul></li></ul><h3 class="header-anchor-post"><strong>Auto-scaling</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§auto-scaling" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/auto-scaling" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p><span>In a nutshell, auto-scaling means dynamically changing the number of replicas for a service based on a set of signals, like CPU utilization, request queue length, etc. </span><a href="https://render.com/autoscaling">Infrastructure platforms</a><span> and </span><a href="https://github.com/kubernetes/autoscaler">orchestration frameworks</a><span> often provide auto-scaling out of the box.</span></p><p>We have to be very careful to understand our systems before deploying auto-scaling. Let’s take the following barebones API architecture:</p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!LrTg!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!LrTg!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png 424w, https://substackcdn.com/image/fetch/$s_!LrTg!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png 848w, https://substackcdn.com/image/fetch/$s_!LrTg!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png 1272w, https://substackcdn.com/image/fetch/$s_!LrTg!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!LrTg!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png" width="1456" height="425" data-attrs="{&quot;src&quot;:&quot;https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/e81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:425,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:null,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:null,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!LrTg!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png 424w, https://substackcdn.com/image/fetch/$s_!LrTg!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png 848w, https://substackcdn.com/image/fetch/$s_!LrTg!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png 1272w, https://substackcdn.com/image/fetch/$s_!LrTg!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><p><span>It might be simple to set up auto-scaling on the API service based on request latency. But that increases the number of concurrent requests to the database. If the database slows down due to increased load, that might lead to the auto-scaler adding even more API service replicas, which adds even more pressure to the database and might cause an outage. We expect this situation to happen whenever we’re I/O bound. This frequently happened at</span><a href="https://www.softwareatscale.dev/p/software-at-scale-002-christine-dodrill#details"> Heroku</a><span>. Reactive auto-scaling should not be seen as a solution to malicious/abusive patterns, and more as a toil reduction measure for human operators, as well as a cost-saving measure for seasonal request loads. It is not an effective measure for sudden, borderline abusive changes in request throughput.</span></p><p>There’s also seasonal auto-scaling - preemptively scaling up instance counts during business hours or some other cadence due to a well known seasonal increases in load. This is generally more well understood, and a very reasonable approach.</p><h2 class="header-anchor-post">Infrastructure / dependency risks<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§infrastructure-dependency-risks" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/infrastructure-dependency-risks" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><h3 class="header-anchor-post">Outlier host detection<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§outlier-host-detection" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/outlier-host-detection" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p><span>Given a sufficiently large number of instances running in a deployment, it’s very likely that a few instances will not be as performant as others. Cloud providers try to automatically replace or terminate poorly performing instances, but often, we have to do that work ourselves. That’s why, it may become relevant to set up </span><a href="https://www.datadoghq.com/blog/outlier-detection-algorithms-at-datadog/">monitoring for outliers</a><span> and </span><a href="https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/outlier#:~:text=Outlier%20detection%20and%20ejection%20is,the%20healthy%20load%20balancing%20set.">prevent requests</a><span> reaching them. A poorly performing instance in a small deployment may cause error amplification if the instance fails requests much more quickly than a well-behaved instance processing successful requests.</span></p><h3 class="header-anchor-post">Load testing<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§load-testing" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/load-testing" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p><span>With a sufficiently complex architecture, it’s very hard to know how a system will perform under increased load. It’s worth understanding through synthetic load so that we know the system’s limits and can plan for future growth or seasonality of requests. Read </span><a href="https://itnext.io/how-i-tripled-load-in-production-for-testing-purposes-at-amazon-518df002465b">this</a><span> for a fascinating insight into load testing at Amazon.</span></p><h1 class="header-anchor-post"><strong>Conclusion</strong><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§conclusion" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/conclusion" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h1><p>To tie this up, an API developer has an array of things to consider to strengthen a public API. A suggested framework:</p><ul><li><p>Come up with a reliability threat model to the API which produces a set of risks.</p><ul><li><p>All APIs have to defend against scanners - this is a standard risk.</p></li><li><p>APIs that allow unauthenticated use have to defend against bots and abuse much more than ones that only allow authenticated use - so some risks are unique to our business and technical context.</p></li></ul></li><li><p><span>Understand the impact of each risk - would the lack of defenses for the risk take down our entire site, or would the impact be limited to a single customer? These answers depend on our pre-existing software architecture. For example, if we have a</span><a href="https://www.techtarget.com/searchcloudcomputing/definition/single-tenancy"> single tenant architecture</a><span>, the blast radius due to an abusive customer is much smaller.</span></p></li><li><p>Consider the mitigation speed for various risks, especially as they tie into your SLOs, as that will drive a discussion into remediation procedures.</p></li></ul><p>With this input, we can come up with mitigations, a software architecture, and development principles and processes to keep our APIs reliable for a long time.</p><h3 class="header-anchor-post">Credits<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§credits" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://www.softwareatscale.dev/i/68658421/credits" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h3><p>Thanks to Allan Reyes for reading early drafts of this post and providing excellent feedback!</p><div class="subscription-widget-wrap"><div class="subscription-widget show-subscribe"><div class="preamble"><p>Thanks for reading Software at Scale! Subscribe for free to receive new posts.</p></div><div data-component-name="SubscribeWidget" class="subscribe-widget"><div class="pencraft pc-display-flex pc-justifyContent-center pc-reset"><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe-widget-preamble"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><input type="hidden" name="source_post_id" value="68658421"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div></div></div></div></div><div class="visibility-check"></div></div><div class="post-footer"><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG border-top-detail-themed-k9TZAY border-bottom-detail-themed-Ua9186 post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (5)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">5</div></button></div><button tabindex="0" type="button" aria-label="View comments (0)" data-href="https://www.softwareatscale.dev/p/thoughts-on-api-reliability/comments" class="pencraft pc-reset pencraft post-ufi-button style-button post-ufi-comment-button no-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-message-circle"><path d="M7.9 20A9 9 0 1 0 4 16.1L2 22Z"></path></svg></g></svg></button><button tabindex="0" type="button" id="radix-P0-53" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button no-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_secondary-S63h9o size_md-gCDS3o"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-left"><path d="m12 19-7-7 7-7"></path><path d="M19 12H5"></path></svg>Previous</button><button tabindex="0" type="button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_secondary-S63h9o size_md-gCDS3o">Next<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-right"><path d="M5 12h14"></path><path d="m12 5 7 7-7 7"></path></svg></button></div></div></div></article></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="visibility-check"></div><div id="discussion" class="pencraft pc-display-flex pc-flexDirection-column pc-gap-16 pc-paddingTop-32 pc-paddingBottom-32 pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-32 pc-reset container"><h4 class="pencraft pc-reset line-height-24-jnGwiv font-display-nhmvtD size-20-P_cSRT weight-bold-DmI9lw reset-IxiVJZ">Discussion about this post</h4><div class="pencraft pc-alignSelf-flex-start pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Select discussion type" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-67-content-comments" data-state="active" id="radix-P0-67-trigger-comments" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Comments</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-67-content-restacks" data-state="inactive" id="radix-P0-67-trigger-restacks" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Restacks</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div></div></div><div id="substack-comments" class="single-post-section comments-section"><div class="container"><div class="visibility-check"></div><div data-test-id="comment-input" class="pencraft pc-display-flex pc-reset flex-grow-rzmknG"><form class="comment-input-form form-CkZ7Kt"><div style="--scale:32px;" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><div style="--scale:32px;" title="User" class="pencraft pc-display-flex pc-width-32 pc-height-32 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png" alt="User's avatar" srcset="https://substackcdn.com/image/fetch/$s_!TnFC!,w_32,h_32,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 1x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_64,h_64,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 2x, https://substackcdn.com/image/fetch/$s_!TnFC!,w_96,h_96,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack.com%2Fimg%2Favatars%2Fdefault-light.png 3x" width="32" height="32" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-8 pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-flexDirection-column pc-boxShadow-xs pc-reset bg-primary-zk6FDl flex-grow-rzmknG border-detail-EGrm7T pc-borderRadius-sm"><textarea name="body" placeholder="Write a comment..." aria-label="Write a comment..." rows="4" class="pencraft input-qHk4bN autogrowing-_ipn9Y textarea-GbEjRX inputText-pV_yWb unstyled-IJLUJU"></textarea></div><div data-state="closed" class="pencraft pc-display-flex pc-flexDirection-column pc-reset overflow-hidden-WdpwT6"></div></div></form></div></div></div></div><div class="single-post-section"><div class="container"><div class="visibility-check"></div><div style="margin-left:-8px;margin-right:-8px;" aria-label="Top Posts Footer" role="region" class="pencraft pc-paddingTop-24 pc-paddingBottom-24 pc-reset"><div class="portable-archive empty-list"><div aria-label="Archive sort tabs" role="navigation" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-8 pc-paddingRight-8 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Tabs" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-80-content-top" data-state="active" id="radix-P0-80-trigger-top" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Top</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-80-content-new" data-state="inactive" id="radix-P0-80-trigger-new" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Latest</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-80-content-community" data-state="inactive" id="radix-P0-80-trigger-community" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Discussions</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></div><div class="portable-archive-list"><p class="portable-archive-empty dt-archive-empty-state">No posts</p></div></div></div></div></div><div class="visibility-check"></div><div class="pencraft pc-display-contents pc-reset pubInvertedTheme-U483dz"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-justifyContent-center pc-padding-48 pc-mobile-padding-24 pc-reset bg-primary-zk6FDl container-jsOc9L"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-gap-24 pc-flexWrap-wrap pc-reset content-jLbYeh"><h3 class="pencraft pc-reset color-primary-zABazT align-center-y7ZD4w line-height-28-s562kJ font-display-nhmvtD size-24-lFU3ly weight-semibold-uqA4FV reset-IxiVJZ">Ready for more?</h3><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe_footer"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB emailInputOnAccentBackground-TfaCGr input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonOnAccentBackground-vmEt94 buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div></div></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset container-K5XPov"></div><div class="footer-wrap publication-footer"><div class="visibility-check"></div><div class="footer themed-background"><div class="container"><div class="footer-terms"><span>© 2026 Utsav Shah</span><span> · </span><a href="https://substack.com/privacy" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Privacy</a><span> ∙ </span><a href="https://substack.com/tos" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Terms</a><span> ∙ </span><a href="https://substack.com/ccpa#personal-data-collected" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Collection notice</a></div><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-center pc-alignItems-center pc-reset footerButtons-ap9Sk7"><a data-native="true" href="https://substack.com/signup?utm_source=substack&amp;utm_medium=web&amp;utm_content=footer" class="footerSubstackCta-v5HWfj"><svg aria-hidden="true" width="1000" height="1000" viewBox="0 0 1000 1000" fill="#ff6719" stroke-width="1.8" stroke="none" xmlns="http://www.w3.org/2000/svg"><g><path d="M764.166 348.371H236.319V419.402H764.166V348.371Z"></path><path d="M236.319 483.752V813.999L500.231 666.512L764.19 813.999V483.752H236.319Z"></path><path d="M764.166 213H236.319V284.019H764.166V213Z"></path></g></svg> Start your Substack</a><a data-native="true" href="https://substack.com/app/app-store-redirect?utm_campaign=app-marketing&amp;utm_content=web-footer-button" class="footerSubstackCta-v5HWfj getTheApp-Yk3w1O noIcon-z7v9D8">Get the app</a></div><div translated class="pencraft pc-reset reset-IxiVJZ footer-slogan-blurb"><a href="https://substack.com" data-native="true">Substack</a> is the home for great culture</div></div></div></div></div><div role="region" aria-label="Notifications (F8)" tabindex="-1" style="pointer-events:none;"><ol tabindex="-1" style="--offset:0px;z-index:1000;" class="viewport-_BM4Bg"></ol></div><div></div><script suppressHydrationWarning nonce="VqkFSleQStUMFdS2hc29nQ">window.__staticRouterHydrationData = JSON.parse("{\"loaderData\":{},\"actionData\":null,\"errors\":null}");</script>
</div>
<script src="https://js.sentry-cdn.com/6c2ff3e3828e4017b7faf7b63e24cdf8.min.js" crossorigin="anonymous"></script>
<script nonce="VqkFSleQStUMFdS2hc29nQ">
window.Sentry && window.Sentry.onLoad(function() {
window.Sentry.init({
environment: window._preloads.sentry_environment,
dsn: window._preloads.sentry_dsn,
})
})
</script>
<script nonce="VqkFSleQStUMFdS2hc29nQ">window._preloads = JSON.parse("{\"cspNonce\":\"VqkFSleQStUMFdS2hc29nQ\",\"isEU\":false,\"language\":\"en\",\"country\":\"US\",\"leaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"enabledLeaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"userLocale\":{\"language\":\"en\",\"region\":\"US\",\"source\":\"accept-language\"},\"base_url\":\"https://www.softwareatscale.dev\",\"stripe_publishable_key\":\"pk_live_51QfnARLDSWi1i85FBpvw6YxfQHljOpWXw8IKi5qFWEzvW8HvoD8cqTulR9UWguYbYweLvA16P7LN6WZsGdZKrNkE00uGbFaOE3\",\"captcha_site_key\":\"6LeI15YsAAAAAPXyDcvuVqipba_jEFQCjz1PFQoz\",\"pub\":{\"apple_pay_disabled\":false,\"apex_domain\":null,\"author_id\":6567678,\"byline_images_enabled\":true,\"bylines_enabled\":false,\"chartable_token\":null,\"community_enabled\":true,\"copyright\":\"Utsav Shah\",\"cover_photo_url\":null,\"created_at\":\"2020-06-13T04:08:52.332Z\",\"custom_domain_optional\":false,\"custom_domain\":\"www.softwareatscale.dev\",\"default_comment_sort\":\"best_first\",\"default_coupon\":null,\"default_group_coupon\":null,\"default_show_guest_bios\":true,\"email_banner_url\":null,\"email_from_name\":\"Software at Scale by Utsav\",\"email_from\":null,\"embed_tracking_disabled\":false,\"explicit\":false,\"expose_paywall_content_to_search_engines\":true,\"fb_pixel_id\":null,\"fb_site_verification_token\":null,\"flagged_as_spam\":false,\"founding_subscription_benefits\":null,\"free_subscription_benefits\":null,\"ga_pixel_id\":\"G-HR35NWP3CE\",\"google_site_verification_token\":\"IORiP7_fax0ESrUYObrVUA0QF8qgq2BzPsISmpFzgfg\",\"google_tag_manager_token\":\"G-HR35NWP3CE\",\"hero_image\":null,\"hero_text\":\"Discussions on running software systems, teams, and products.\",\"hide_intro_subtitle\":false,\"hide_intro_title\":false,\"hide_podcast_feed_link\":false,\"homepage_type\":\"magaziney\",\"id\":55725,\"image_thumbnails_always_enabled\":false,\"invite_only\":false,\"hide_podcast_from_pub_listings\":false,\"hide_podcast_from_pub_listings_prior_to\":null,\"language\":\"en\",\"logo_url_wide\":null,\"logo_url\":\"https://substackcdn.com/image/fetch/$s_!NNuR!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png\",\"minimum_group_size\":2,\"moderation_enabled\":true,\"name\":\"Software at Scale\",\"paid_subscription_benefits\":null,\"parsely_pixel_id\":null,\"chartbeat_domain\":null,\"payments_state\":\"disabled\",\"paywall_free_trial_enabled\":false,\"podcast_art_url\":\"https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/3b763612-d8e3-4199-95d2-d9e9a8d2a180_3000x3000.png\",\"paid_podcast_episode_art_url\":null,\"podcast_byline\":null,\"podcast_description\":\"Software at Scale is where we discuss the technical stories behind large software applications.\",\"podcast_enabled\":true,\"podcast_feed_url\":null,\"podcast_title\":null,\"post_preview_limit\":null,\"primary_user_id\":6567678,\"require_clickthrough\":false,\"show_pub_podcast_tab\":false,\"show_recs_on_homepage\":true,\"subdomain\":\"reliability\",\"subscriber_invites\":0,\"support_email\":null,\"theme_var_background_pop\":\"#0068ef\",\"theme_var_color_links\":false,\"theme_var_cover_bg_color\":null,\"trial_end_override\":null,\"twitter_pixel_id\":\"o6rzq\",\"type\":\"newsletter\",\"post_reaction_faces_enabled\":true,\"is_personal_mode\":false,\"plans\":null,\"stripe_user_id\":null,\"stripe_country\":null,\"stripe_publishable_key\":null,\"stripe_platform_account\":null,\"automatic_tax_enabled\":null,\"author_name\":\"Utsav Shah\",\"author_handle\":\"reliability\",\"author_photo_url\":\"https://substackcdn.com/image/fetch/$s_!Ta5D!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F0f431e7e-fd06-4ab7-a5df-cda6f979e27a_1536x1536.jpeg\",\"author_bio\":\"Software Engineer interested in building simple, self correcting and reliable systems. Twitter: @utsav_sha\",\"twitter_screen_name\":\"utsav_sha\",\"has_custom_tos\":false,\"has_custom_privacy\":false,\"theme\":{\"background_pop_color\":\"#1a2e05\",\"web_bg_color\":\"#fafafa\",\"cover_bg_color\":\"#fafafa\",\"publication_id\":55725,\"color_links\":null,\"font_preset_heading\":\"heavy_sans\",\"font_preset_body\":\"sans\",\"font_family_headings\":null,\"font_family_body\":null,\"font_family_ui\":null,\"font_size_body_desktop\":null,\"print_secondary\":null,\"custom_css_web\":null,\"custom_css_email\":null,\"home_hero\":\"magaziney\",\"home_posts\":\"list\",\"home_show_top_posts\":false,\"hide_images_from_list\":false,\"home_hero_alignment\":\"left\",\"home_hero_show_podcast_links\":true,\"default_post_header_variant\":null,\"custom_header\":null,\"custom_footer\":null,\"social_media_links\":null,\"font_options\":null,\"section_template\":null,\"custom_subscribe\":null,\"design_template\":null,\"design_template_options\":null},\"threads_v2_settings\":{\"first_thread_email_sent_at\":\"2022-11-04T02:53:50.304+00:00\",\"activated_at\":\"2022-11-04T02:53:19.96+00:00\",\"reader_thread_notifications_enabled\":false,\"create_thread_minimum_role\":\"contributor\",\"photo_replies_enabled\":true,\"boost_free_subscriber_chat_preview_enabled\":false,\"push_suppression_enabled\":false},\"default_group_coupon_percent_off\":null,\"default_group_coupon_include_founding\":null,\"pause_return_date\":null,\"has_posts\":true,\"has_recommendations\":true,\"first_post_date\":\"2020-11-03T16:51:18.309Z\",\"has_podcast\":true,\"has_free_podcast\":true,\"has_subscriber_only_podcast\":false,\"has_community_content\":true,\"rankingDetail\":\"Launched 6 years ago\",\"rankingDetailFreeIncluded\":\"Thousands of subscribers\",\"rankingDetailOrderOfMagnitude\":0,\"rankingDetailFreeIncludedOrderOfMagnitude\":1000,\"rankingDetailFreeSubscriberCount\":\"Over 1,000 subscribers\",\"rankingDetailByLanguage\":{\"ar\":{\"rankingDetail\":\"\u062A\u0645 \u0627\u0644\u0625\u0637\u0644\u0627\u0642 6 years ago\"},\"ca\":{\"rankingDetail\":\"S\u2019ha llan\u00E7at fa 6 anys\"},\"da\":{\"rankingDetail\":\"Lancering 6 \u00E5r\"},\"de\":{\"rankingDetail\":\"vor 6 Jahren gelauncht\"},\"es\":{\"rankingDetail\":\"Lanzado hace 6 a\u00F1os\"},\"fr\":{\"rankingDetail\":\"Lanc\u00E9 il y a 6 ann\u00E9es\"},\"id\":{\"rankingDetail\":\"Diluncurkan 6 tahun yang lalu\"},\"ja\":{\"rankingDetail\":\"\u958B\u59CB\u65E5 6\u5E74\u524D\"},\"ko\":{\"rankingDetail\":\"6\uB144 \uC804\uC5D0 \uCD9C\uC2DC\"},\"nb\":{\"rankingDetail\":\"Lansert 6 \u00E5r\"},\"nl\":{\"rankingDetail\":\"Gelanceerd 6 jaar geleden\"},\"pl\":{\"rankingDetail\":\"Uruchomiono 6 lat temu\"},\"pt\":{\"rankingDetail\":\"Lan\u00E7ado 6 anos\"},\"pt-br\":{\"rankingDetail\":\"Lan\u00E7ado 6 anos\"},\"en-gb\":{\"rankingDetail\":\"Launched 6 years ago\"},\"it\":{\"rankingDetail\":\"Lanciato 6 anni\"},\"tr\":{\"rankingDetail\":\"6 y\u0131l ba\u015Flat\u0131ld\u0131\"},\"sv\":{\"rankingDetail\":\"Lanserad 6 \u00E5r sedan\"},\"fi\":{\"rankingDetail\":\"Launched 6 vuotta\"},\"is\":{\"rankingDetail\":\"Launched 6 \u00E1r\"},\"zh-hans\":{\"rankingDetail\":\"\u4E8E6 \u5E74\u524D\u4E0A\u7EBF\"},\"zh-hant\":{\"rankingDetail\":\"\u65BC 6 \u5E74\u524D \u63A8\u51FA\"},\"en\":{\"rankingDetail\":\"Launched 6 years ago\"}},\"freeSubscriberCount\":\"1,000\",\"freeSubscriberCountOrderOfMagnitude\":\"1.8K+\",\"author_bestseller_tier\":0,\"author_badge\":null,\"disable_monthly_subscriptions\":false,\"disable_annual_subscriptions\":false,\"hide_post_restacks\":false,\"notes_feed_enabled\":true,\"showIntroModule\":false,\"isPortraitLayout\":false,\"last_chat_post_at\":\"2023-02-01T17:18:04.383Z\",\"primary_profile_name\":\"Utsav Shah\",\"primary_profile_photo_url\":\"https://substackcdn.com/image/fetch/$s_!Ta5D!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F0f431e7e-fd06-4ab7-a5df-cda6f979e27a_1536x1536.jpeg\",\"no_follow\":false,\"sponsorshipCampaigns\":{},\"paywall_chat\":\"free\",\"sections\":[],\"podcastTabInfo\":null,\"multipub_migration\":null,\"navigationBarItems\":[],\"has_active_perks\":false,\"contributors\":[{\"name\":\"Utsav Shah\",\"handle\":\"reliability\",\"role\":\"admin\",\"owner\":true,\"user_id\":6567678,\"photo_url\":\"https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/0f431e7e-fd06-4ab7-a5df-cda6f979e27a_1536x1536.jpeg\",\"bio\":\"Software Engineer interested in building simple, self correcting and reliable systems. Twitter: @utsav_sha\",\"status\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null}}],\"threads_v2_enabled\":true,\"viralGiftsConfig\":null,\"no_index\":false,\"tier\":2,\"can_set_google_site_verification\":true,\"can_have_sitemap\":true,\"founding_plan_name_english\":\"Founding Member\",\"base_url\":\"https://www.softwareatscale.dev\",\"hostname\":\"www.softwareatscale.dev\",\"is_on_substack\":false,\"spotify_podcast_settings\":{\"id\":4734,\"publication_id\":55725,\"section_id\":null,\"spotify_uri\":null,\"spotify_podcast_title\":null,\"created_at\":\"2024-04-11T18:05:40.194Z\",\"updated_at\":\"2024-04-11T18:05:40.194Z\",\"currently_published_on_spotify\":false,\"spotify_show_url\":\"\"},\"unified_podcast_settings\":null,\"podcastPalette\":{\"Vibrant\":{\"rgb\":[92,70,172],\"population\":94},\"DarkVibrant\":{\"rgb\":[49,39,85],\"population\":306},\"LightVibrant\":{\"rgb\":[145,196,232],\"population\":63},\"Muted\":{\"rgb\":[84,84,164],\"population\":2},\"DarkMuted\":{\"rgb\":[76,76,78],\"population\":46},\"LightMuted\":{\"rgb\":[138,165,191],\"population\":42}},\"pageThemes\":{\"podcast\":null},\"multiple_pins\":true,\"supports_ip_content_unlock\":false,\"appTheme\":{\"colors\":{\"accent\":{\"name\":\"#1a2e05\",\"primary\":{\"r\":26,\"g\":46,\"b\":5,\"a\":1},\"primary_hover\":{\"r\":18,\"g\":30,\"b\":0,\"a\":1},\"primary_elevated\":{\"r\":18,\"g\":30,\"b\":0,\"a\":1},\"secondary\":{\"r\":26,\"g\":46,\"b\":5,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":26,\"g\":46,\"b\":5,\"a\":0.2},\"bg_hover\":{\"r\":26,\"g\":46,\"b\":5,\"a\":0.3},\"dark\":{\"primary\":{\"r\":59,\"g\":80,\"b\":39,\"a\":1},\"primary_hover\":{\"r\":76,\"g\":97,\"b\":55,\"a\":1},\"primary_elevated\":{\"r\":76,\"g\":97,\"b\":55,\"a\":1},\"secondary\":{\"r\":59,\"g\":80,\"b\":39,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":59,\"g\":80,\"b\":39,\"a\":0.2},\"bg_hover\":{\"r\":59,\"g\":80,\"b\":39,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.8},\"secondary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.6},\"tertiary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.4},\"accent\":{\"r\":26,\"g\":46,\"b\":5,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":112,\"g\":135,\"b\":90,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"primary_elevated\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"secondary\":{\"r\":238,\"g\":238,\"b\":238,\"a\":1},\"secondary_elevated\":{\"r\":206.90096477355226,\"g\":206.90096477355175,\"b\":206.9009647735519,\"a\":1},\"tertiary\":{\"r\":219,\"g\":219,\"b\":219,\"a\":1},\"quaternary\":{\"r\":182,\"g\":182,\"b\":182,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}}}},\"portalAppTheme\":{\"colors\":{\"accent\":{\"name\":\"#1a2e05\",\"primary\":{\"r\":26,\"g\":46,\"b\":5,\"a\":1},\"primary_hover\":{\"r\":13,\"g\":23,\"b\":3,\"a\":1},\"primary_elevated\":{\"r\":26,\"g\":46,\"b\":5,\"a\":1},\"secondary\":{\"r\":26,\"g\":46,\"b\":5,\"a\":1},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3},\"dark\":{\"primary\":{\"r\":59,\"g\":80,\"b\":39,\"a\":1},\"primary_hover\":{\"r\":76,\"g\":97,\"b\":55,\"a\":1},\"primary_elevated\":{\"r\":76,\"g\":97,\"b\":55,\"a\":1},\"secondary\":{\"r\":59,\"g\":80,\"b\":39,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":59,\"g\":80,\"b\":39,\"a\":0.2},\"bg_hover\":{\"r\":59,\"g\":80,\"b\":39,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"secondary\":{\"r\":132,\"g\":133,\"b\":133,\"a\":1},\"tertiary\":{\"r\":144,\"g\":144,\"b\":144,\"a\":1},\"accent\":{\"r\":26,\"g\":46,\"b\":5,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":112,\"g\":135,\"b\":90,\"a\":1}}},\"bg\":{\"name\":\"#fafafa\",\"hue\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"tint\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"primary\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"primary_hover\":{\"r\":235,\"g\":235,\"b\":235,\"a\":1},\"primary_elevated\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"secondary\":{\"r\":235,\"g\":235,\"b\":235,\"a\":1},\"secondary_elevated\":{\"r\":235,\"g\":235,\"b\":235,\"a\":1},\"tertiary\":{\"r\":216,\"g\":216,\"b\":216,\"a\":1},\"quaternary\":{\"r\":179,\"g\":179,\"b\":179,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}},\"wordmark_bg\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"is_dark\":false},\"fonts\":{\"heading\":\"heavy_sans\",\"body\":\"sans\"}},\"logoPalette\":{\"Vibrant\":{\"rgb\":[92,70,172],\"population\":94},\"DarkVibrant\":{\"rgb\":[49,40,85],\"population\":306},\"LightVibrant\":{\"rgb\":[148,204,236],\"population\":1},\"Muted\":{\"rgb\":[84,84,164],\"population\":2},\"DarkMuted\":{\"rgb\":[74,83,114],\"population\":24},\"LightMuted\":{\"rgb\":[179,184,190],\"population\":343}}},\"confirmedLogin\":false,\"hide_intro_popup\":false,\"block_auto_login\":false,\"domainInfo\":{\"isSubstack\":false,\"customDomain\":\"www.softwareatscale.dev\"},\"experimentFeatures\":{},\"experimentExposures\":{},\"siteConfigs\":{\"score_upsell_email\":\"control\",\"first_chat_email_enabled\":true,\"new_commenter_approval\":false,\"pub_update_opennode_api_key\":false,\"notes_video_max_duration_minutes\":15,\"show_content_label_age_gating_in_feed\":false,\"zendesk_automation_cancellations\":false,\"mfa_action_box_enabled\":false,\"publication_max_bylines\":35,\"no_contest_charge_disputes\":false,\"scheduled_post_share_assets_enabled\":false,\"publication_tabs_reorder\":false,\"comp_expiry_email_new_copy\":\"NONE\",\"free_unlock_required\":false,\"enable_section_group_pages\":false,\"enable_post_summarization\":false,\"live_stream_host_warning_message\":\"\",\"bitcoin_enabled\":false,\"minimum_ios_os_version\":\"17.0.0\",\"show_entire_square_image\":false,\"hide_subscriber_count\":false,\"fit_in_live_stream_player\":false,\"publication_author_display_override\":\"\",\"enable_saved_segments\":true,\"generate_pdf_tax_report\":false,\"hide_post_sidebar\":false,\"show_generic_post_importer\":false,\"enable_pledges_modal\":true,\"community_automod_non_english_enabled\":true,\"audio_clips_enabled\":false,\"enable_videos_page\":false,\"exempt_from_gtm_filter\":false,\"group_sections_and_podcasts_in_menu\":false,\"boost_optin_modal_enabled\":true,\"standards_and_enforcement_features_enabled\":false,\"pub_creation_captcha_behavior\":\"risky_pubs_or_rate_limit\",\"post_blogspot_importer\":false,\"enable_high_res_background_uploading\":false,\"pub_tts_override\":\"default\",\"disable_monthly_subscriptions\":false,\"skip_welcome_email\":false,\"chat_reader_thread_notification_default\":false,\"scheduled_pinned_posts\":false,\"disable_redirect_outbound_utm_params\":false,\"reader_gift_referrals_enabled\":true,\"dont_show_guest_byline\":false,\"like_comments_enabled\":true,\"enable_theme_editor_deferred_saves\":false,\"enable_sponsorship_campaigns_advanced\":false,\"temporal_livestream_ended_draft\":true,\"enable_pangram_ai_detection\":true,\"fallback_to_archive_search_on_section_pages\":false,\"livekit_track_egress_custom_base_url\":\"http://livekit-egress-custom-recorder-participant-test.s3-website-us-east-1.amazonaws.com\",\"welcome_screen_blurb_override\":\"\",\"like_posts_enabled\":true,\"twitter_player_card_enabled\":true,\"feed_promoted_user\":false,\"show_note_stats_for_all_notes\":false,\"section_specific_csv_imports_enabled\":false,\"disable_podcast_feed_description_cta\":false,\"bypass_profile_substack_logo_detection\":false,\"use_preloaded_player_sources\":false,\"list_pruning_enabled\":false,\"facebook_connect\":false,\"opt_in_to_sections_during_subscribe\":false,\"underlined_colored_links\":false,\"enable_efficient_digest_embed\":false,\"enable_aligned_images\":false,\"max_image_upload_mb\":64,\"threads_suggested_ios_version\":null,\"pledges_disabled\":false,\"threads_minimum_ios_version\":812,\"hide_podcast_email_setup_link\":false,\"subscribe_captcha_behavior\":\"default\",\"publication_ban_sample_rate\":0,\"custom_themes_substack_subscribe_modal\":false,\"ios_post_share_assets_screenshot_trigger\":\"control\",\"opt_in_to_sections_during_subscribe_include_main_pub_newsletter\":false,\"continue_support_cta_in_newsletter_emails\":false,\"bloomberg_syndication_enabled\":false,\"allow_document_freeze\":false,\"test_age_gate_user\":false,\"podcast_main_feed_is_firehose\":false,\"pub_app_incentive_gift\":\"\",\"no_embed_redirect\":false,\"customized_email_from_name_for_new_follow_emails\":\"treatment\",\"spotify_open_access_sandbox_mode\":false,\"disable_custom_nav_menu\":false,\"enable_livestream_name_cards\":false,\"fullstory_enabled\":false,\"chat_reply_poll_interval\":3,\"thefp_enable_email_upsell_banner\":false,\"force_pub_links_to_use_subdomain\":false,\"always_show_cookie_banner\":false,\"hide_media_download_option\":false,\"hide_post_restacks\":false,\"feed_item_source_debug_mode\":false,\"twitter_embed_video_playback_enabled\":false,\"publication_homepage_title_display_override\":\"\",\"enable_email_automation_stats\":false,\"post_preview_highlight_byline\":false,\"4k_video\":false,\"enable_islands_section_intent_screen\":false,\"post_metering_enabled\":false,\"notifications_disabled\":\"\",\"cross_post_notification_threshold\":1000,\"facebook_connect_prod_app\":true,\"force_into_pymk_ranking\":false,\"minimum_android_version\":756,\"group_size_change_notification_email_enabled\":false,\"enable_transcription_translations\":false,\"use_og_image_as_twitter_image_for_post_previews\":false,\"always_use_podcast_channel_art_as_episode_art_in_rss\":false,\"seo_tier_override\":\"NONE\",\"editor_role_enabled\":false,\"no_follow_links\":false,\"publisher_api_enabled\":false,\"zendesk_support_priority\":\"default\",\"enable_post_clips_stats\":false,\"enable_subscriber_referrals_awards\":true,\"ios_profile_themes_feed_permalink_enabled\":false,\"reader_survey_during_subscribe_enabled\":true,\"use_publication_language_for_transcription\":false,\"show_substack_funded_gifts_tooltip\":true,\"disable_ai_transcription\":false,\"per_pub_notification_settings\":false,\"live_stream_creation_enabled\":true,\"disable_card_element_in_europe\":false,\"web_growth_item_promotion_threshold\":0,\"bundle_subscribe_enabled\":false,\"web_vitals_sample_rate\":0,\"allow_live_stream_auto_takedown\":\"true\",\"mobile_publication_attachments_enabled\":false,\"enable_design_templates_events_page\":false,\"enable_survey_response_crm_filters\":false,\"disable_personal_substack_initialization\":false,\"section_specific_welcome_pages\":false,\"local_payment_methods\":\"control\",\"publisher_api_cancel_comp\":false,\"posts_in_rss_feed\":20,\"publisher_dashboard_section_selector\":false,\"reader_surveys_platform_question_order\":\"36,1,4,2,3,5,6,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35\",\"login_guard_app_link_in_email\":true,\"community_moderators_enabled\":false,\"enable_custom_theme\":false,\"monthly_sub_is_one_off\":false,\"unread_notes_activity_digest\":\"control\",\"display_cookie_settings\":false,\"welcome_page_query_params\":false,\"enable_free_podcast_urls\":false,\"comp_expiry_emails_disabled\":false,\"enable_description_on_polls\":false,\"use_microlink_for_instagram_embeds\":false,\"free_signup_confirmation_behavior\":\"with_email_validation\",\"ios_post_stats_for_admins\":false,\"enable_design_templates\":false,\"use_livestream_post_media_composition\":true,\"section_specific_preambles\":false,\"pub_export_temp_disable\":false,\"show_menu_on_posts\":false,\"ios_post_subscribe_web_routing\":true,\"enable_pangram_detection_settings\":false,\"freewalls_enabled_web\":false,\"app_onboarding_survey_email\":false,\"republishing_enabled\":false,\"app_mode\":false,\"show_phone_banner\":false,\"minimum_ios_version\":2200,\"enable_author_pages\":false,\"enable_decagon_chat\":true,\"first_month_upsell\":\"control\",\"new_user_checklist_enabled\":\"use_follower_count\",\"disable_high_res_recording\":false,\"latex_upgraded_inline\":false,\"anonymous_post_metering_enabled\":false,\"rss_verification_code\":\"\",\"notification_post_emails\":\"experiment\",\"chat_suppress_contributor_push_option_enabled\":false,\"enable_subscriber_tags\":true,\"media_feed_app_prepend_inbox_limit\":0,\"export_hooks_enabled\":false,\"audio_encoding_bitrate\":null,\"extra_seats_coupon_type\":false,\"post_subdomain_universal_links\":false,\"post_import_max_file_size\":26214400,\"feed_promoted_video_publication\":false,\"livekit_reconnect_slate_url\":\"https://mux-livestream-assets.s3.us-east-1.amazonaws.com/custom-disconnect-slate-tall.png\",\"exclude_from_pymk_suggestions\":false,\"publication_ranking_variant\":\"experiment\",\"disable_annual_subscriptions\":false,\"hack_jane_manchun_wong\":true,\"android_enable_auto_gain_control\":true,\"allow_coupons_on_upgrade\":false,\"test_au_age_gate_user\":false,\"pub_auto_moderation_enabled\":false,\"disable_live_stream_ai_trimming_by_default\":false,\"disable_deletion\":false,\"livekit_egress_custom_base_url\":\"http://livekit-egress-custom-recorder.s3-website-us-east-1.amazonaws.com\",\"clip_focused_video_upload_flow\":false,\"live_stream_max_guest_users\":2,\"android_upgrade_alert_dialog_reincarnated\":true,\"enable_video_seo_data\":false,\"can_reimport_unsubscribed_users_with_2x_optin\":false,\"founding_upgrade_during_gift_disabled\":false,\"thefp_enable_section_groups\":false,\"review_incoming_email\":\"default\",\"enable_founding_gifts\":false,\"enable_sponsorship_campaigns\":false,\"thefp_forum_polls_enabled\":false,\"thefp_enable_embed_media_links\":false,\"sort_modal_search_results\":false,\"default_thumbnail_time\":10,\"load_test_unichat\":false,\"live_stream_head_alignment_guide\":false,\"free_press_combo_subscribe_flow_enabled\":false,\"enable_code_embed_toc\":false,\"enable_publication_tax_settings\":false,\"gifts_from_substack_feature_available\":true,\"disable_ai_clips\":false,\"enable_elevenlabs_voiceovers\":false,\"thefp_enable_transcripts\":false,\"show_simple_post_editor\":false,\"instacart_integration_enabled\":false,\"enable_publication_podcasts_page\":false,\"enable_direct_message_request_bypass\":false,\"enable_apple_news_sync\":false,\"live_stream_in_trending_topic_overrides\":\"\",\"free_press_newsletter_promo_enabled\":false,\"account_based_post_metering_enabled\":false,\"disable_live_stream_reactions\":false,\"instacart_partner_id\":\"\",\"enable_adhoc_email_scheduling\":true,\"media_feed_prepend_inbox_limit\":35,\"welcome_page_no_opt_out\":false,\"enable_caption_translation_tools\":false,\"section_specific_postscripts\":false,\"edit_profile_theme_colors\":false,\"disable_clipping_for_readers\":false,\"enable_apple_podcast_delivery_self_serve_pem_files\":false,\"enable_press_kit_preview_modal\":false,\"enable_rnc_page\":true,\"live_stream_video_degradation_preference\":\"maintainFramerate\",\"enable_high_follower_dm\":true,\"pause_app_badges\":false,\"profile_feed_expanded_inventory\":false,\"livekit_mux_latency_mode\":\"low\",\"feed_juiced_user\":0,\"global_iap_plan_setup_countries\":\"AE,AG,AL,AM,AT,AU,BA,BG,BH,BJ,BN,BS,BT,BW,CH,CI,CR,CY,CZ,DK,DZ,EC,EE,ET,FI,GM,GT,GY,HK,HR,HU,ID,IL,IN,IS,JM,JO,KE,KW,LC,LI,LK,LT,LU,LV,MA,MD,MG,MK,MN,MT,MU,MX,MY,MZ,NA,NO,NZ,OM,PA,PE,PH,PK,QA,RO,RS,RW,SE,SG,SI,SK,SN,SV,TH,TN,TR,TT,TW,TZ,UZ,VN,ZA\",\"enable_collapsed_content\":false,\"enable_polymarket_expandable_embeds\":true,\"publisher_dashboard_group_subs_sidebar\":false,\"use_advanced_commerce_api_for_iap\":false,\"skip_free_preview_language_in_podcast_notes\":false,\"larger_wordmark_on_publication_homepage\":false,\"enable_mobile_stats_for_admins\":false,\"enable_livestream_youtube_simulcast\":false,\"enable_persona_sandbox_environment\":false,\"allowed_email_domains\":\"mozmail.com\",\"create_nav_item_from_tag\":false,\"bypass_single_unlock_token_limit\":false,\"enable_code_embeds\":false,\"add_section_and_tag_metadata\":false,\"daily_promoted_notes_enabled\":true,\"enable_segments_ui_revamp\":true,\"enable_islands_cms\":false,\"enable_livestream_combined_stats\":false,\"redirect_to_pub_after_welcome_signup\":false,\"chartbeat_video_enabled\":false,\"enable_drip_campaigns\":false,\"enable_filtered_posts_on_homepage\":false,\"post_management_search_engine\":\"elasticsearch\",\"new_bestseller_leaderboard_feed_item_enabled\":false,\"feed_main_disabled\":false,\"enable_account_settings_revamp\":false,\"thefp_enable_fp_recirc_block\":false,\"top_search_variant\":\"control\",\"enable_debug_logs_ios\":false,\"show_pub_content_on_profile_for_pub_id\":0,\"show_pub_content_on_profile\":false,\"livekit_track_egress\":true,\"onboarding_suggestions_search\":\"experiment\",\"feed_tuner_enabled\":false,\"livekit_mux_latency_mode_rtmp\":\"low\",\"google_login_enabled\":false,\"livekit_high_quality_egress\":false,\"iap_announcement_blog_url\":\"\",\"disable_code_embed_chrome\":false,\"ios_livestream_feedback\":false,\"founding_plan_upgrade_warning\":false,\"ios_mediaplayer_reply_bar_v2\":false,\"enable_notification_email_batching\":true,\"note_author_poll_results_visible_by_default\":true,\"ios_hide_portal_tab_bar\":false,\"follow_upsell_rollout_percentage\":30,\"gift_article_enabled\":false,\"live_stream_invite_ttl_seconds\":900000,\"include_founding_plans_coupon_option\":false,\"thefp_enable_cancellation_discount_offer\":false,\"android_enable_edit_profile_theme\":false,\"disable_audio_enhancement\":false,\"pub_search_variant\":\"control\",\"ignore_video_in_notes_length_limit\":false,\"substack_gtm_enabled\":false,\"allow_long_videos\":true,\"enable_automation_filters\":false,\"use_enhanced_video_embed_player\":true,\"founding_upgrade_preview\":false,\"enable_viewing_all_livestream_viewers\":false,\"send_subscription_canceled_email\":false,\"enable_clip_prompt_variant_filtering\":true,\"chartbeat_enabled\":false,\"dpn_ranking_enabled\":true,\"enable_custom_email_css\":false,\"thefp_enable_shows_page\":false,\"enable_apple_podcast_auto_publish\":false,\"linkedin_profile_search_enabled\":false,\"publication_has_own_app\":false,\"suggested_minimum_ios_version\":0,\"thebulwark_enable_footer_actions\":false,\"enable_posts_to_segments\":true,\"trending_topics_module_long_term_experiment\":\"control\",\"phone_verification_fallback_to_twilio\":false,\"enable_suggested_searches\":true,\"enable_subscription_notification_email_batching\":true,\"feed_controls\":\"control\",\"thefp_enable_plans_modal_v2\":false,\"a24_redemption_link\":\"\",\"design_templates_launch_phase\":\"off\",\"post_custom_email_from_name_enabled\":false,\"ios_live_stream_auto_gain_enabled\":true,\"enable_publication_tts_player\":false,\"enable_group_direct_messages\":true,\"enable_notes_admins\":false,\"thefp_show_pub_app_callout_on_post\":false,\"search_ranker_variant\":\"experiment\",\"forced_featured_topic_id\":\"\",\"ios_audio_captions_disabled\":false,\"web_podcasts_tab\":false,\"related_posts_enabled\":false,\"thefp_enable_homepage_secondary_nav\":false,\"ios_live_stream_pip_dismiss_v4\":\"control\",\"reply_rate_limit_max_distinct_users_daily\":110,\"web_post_metering_enabled\":true,\"search_retrieval_variant\":\"experiment\",\"mobile_user_attachments_enabled\":false,\"web_profile_publication_recommendations_carousel\":true,\"publisher_banner\":\"\",\"enable_sponsorship_profile\":false,\"gift_article_button_enabled\":false,\"reply_rate_limit_max_distinct_users_monthly\":600,\"desktop_live_stream_screen_share_audio_enabled\":false,\"creator_match\":true,\"ai_image_generation_enabled\":true,\"ios_default_coupon_enabled\":true,\"automod_admin_hidden_comments_tab\":true,\"video_clip_suggestions_enabled\":false,\"publisher_dashboard_home_layout_experiment\":\"control\",\"android_polymarket_embed_search\":false,\"audience_section_editor_help_link\":\"https://support.substack.com/hc/en-us/articles/50558240901268-How-do-I-add-audience-specific-content-to-a-post-on-Substack\",\"portal_ranking_variant\":\"experiment\",\"permalink_reply_ranking_variant\":\"experiment\",\"allow_feed_category_filtering\":false,\"enable_dynamic_content\":true,\"enable_thumbnail_editor_media_settings_integration\":false,\"subscriber_profile_survey_responses_enabled\":true,\"private_live_streaming_enabled\":true,\"desktop_live_stream_safe_framing\":0.8,\"translate_posts_on_standalone_sites\":true,\"bundle_subscribe_default_to_regular_plan\":false,\"enable_standardized_sponsorship_formats\":false,\"enable_pencraft_signin_form\":true,\"thefp_enable_secondary_section_redesign\":false,\"bignight_page_enabled\":true,\"thefp_paywall_annual_plans_experiment\":\"control\",\"enable_launch_assistant\":false,\"enable_youtube_exploration\":true,\"publisher_dashboard_nav_refresh\":false,\"thefp_enable_bundle_post_follow\":false,\"web_notes_tiptap_v3_experiment\":\"treatment_tiptap_v3\",\"recipe_rich_text_enabled\":true,\"publisher_dashboard_nav_refresh_experiment\":\"treatment\",\"chat_channel_creation_enabled\":false,\"web_onboarding_content_type_preferences\":\"experiment\",\"web_mobile_asset_handoff\":false},\"trackingTokens\":{\"fbPixelId\":null,\"gaPixelId\":\"G-HR35NWP3CE\",\"twitterPixelId\":\"o6rzq\",\"parselyPixelId\":null,\"googleTagManagerToken\":\"G-HR35NWP3CE\",\"googleAnalytics4Token\":null,\"twitterPixelSignupEventId\":null,\"twitterPixelSubscribeEventId\":null},\"publicationSettings\":{\"block_ai_crawlers\":false,\"credit_token_enabled\":false,\"custom_tos_and_privacy\":false,\"did_identity\":null,\"disable_optimistic_bank_payments\":false,\"display_welcome_page_details\":true,\"payment_pledges_enabled\":true,\"enable_drop_caps\":false,\"enable_post_page_conversion\":true,\"enable_prev_next_nav\":true,\"enable_restacking\":true,\"founding_group_subscriptions_enabled\":false,\"gifts_from_substack_disabled\":false,\"google_analytics_4_token\":null,\"group_sections_and_podcasts_in_menu_enabled\":false,\"live_stream_homepage_visibility\":\"contributorsAndAdmins\",\"live_stream_homepage_style\":\"autoPlay\",\"live_stream_replay_enabled\":true,\"medium_length_description\":\"\",\"notes_feed_enabled\":true,\"paywall_unlock_tokens\":false,\"post_preview_crop_gravity\":\"center\",\"post_preview_radius\":\"xs\",\"reader_referrals_enabled\":false,\"reader_referrals_leaderboard_enabled\":false,\"seen_coming_soon_explainer\":false,\"seen_google_analytics_migration_modal\":true,\"local_currency_modal_seen\":false,\"local_payment_methods_modal_seen\":false,\"twitter_pixel_signup_event_id\":null,\"twitter_pixel_subscribe_event_id\":null,\"use_inclusive_sales_tax\":false,\"use_local_currency\":false,\"welcome_page_opt_out_text\":\"No thanks\",\"cookie_settings\":\"\",\"show_restacks_below_posts\":true,\"holiday_gifting_post_header\":false,\"homepage_message_text\":\"\",\"homepage_message_link\":\"\",\"about_us_author_ids\":\"\",\"archived_section_ids\":\"\",\"column_section_ids\":\"\",\"fp_primary_column_section_ids\":\"\",\"event_section_ids\":\"\",\"podcasts_metadata\":\"\",\"video_section_ids\":\"\",\"post_metering_enabled\":false,\"account_based_post_metering_enabled\":false,\"use_custom_theme\":false,\"reply_rules\":null,\"automatic_moderation_enabled\":true,\"auto_translate_enabled\":true,\"additional_post_languages_enabled\":false,\"high_res_recording_beta\":false},\"publicationUserSettings\":null,\"userSettings\":{\"user_id\":null,\"activity_likes_enabled\":true,\"dashboard_nav_refresh_enabled\":false,\"is_guest_post_enabled\":true,\"invite_friends_nux_dismissed_at\":null,\"suggestions_feed_item_last_shown_at\":null,\"last_notification_alert_shown_at\":null,\"disable_reply_hiding\":false,\"newest_seen_chat_item_published_at\":null,\"explicitContentEnabled\":false,\"contactMatchingEnabled\":false,\"messageRequestLevel\":\"everyone\",\"liveStreamAcceptableInviteLevel\":\"everyone\",\"liveStreamAcceptableChatLevel\":\"everyone\",\"creditTokensTreatmentExposed\":false,\"appBadgeIncludesChat\":false,\"autoPlayVideo\":true,\"autoAdvanceVideo\":true,\"smart_delivery_enabled\":false,\"chatbotTermsLastAcceptedAt\":null,\"has_seen_notes_post_app_upsell\":false,\"first_note_id\":null,\"show_concurrent_live_stream_viewers\":false,\"edit_profile_feed_item_dismissed_at\":null,\"mobile_permalink_app_upsell_seen_at\":null,\"new_user_checklist_enabled\":false,\"has_seen_youtube_shorts_auto_publish_announcement\":false,\"has_seen_publish_youtube_connect_upsell\":false,\"notificationQualityFilterEnabled\":true,\"hasSeenOnboardingNewslettersScreen\":false,\"bestsellerBadgeEnabled\":true,\"hasSelfIdentifiedAsCreator\":false,\"autoTranslateEnabled\":true,\"autoTranslateBlocklist\":[],\"fineTuneFeedItemSeenAt\":null,\"fineTuneFeedItemDismissedAt\":null,\"aiGeneratedTextMode\":\"show\",\"feedVideoPreference\":5,\"feedAudioPreference\":5,\"feedLongerPostsPreference\":5,\"feedChatPreference\":5},\"mux_env_key\":\"u42pci814i6011qg3segrcpp9\",\"persona_environment_id\":\"env_o1Lbk4JhpY4PmvNkwaBdYwe5Fzkt\",\"sentry_environment\":\"production\",\"launchWelcomePage\":false,\"pendingInviteForActiveLiveStream\":null,\"isEligibleForLiveStreamCreation\":true,\"webviewPlatform\":null,\"noIndex\":false,\"post\":{\"audience\":\"everyone\",\"audience_before_archived\":null,\"canonical_url\":\"https://www.softwareatscale.dev/p/thoughts-on-api-reliability\",\"default_comment_sort\":null,\"editor_v2\":false,\"exempt_from_archive_paywall\":false,\"free_unlock_required\":false,\"id\":68658421,\"podcast_art_url\":null,\"podcast_duration\":null,\"podcast_preview_upload_id\":null,\"podcast_upload_id\":null,\"podcast_url\":null,\"post_date\":\"2022-09-30T17:17:52.150Z\",\"updated_at\":\"2023-02-12T16:48:27.093Z\",\"publication_id\":55725,\"search_engine_description\":null,\"search_engine_title\":null,\"section_id\":null,\"should_send_free_preview\":false,\"show_guest_bios\":true,\"slug\":\"thoughts-on-api-reliability\",\"social_title\":null,\"subtitle\":\"\",\"teaser_post_eligible\":true,\"title\":\"Thoughts on API Reliability\",\"type\":\"newsletter\",\"video_upload_id\":null,\"write_comment_permissions\":\"everyone\",\"meter_type\":\"none\",\"live_stream_id\":null,\"is_published\":true,\"restacks\":0,\"reactions\":{\"\u2764\":5},\"top_exclusions\":[],\"pins\":[],\"section_pins\":[],\"has_shareable_clips\":false,\"previous_post_slug\":\"software-at-scale-50-redefining-labor\",\"next_post_slug\":\"software-at-scale-51-usage-based\",\"cover_image\":\"https://substackcdn.com/image/fetch/$s_!MwBD!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png\",\"cover_image_is_square\":false,\"cover_image_is_explicit\":false,\"videoUpload\":null,\"podcastFields\":null,\"podcastUpload\":null,\"podcastPreviewUpload\":null,\"voiceover_upload_id\":null,\"voiceoverUpload\":null,\"has_voiceover\":false,\"description\":\"Building and maintaining a public API is a unique software engineering challenge.\",\"body_html\":\"<p>Building and maintaining a public API is a unique software engineering challenge. We have to consider (roughly prioritized):</p><ul><li><p>Security - exposing a service on the public internet has a variety of security implications - we want authentication/authorization as airtight as possible.</p></li><li><p>Reliability - the API is not useful if it\u2019s down often.</p></li><li><p>Ease of use - it should be as frictionless as possible for developers to work with the API. Authentication flows, pagination, documentation should be as smooth and consistent as possible.</p></li><li><p>Stability - public API callers cannot change in lockstep with the internal codebase, so long term stability guarantees should be thought through.</p></li><li><p>Internal Developer Velocity - it should be simple for other developers in the organization to add new endpoints.</p></li></ul><p>This is a vast and complex area, so I want to focus on one topic - API reliability. Once we want to consider our API<a href=\\\"https://en.wikipedia.org/wiki/Software_release_life_cycle#General_availability_(GA)\\\"> generally available</a> for a non-trivial set of users, we need to ensure it will stay reliable. In this post, we discuss common API reliability risks, mitigations, and a framework to keep our APIs reliable for the long term.</p><div class=\\\"subscription-widget-wrap-editor\\\" data-attrs=\\\"{&quot;url&quot;:&quot;https://www.softwareatscale.dev/subscribe?&quot;,&quot;text&quot;:&quot;Subscribe&quot;,&quot;language&quot;:&quot;en&quot;}\\\" data-component-name=\\\"SubscribeWidgetToDOM\\\"><div class=\\\"subscription-widget show-subscribe\\\"><div class=\\\"preamble\\\"><p class=\\\"cta-caption\\\">Thanks for reading Software at Scale! Subscribe for free to receive new posts and support my work.</p></div><form class=\\\"subscription-widget-subscribe\\\"><input type=\\\"email\\\" class=\\\"email-input\\\" name=\\\"email\\\" placeholder=\\\"Type your email\u2026\\\" tabindex=\\\"-1\\\"><input type=\\\"submit\\\" class=\\\"button primary\\\" value=\\\"Subscribe\\\"><div class=\\\"fake-input-wrapper\\\"><div class=\\\"fake-input\\\"></div><div class=\\\"fake-button\\\"></div></div></form></div></div><h1><strong>Reliability Risk Model</strong></h1><p>Just like <a href=\\\"https://slsa.dev/spec/v0.1/threats\\\">security threat models</a>, it\u2019s useful to think about all the risks and potential scenarios that our API should be able to defend against.&nbsp;</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!MwBD!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!MwBD!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png 424w, https://substackcdn.com/image/fetch/$s_!MwBD!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png 848w, https://substackcdn.com/image/fetch/$s_!MwBD!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png 1272w, https://substackcdn.com/image/fetch/$s_!MwBD!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!MwBD!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png\\\" width=\\\"1456\\\" height=\\\"771\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:771,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:null,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:null,&quot;href&quot;:null,&quot;belowTheFold&quot;:false,&quot;topImage&quot;:true,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!MwBD!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png 424w, https://substackcdn.com/image/fetch/$s_!MwBD!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png 848w, https://substackcdn.com/image/fetch/$s_!MwBD!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png 1272w, https://substackcdn.com/image/fetch/$s_!MwBD!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F9f337830-88cf-44dd-ae89-583d8be994f1_1600x847.png 1456w\\\" sizes=\\\"100vw\\\" fetchpriority=\\\"high\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><p>Breaking down risks into four categories:</p><p><strong>Development risks</strong>: New faulty application code or infrastructure as code changes can lead to reliability issues. Additionally, application architecture and engineering practices also dictate ongoing reliability. For example, the right RPC library will enforce request timeouts and prevent overloads.</p><p><strong>Deployment risks</strong>: The set of issues that prevent safe deployments. For example, the time it takes for a team to roll back a bad code change, and the likelihood that an on-call is empowered and trained to take a quick set of mitigations. There\u2019s also the risk impact around faulty changes that can be tuned. For example, a faulty change might lead to a small availability impact to a canary environment, or a global outage, depending on our infrastructure and processes.</p><p><strong>Abuse and load risks</strong>: APIs can go down even if there\u2019s no changes going out due to changes in client behavior. Bad actors, and good but misconfigured clients are almost indistinguishable in most cases - and there\u2019s a set of risks around abusive request patterns that have to be considered.</p><p><strong>Infrastructure/dependency/operator risks: </strong>APIs are hosted somewhere, whether it may be a single machine in an office, or on thousands of cloud instances, and there\u2019s physical infrastructure that might have issues. Alternatively, software infrastructure that our API depends on, like a caching layer, might fail and cause cascading issues.</p><p>Examples from my past experience/conversations:</p><ul><li><p>Bad code deployments - new bugs/misconfigurations that take our API down.</p></li><li><p>Scanners that check for known vulnerabilities of common deployments. For example, Wordpress scanners that check if our admin site is publicly available. These might cause a large request volume.</p></li><li><p>Security scanners deployed by the company\u2019s security team that perform the same operations as above, but are less concerned about getting rate limited/banned. So the request throughput may even be higher.</p></li><li><p>Cron jobs that make large bursts of requests on the hour that a service isn\u2019t provisioned for.</p></li><li><p>Large bursts of requests when a particular resource goes viral (<a href=\\\"https://news.ycombinator.com/item?id=20147951\\\">the HackerNews hug of death</a>).</p></li><li><p>Large customers attempting a programmatic data-export/migration that cause database load due to<a href=\\\"https://stackoverflow.com/questions/49857507/understanding-hot-keys-in-dynamo-db\\\"> hot keys</a>/contention.</p></li><li><p>Extremely large customers performing unbounded API operations.</p><ul><li><p>At one of my previous jobs, we had a concept of users and teams, and teams were generally not more than hundreds of users. But we signed up with a university, and they added forty thousand students to a one team. None of our features and APIs were designed for that sized team.</p></li></ul></li><li><p>Internal services or batch jobs that overload a service that serves external requests.</p></li><li><p>A single bad host in a small deployment that causes a large availability hit due to error amplification. </p><ul><li><p>For example, successful RPCs to the service generally take hundreds of milliseconds, but failures return almost instantaneously. Therefore, more requests get routed to the bad host since it has more capacity.</p></li></ul></li><li><p>The website is down but the on-call engineer does not feel comfortable rolling back and is waiting for a 90 minute CI build for a forward fix to go out.</p></li></ul><p>It\u2019s worth coming up with a set of risks for our API, as it helps guide a set of principles to operate on and prioritize mitigations.</p><h1><strong>Mitigations</strong></h1><p>These are some common mitigations for various reliability risks.</p><h2>Development Risks</h2><h3><strong>Bounded operations</strong></h3><p>Our first observation is that most API defense measures like rate limiting, request quotas, and timeouts take place per operation. For example, rate limits are most often set up on requests per second. Therefore, we never want a single API operation to do unbounded work. Examples:</p><ul><li><p>List operations should always be paginated.</p><ul><li><p>Let\u2019s say we have a ListUsers API call - this might be very cheap for small customers, but expensive for large customers - there\u2019s increased database load, network bandwidth and serialization cost for the same operation. We have to tune the rate limit for our largest customers, which increases our developer friction.</p></li><li><p>In addition, the number of returned items have to be capped. Otherwise, pagination is not useful.</p></li></ul></li><li><p>Set operations that need to operate on a list of items should always take a list of IDs.</p><ul><li><p>And limit the number of items that can be operated on at a time.</p></li></ul></li></ul><p>This is likely the highest ROI principle to keep in mind - it\u2019s easy to add more rate limiting infrastructure after the fact, but it\u2019s hard to convert an unpaginated operation into a paginated one, because users will depend on older behavior.</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!Tna_!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!Tna_!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png 424w, https://substackcdn.com/image/fetch/$s_!Tna_!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png 848w, https://substackcdn.com/image/fetch/$s_!Tna_!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png 1272w, https://substackcdn.com/image/fetch/$s_!Tna_!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!Tna_!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png\\\" width=\\\"1456\\\" height=\\\"296\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/c7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:296,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:null,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:null,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!Tna_!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png 424w, https://substackcdn.com/image/fetch/$s_!Tna_!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png 848w, https://substackcdn.com/image/fetch/$s_!Tna_!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png 1272w, https://substackcdn.com/image/fetch/$s_!Tna_!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fc7efe102-64b5-4b36-8257-993ebb00295f_1456x296.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a><figcaption class=\\\"image-caption\\\">An example email from a vendor adding pagination to their APIs</figcaption></figure></div><h3><strong>Request timeouts</strong></h3><p>Low request timeouts are primarily an expectation setting mechanism and guardrail - if our API operations take too long, timeouts inform users that the operations have failed. If this happens sufficiently often, ideally, the API operator finds out about it through observability tools or support tickets, and can change the operation to do less work. It forces us to time-bound API operations. Ideally, the timeout also aborts the request so that the operation does not do any more work, but that\u2019s often hard to implement correctly. For example, it\u2019s really tricky to abort an<a href=\\\"https://stackoverflow.com/questions/69545993/express-js-abort-request-on-timeout?rq=1\\\"> ongoing request in</a> Node. These little things are deeply considered in<a href=\\\"https://grpc.io/blog/deadlines/\\\"> RPC libraries</a> and implemented best by them - don\u2019t roll your own request timeouts!</p><p>It\u2019s very easy to bump up request timeouts as a cheap mitigation to a user perceived bug, but resist this temptation/schedule work to clean this up.</p><h2>Deployment Risks</h2><h3>Change management&nbsp;</h3><p>Once a faulty change has been rolled out, there needs to be a clear set of owners or DRIs who are trained or feel empowered to mitigate the issue. For business critical services, consider having an on-call rotation of engineers who are trained and equipped to drive mitigations.</p><h4>Emergency rollbacks</h4><p>For stateless services, a rollback is almost always the right approach to mitigate code change related outages. Rollbacks don\u2019t need to go through the full test suite, as long as we can guarantee that service is being deployed at a change that was previously tested and deployed - which greatly helps with mitigation time.</p><h3>API validation tests</h3><p>A staging deployment that has no reliability bearings on the production deployment is a great test bed for risky changes and continuous <a href=\\\"https://docs.datadoghq.com/synthetics/\\\">API validation tests</a>.</p><h3>Canary deployments / Staged rollouts</h3><p>After a certain scale, it\u2019s worth considering a separate canary deployment which receives code changes before the rest of the fleet. This helps prevent large outages if a faulty code change hasn\u2019t been tested well enough in other environments. A canary deployment has the advantage of being most similar to the production environment, so it helps testing changes in a more realistic scenario. More sophisticated setups may benefit from <a href=\\\"https://netflixtechblog.com/automated-canary-analysis-at-netflix-with-kayenta-3260bc7acc69\\\">Canary Analysis</a>.</p><p>A canary deployment has an unexpected benefit of being a load request<a href=\\\"https://www.softwareatscale.dev/p/why-you-should-frequently-turn-down\\\"> playground</a> and can help us understand our load limits. For example, by introducing a load balancer that automatically retries on 500s and spreads requests across canary and production deployments, we can afford to have a few requests to our canaries fail without impacting global availability. This lets us run experiments, like tuning down the number of canary replicas and see how our instances operate under load. This area is worth investing in if we run a high throughput, business critical service.</p><p>Additionally, larger, multi-region or stateful services should be deployed in stages to reduce the blast radius of faulty changes.</p><h3>Separating live and batch traffic</h3><p>With a service oriented architecture, it\u2019s very simple to overload an externally facing, critical service with an internal batch job that needs to RPC to the service. Even worse, I\u2019ve seen cases where mobile applications run end to end tests against the production service. The major downside here is that completely disabling such a workflow becomes infeasible over time, since other business critical parts of the system depend on the API.</p><p>It\u2019s worth trying to completely separate traffic tiers, or even deployments, to prevent non-critical traffic from taking down our system. The Google <a href=\\\"https://sre.google/sre-book/handling-overload/\\\">SRE book has a good notion on criticality</a>.</p><h2>Abuse and load risks</h2><h3>Token limits</h3><p>Simple limits can go a long way in helping with API reliability. For example, a limit on the number of API tokens created per user can prevent a malicious bot-farm from working around disabled/rate-limited API tokens.</p><h3><strong>Admin tools</strong></h3><p>Consider having an option to disable a misbehaving API key in an admin portal. This significantly reduces the amount of time it will take to mitigate an outage, and the on-call engineer will thank us for it. Having this ability to disable misbehaving API keys relies on having enough observability to know which clients are being the most troublesome. The simplest path is to have sampled logs on request counts, broken down by customer ID/user ID/hashed API token/token ID so we can find the offender easily.</p><h3><strong>Web Application Firewalls (WAFs)</strong></h3><p>WAFs (<a href=\\\"https://aws.amazon.com/waf/\\\">AWS</a>,<a href=\\\"https://www.cloudflare.com/learning/ddos/glossary/web-application-firewall-waf/\\\"> Cloudflare</a>) are multi-purpose, and help with basic security posture as well as coarse rate limiting.</p><p>WAFs often let us set up rate limiting rules per IP address, and some even let us set up rules on<a href=\\\"https://cloud.google.com/armor/docs/rules-language-reference\\\"> request headers/bodies</a>. They\u2019re good at preventing misconfigured clients, scanners, and other tools from hitting our service at all, so that more sophisticated rate limiting only needs to be applied at another layer with lower request volumes. Another advantage is that they\u2019re very cheap -<a href=\\\"https://aws.amazon.com/waf/pricing/\\\"> AWS charges 60 cents per million requests</a> + a flat fee, and in terms of engineering time - it\u2019s very easy to add it in a cloud console/<a href=\\\"https://en.wikipedia.org/wiki/Infrastructure_as_code\\\">IaC</a>. Finally, WAFs often have an option to manually block requests from an I.P. address, which can come in very handy during emergencies. Open source proxies like Envoy have enough configurability to enable WAF-like functionality, and may<a href=\\\"https://github.com/envoyproxy/envoy/issues/7918\\\"> support holistic in-built ones</a> some day.</p><p>A limitation of WAFs is that they only apply on small time frames, like \u201Crequest count over the past five minutes\u201D. So they won\u2019t really help with cron-jobs with a greater than five minute cadence, and they\u2019re completely useless for the first few minutes of an outage. Another issue is that many universities and corporate entities have a <a href=\\\"https://en.wikipedia.org/wiki/Network_address_translation\\\">NAT gateway</a>, which all requests through a single I.P. address. So over-aggressive WAF rules are certain to block legitimate traffic.</p><h3><strong>API Gateways</strong></h3><p>API Gateways are a nebulous topic. They provide a central point for our internal services with the external internet, for example, you can serve requests through different services for different routes on the same domain. For the purposes of reliability, they can be thought of as a layer of abstraction between the chaotic outside internet and internal systems. For example, if our setup will ever have more than a single service available over the external internet, we will have to think about the same measures - WAF, observability, timeouts, rate limits, for all such services. Whereas if all our services are protected by the gateway, then these defense measures only need to be set up once. And over time, as we might want to centralize other aspects, like authentication/authorization, and the gateway\u2019s routing logic can be updated accordingly and without needing to duplicate this logic across several services.</p><p>In practice,<a href=\\\"https://aws.amazon.com/api-gateway/\\\"> API Gateways</a> often provide<a href=\\\"https://zuplo.com/\\\"> rate limiting</a> as well, so we don\u2019t have to re-implement it in-house. They offer capabilities like<a href=\\\"https://docs.aws.amazon.com/apigateway/latest/developerguide/canary-release.html\\\"> canary deployments</a> to reduce the blast radius of code deployments. So it\u2019s worth considering whether to start an API architecture with a gateway.</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!VVKN!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!VVKN!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png 424w, https://substackcdn.com/image/fetch/$s_!VVKN!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png 848w, https://substackcdn.com/image/fetch/$s_!VVKN!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png 1272w, https://substackcdn.com/image/fetch/$s_!VVKN!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!VVKN!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png\\\" width=\\\"1456\\\" height=\\\"1037\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:1037,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:null,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:null,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!VVKN!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png 424w, https://substackcdn.com/image/fetch/$s_!VVKN!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png 848w, https://substackcdn.com/image/fetch/$s_!VVKN!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png 1272w, https://substackcdn.com/image/fetch/$s_!VVKN!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F1e095335-9d39-4323-b3dd-b2c51c155d71_1600x1140.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><h3><strong>Rate limiting</strong></h3><p>The value proposition of rate limiting is fairly straightforward. The main question of rate limiting is always whether to fail open or fail closed.</p><ul><li><p>If the rate limiter fails open - we have to consider the case where an abusive client causes the rate limiter to go down. We need to be vigilant about monitoring operational errors in the rate limiter, since it can silently be disabled, which might lead to an outage.</p></li><li><p>If it fails closed - we have to trust another infrastructure component with not degrading the availability of our API. For example, if our API has an availability SLA, then our rate limiter needs to meet or exceed that.</p><ul><li><p>See the discussion on load testing below.</p></li></ul></li></ul><h3><strong>Auto-scaling</strong></h3><p>In a nutshell, auto-scaling means dynamically changing the number of replicas for a service based on a set of signals, like CPU utilization, request queue length, etc. <a href=\\\"https://render.com/autoscaling\\\">Infrastructure platforms</a> and <a href=\\\"https://github.com/kubernetes/autoscaler\\\">orchestration frameworks</a> often provide auto-scaling out of the box.</p><p>We have to be very careful to understand our systems before deploying auto-scaling. Let\u2019s take the following barebones API architecture:</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!LrTg!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!LrTg!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png 424w, https://substackcdn.com/image/fetch/$s_!LrTg!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png 848w, https://substackcdn.com/image/fetch/$s_!LrTg!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png 1272w, https://substackcdn.com/image/fetch/$s_!LrTg!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!LrTg!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png\\\" width=\\\"1456\\\" height=\\\"425\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/e81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:425,&quot;width&quot;:1456,&quot;resizeWidth&quot;:null,&quot;bytes&quot;:null,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:null,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!LrTg!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png 424w, https://substackcdn.com/image/fetch/$s_!LrTg!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png 848w, https://substackcdn.com/image/fetch/$s_!LrTg!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png 1272w, https://substackcdn.com/image/fetch/$s_!LrTg!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2Fe81a1136-86ef-4e34-b09a-1e1b557c5597_1456x425.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><p>It might be simple to set up auto-scaling on the API service based on request latency. But that increases the number of concurrent requests to the database. If the database slows down due to increased load, that might lead to the auto-scaler adding even more API service replicas, which adds even more pressure to the database and might cause an outage. We expect this situation to happen whenever we\u2019re I/O bound. This frequently happened at<a href=\\\"https://www.softwareatscale.dev/p/software-at-scale-002-christine-dodrill#details\\\"> Heroku</a>. Reactive auto-scaling should not be seen as a solution to malicious/abusive patterns, and more as a toil reduction measure for human operators, as well as a cost-saving measure for seasonal request loads. It is not an effective measure for sudden, borderline abusive changes in request throughput.</p><p>There\u2019s also seasonal auto-scaling - preemptively scaling up instance counts during business hours or some other cadence due to a well known seasonal increases in load. This is generally more well understood, and a very reasonable approach.</p><h2>Infrastructure / dependency risks</h2><h3>Outlier host detection</h3><p>Given a sufficiently large number of instances running in a deployment, it\u2019s very likely that a few instances will not be as performant as others. Cloud providers try to automatically replace or terminate poorly performing instances, but often, we have to do that work ourselves. That\u2019s why, it may become relevant to set up <a href=\\\"https://www.datadoghq.com/blog/outlier-detection-algorithms-at-datadog/\\\">monitoring for outliers</a> and <a href=\\\"https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/outlier#:~:text=Outlier%20detection%20and%20ejection%20is,the%20healthy%20load%20balancing%20set.\\\">prevent requests</a> reaching them. A poorly performing instance in a small deployment may cause error amplification if the instance fails requests much more quickly than a well-behaved instance processing successful requests.</p><h3>Load testing</h3><p>With a sufficiently complex architecture, it\u2019s very hard to know how a system will perform under increased load. It\u2019s worth understanding through synthetic load so that we know the system\u2019s limits and can plan for future growth or seasonality of requests. Read <a href=\\\"https://itnext.io/how-i-tripled-load-in-production-for-testing-purposes-at-amazon-518df002465b\\\">this</a> for a fascinating insight into load testing at Amazon.</p><h1><strong>Conclusion</strong></h1><p>To tie this up, an API developer has an array of things to consider to strengthen a public API. A suggested framework:</p><ul><li><p>Come up with a reliability threat model to the API which produces a set of risks.</p><ul><li><p>All APIs have to defend against scanners - this is a standard risk.</p></li><li><p>APIs that allow unauthenticated use have to defend against bots and abuse much more than ones that only allow authenticated use - so some risks are unique to our business and technical context.</p></li></ul></li><li><p>Understand the impact of each risk - would the lack of defenses for the risk take down our entire site, or would the impact be limited to a single customer? These answers depend on our pre-existing software architecture. For example, if we have a<a href=\\\"https://www.techtarget.com/searchcloudcomputing/definition/single-tenancy\\\"> single tenant architecture</a>, the blast radius due to an abusive customer is much smaller.</p></li><li><p>Consider the mitigation speed for various risks, especially as they tie into your SLOs, as that will drive a discussion into remediation procedures.</p></li></ul><p>With this input, we can come up with mitigations, a software architecture, and development principles and processes to keep our APIs reliable for a long time.</p><h3>Credits</h3><p>Thanks to Allan Reyes for reading early drafts of this post and providing excellent feedback!</p><div class=\\\"subscription-widget-wrap-editor\\\" data-attrs=\\\"{&quot;url&quot;:&quot;https://www.softwareatscale.dev/subscribe?&quot;,&quot;text&quot;:&quot;Subscribe&quot;,&quot;language&quot;:&quot;en&quot;}\\\" data-component-name=\\\"SubscribeWidgetToDOM\\\"><div class=\\\"subscription-widget show-subscribe\\\"><div class=\\\"preamble\\\"><p class=\\\"cta-caption\\\">Thanks for reading Software at Scale! Subscribe for free to receive new posts.</p></div><form class=\\\"subscription-widget-subscribe\\\"><input type=\\\"email\\\" class=\\\"email-input\\\" name=\\\"email\\\" placeholder=\\\"Type your email\u2026\\\" tabindex=\\\"-1\\\"><input type=\\\"submit\\\" class=\\\"button primary\\\" value=\\\"Subscribe\\\"><div class=\\\"fake-input-wrapper\\\"><div class=\\\"fake-input\\\"></div><div class=\\\"fake-button\\\"></div></div></form></div></div>\",\"has_dynamic_content\":false,\"truncated_body_text\":\"Building and maintaining a public API is a unique software engineering challenge. We have to consider (roughly prioritized):\",\"wordcount\":2829,\"post_preview_limit\":null,\"language\":\"en\",\"postTags\":[],\"postCountryBlocks\":[],\"headlineTest\":null,\"coverImagePalette\":{\"Vibrant\":{\"rgb\":[245.55555555555557,40.92592592592589,9.44444444444443],\"population\":0},\"DarkVibrant\":{\"rgb\":[127.68888888888891,21.281481481481457,4.911111111111095],\"population\":0},\"LightVibrant\":{\"rgb\":[252,187,177],\"population\":22},\"Muted\":{\"rgb\":[154,154,154],\"population\":43},\"DarkMuted\":{\"rgb\":[147.33333333333334,24.55555555555552,5.6666666666666465],\"population\":0},\"LightMuted\":{\"rgb\":[191,191,191],\"population\":83}},\"publishedBylines\":[{\"id\":6567678,\"name\":\"Utsav Shah\",\"handle\":\"reliability\",\"photo_url\":\"https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/0f431e7e-fd06-4ab7-a5df-cda6f979e27a_1536x1536.jpeg\",\"bio\":\"Software Engineer interested in building simple, self correcting and reliable systems. Twitter: @utsav_sha\",\"profile_set_up_at\":\"2022-02-28T22:12:41.936Z\",\"reader_installed_at\":\"2022-03-09T16:36:54.472Z\",\"publicationUsers\":[{\"id\":251742,\"user_id\":6567678,\"publication_id\":55725,\"role\":\"admin\",\"public\":true,\"is_primary\":true,\"publication\":{\"id\":55725,\"name\":\"Software at Scale\",\"subdomain\":\"reliability\",\"custom_domain\":\"www.softwareatscale.dev\",\"custom_domain_optional\":false,\"hero_text\":\"Discussions on running software systems, teams, and products.\",\"logo_url\":\"https://bucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com/public/images/19cc5488-ca75-4af8-a903-6971f778093a_1000x1000.png\",\"author_id\":6567678,\"primary_user_id\":6567678,\"theme_var_background_pop\":\"#0068ef\",\"created_at\":\"2020-06-13T04:08:52.332Z\",\"email_from_name\":\"Software at Scale by Utsav\",\"copyright\":\"Utsav Shah\",\"founding_plan_name\":null,\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"disabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"magaziney\",\"is_personal_mode\":false,\"logo_url_wide\":null}},{\"id\":6993066,\"user_id\":6567678,\"publication_id\":6852092,\"role\":\"admin\",\"public\":true,\"is_primary\":false,\"publication\":{\"id\":6852092,\"name\":\"Questions Worth Asking\",\"subdomain\":\"nishkama2\",\"custom_domain\":null,\"custom_domain_optional\":false,\"hero_text\":\"\",\"logo_url\":null,\"author_id\":6567678,\"primary_user_id\":null,\"theme_var_background_pop\":\"#FF6719\",\"created_at\":\"2025-11-09T14:04:08.104Z\",\"email_from_name\":\"Utsav from Questions Worth Asking\",\"copyright\":\"Utsav Shah\",\"founding_plan_name\":null,\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"disabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"newspaper\",\"is_personal_mode\":false,\"logo_url_wide\":null}}],\"twitter_screen_name\":\"utsav_sha\",\"is_guest\":false,\"bestseller_tier\":null,\"status\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null}}],\"reaction\":null,\"reaction_count\":5,\"comment_count\":0,\"child_comment_count\":0,\"audio_items\":[{\"post_id\":68658421,\"voice_id\":\"en-US-JennyNeural\",\"audio_url\":\"https://substack-video.s3.amazonaws.com/video_upload/post/68658421/tts/e8af38d0-6d23-4c7c-bfe8-d4813cda7d24/en-US-JennyNeural.mp3\",\"type\":\"tts\",\"status\":\"completed\"}],\"is_geoblocked\":false,\"hasCashtag\":false},\"comments\":[],\"canonicalUrl\":\"https://www.softwareatscale.dev/p/thoughts-on-api-reliability\",\"inlineComments\":false,\"readerIsSearchCrawler\":false,\"ogUrl\":\"https://www.softwareatscale.dev/p/thoughts-on-api-reliability\",\"bannedFromNotes\":false,\"themeVariables\":{\"color_theme_bg_pop\":\"#1a2e05\",\"background_pop\":\"#1a2e05\",\"color_theme_bg_web\":\"#fafafa\",\"cover_bg_color\":\"#fafafa\",\"cover_bg_color_secondary\":\"#ebebeb\",\"background_pop_darken\":\"#0d1703\",\"print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_darken\":\"#0d1703\",\"color_theme_print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_20\":\"rgba(26, 46, 5, 0.2)\",\"color_theme_bg_pop_30\":\"rgba(26, 46, 5, 0.3)\",\"print_pop\":\"#1a2e05\",\"color_theme_accent\":\"#1a2e05\",\"cover_print_primary\":\"#363737\",\"cover_print_secondary\":\"#757575\",\"cover_print_tertiary\":\"#b6b6b6\",\"cover_border_color\":\"#1a2e05\",\"font_family_headings_preset\":\"'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'\",\"font_weight_headings_preset\":900,\"font_family_body_preset\":\"'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'\",\"font_weight_body_preset\":400,\"font_preset_heading\":\"heavy_sans\",\"font_preset_body\":\"sans\",\"home_hero\":\"magaziney\",\"home_posts\":\"list\",\"web_bg_color\":\"#fafafa\",\"background_contrast_1\":\"#ebebeb\",\"background_contrast_2\":\"#d8d8d8\",\"background_contrast_3\":\"#b3b3b3\",\"background_contrast_4\":\"#909090\",\"background_contrast_5\":\"#4f4f4f\",\"color_theme_bg_contrast_1\":\"#ebebeb\",\"color_theme_bg_contrast_2\":\"#d8d8d8\",\"color_theme_bg_contrast_3\":\"#b3b3b3\",\"color_theme_bg_contrast_4\":\"#909090\",\"color_theme_bg_contrast_5\":\"#4f4f4f\",\"color_theme_bg_elevated\":\"#fafafa\",\"color_theme_bg_elevated_secondary\":\"#ebebeb\",\"color_theme_bg_elevated_tertiary\":\"#d8d8d8\",\"color_theme_detail\":\"#e1e1e1\",\"background_contrast_pop\":\"rgba(26, 46, 5, 0.4)\",\"color_theme_bg_contrast_pop\":\"rgba(26, 46, 5, 0.4)\",\"theme_bg_is_dark\":\"0\",\"print_on_web_bg_color\":\"#363737\",\"print_secondary_on_web_bg_color\":\"#848585\",\"background_pop_rgb\":\"26, 46, 5\",\"color_theme_bg_pop_rgb\":\"26, 46, 5\",\"color_theme_accent_rgb\":\"26, 46, 5\"},\"recentEpisodes\":null,\"trackFrontendVisit\":true,\"activeLiveStream\":null,\"freeTrialCoupon\":null,\"defaultCoupon\":null,\"isChatActive\":false,\"publicationPageAlert\":null,\"features\":{},\"browser\":{\"name\":\"Chrome\",\"version\":\"125.0\",\"major\":\"125\"},\"showCookieBanner\":false,\"disabledCookies\":[],\"dd_env\":\"prod\",\"dd_ti\":true}")</script>
<script nonce="VqkFSleQStUMFdS2hc29nQ">window._analyticsConfig = JSON.parse("{\"properties\":{\"subdomain\":\"reliability\",\"publication_id\":55725,\"has_plans\":false,\"pub_community_enabled\":true,\"is_personal_publication\":false,\"is_subscribed\":false,\"is_free_subscribed\":false,\"is_author\":false,\"is_contributor\":false,\"is_admin\":false,\"is_founding\":false},\"adwordsAccountId\":\"AW-316245675\",\"adwordsEventSendTo\":\"Tf76CKqcyL4DEKuN5pYB\"}")</script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/lib-router.7371871e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/55188.571db0c6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/62357.a0731d37.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53754.d196682b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/33485.2686bb5d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/2237.ae8857e4.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37707.4614ce58.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9351.41564edd.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80821.b2140fcd.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44292.f3dc2b43.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70291.7aa24e85.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17499.1820773d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57975.765aca5a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/19714.2f1bdde5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48320.4e726ad5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93000.f03f24de.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44014.f579d62b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/43745.80c75c78.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/60892.92b9fc9c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/40646.258e49f2.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/87906.d493457b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/36479.ce61e623.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17706.015b0f71.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93977.f1207089.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37606.b91ab2ef.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94311.b93f6845.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89356.66161d1e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/21262.211a7337.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47925.24a01d36.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80220.03bd4c41.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71505.3dc43e33.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11100.5d148f2c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/3663.86d95e8c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18725.a5a682c3.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/86118.ff1ec843.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93519.7fe128ac.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/84828.c78aabc1.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/5075.d6e9f8ec.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/29231.a7af5911.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56725.2b32bbb6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/42336.f1c17ded.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89879.1d0af2b9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70793.951c11cb.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/39522.58e078b0.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53164.dfde36c8.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74584.94c6b3f1.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71199.a327e9f4.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63266.8bb0d59c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47358.8f5c5991.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4492.cb326b80.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/88089.680833b5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/97348.df6a856d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/52109.cf3bcc2c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56256.8a1bc5dd.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/77410.3f20fb36.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/25058.bdbc4e21.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/58483.3adddb8a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94881.0a97506f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27386.09a9082f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/10174.7f0face7.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/92610.a4fc1daf.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91214.cb10b4b8.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11244.b723f409.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99260.28b0ed6a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48445.7d5c5ba3.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/76768.9a2f2bae.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/41932.6c688f22.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/12114.979fb38e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48409.9732c6f9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/26252.4f5ae3be.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53871.2923c5c9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18494.588a06d1.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99068.bd2e1fba.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91850.79d3f415.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/49047.c7f428fb.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/16075.098bcbcf.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/28856.2eb51db5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/95396.72d1d844.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63175.bbf1c283.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74628.2e8ed950.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/32552.889dd584.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27313.c287f2d8.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/mainReactRouter.6c265246.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91507.9b9761ae.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/54770.b40bf629.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/30054.07cd629b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78118.35b48b09.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57051.f37fb930.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78191.24075434.js" charset="utf-8"></script>
<script nonce="VqkFSleQStUMFdS2hc29nQ" nomodule>
(function() {
var message = 'Your browser does not support modern JavaScript modules. Please upgrade your browser for the best experience.';
var warningDiv = document.createElement('div');
warningDiv.style.color = 'red';
warningDiv.style.padding = '10px';
warningDiv.style.margin = '10px 0';
warningDiv.style.border = '1px solid red';
warningDiv.style.backgroundColor = 'lightyellow';
warningDiv.innerText = message;
document.body.prepend(warningDiv);
})();
</script>
<!-- Datadog Analytics -->
<script nonce="VqkFSleQStUMFdS2hc29nQ">
(function(h,o,u,n,d) {
h=h[d]=h[d]||{q:[],onReady:function(c){h.q.push(c)}}
d=o.createElement(u);d.async=1;d.src=n
n=o.getElementsByTagName(u)[0];n.parentNode.insertBefore(d,n)
})(window,document,'script','https://www.datadoghq-browser-agent.com/us1/v5/datadog-rum.js','DD_RUM')
window.DD_RUM.onReady(function() {
window.DD_RUM.init({
clientToken: 'puba71073f072643721169b68f352438710',
applicationId: '2e321b35-c76b-4073-8d04-cc9a10461793',
site: 'datadoghq.com',
service: 'web',
env: window._preloads.dd_env,
version: '8b4c00716b2bc0f7ca1fded553014f7e37b5b07a',
sessionSampleRate: 1,
sessionReplaySampleRate: 10,
trackUserInteractions: window._preloads.dd_ti,
trackResources: true,
trackLongTasks: true,
defaultPrivacyLevel: 'mask-user-input',
allowedTracingUrls: [/https?:\/\/(.+\/.)?substack(cdn)?\.com/]
});
})
</script>
<!-- End Datadog Analytics -->
<!-- Cloudflare Web Analytics -->
<script defer src="https://static.cloudflareinsights.com/beacon.min.js" data-cf-beacon='{"token": "216309cffb464db4b0e02daf0b8e8060"}'></script>
<!-- End Cloudflare Web Analytics -->
<!-- Fallback tracking pixels -->
<noscript>
<img height="1" width="1" style="display:none" alt="" src="https://analytics.twitter.com/i/adsct?p_id=Twitter&p_user_id=0&txn_id=o6rzq&events=%5B%5B%22pageview%22%2Cnull%5D%5D&tw_sale_amount=0&tw_order_quantity=0&tw_iframe_status=0"/>
<img height="1" width="1" style="display:none" alt="" src="https://t.co/i/adsct?p_id=Twitter&p_user_id=0&txn_id=o6rzq&events=%5B%5B%22pageview%22%2Cnull%5D%5D&tw_sale_amount=0&tw_order_quantity=0&tw_iframe_status=0"/>
</noscript>
<script nonce="VqkFSleQStUMFdS2hc29nQ">
!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
},s.version='1.1',s.queue=[],u=t.createElement(n),u.async=!0,u.src='https://static.ads-twitter.com/uwt.js',
a=t.getElementsByTagName(n)[0],a.parentNode.insertBefore(u,a))}(window,document,'script');
twq('config','o6rzq');
</script>
<noscript>
<style>
#nojs-banner {
position: fixed;
bottom: 0;
left: 0;
padding: 16px 16px 16px 32px;
width: 100%;
box-sizing: border-box;
background: red;
color: white;
font-family: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
font-size: 13px;
line-height: 13px;
}
#nojs-banner a {
color: inherit;
text-decoration: underline;
}
</style>
<div id="nojs-banner">
This site requires JavaScript to run correctly. Please <a href="https://enable-javascript.com/" target="_blank">turn on JavaScript</a> or unblock scripts
</div>
</noscript>
<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" nonce="VqkFSleQStUMFdS2hc29nQ" data-cf-beacon='{"version":"2024.11.0","token":"68cfe66b5c4749e2ba64d4d9640c04c0","spa":2}' crossorigin="anonymous"></script>
</body>
</html>