Files
nexus/sreweekly/articles/456/02-how-dropbox-saved-millions-of-dollars-by-building-a-load-balancer.html
2026-09-12 17:23:01 +08:00

531 lines
199 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en" dir="ltr">
<head>
<meta charset="utf-8" />
<meta name="norton-safeweb-site-verification" content="24usqpep0ejc5w6hod3dulxwciwp0djs6c6ufp96av3t4whuxovj72wfkdjxu82yacb7430qjm8adbd5ezlt4592dq4zrvadcn9j9n-0btgdzpiojfzno16-fnsnu7xd" />
<link rel="preconnect" href="https://substackcdn.com" />
<title data-rh="true">How Dropbox Saved Millions of Dollars by Building a Load Balancer</title>
<meta data-rh="true" name="theme-color" content="#ffffff"/><meta data-rh="true" property="og:type" content="article"/><meta data-rh="true" property="og:title" content="How Dropbox Saved Millions of Dollars by Building a Load Balancer"/><meta data-rh="true" name="twitter:title" content="How Dropbox Saved Millions of Dollars by Building a Load Balancer"/><meta data-rh="true" name="description" content="Dropbox saved resources by creating a superior version of a tool everyone uses"/><meta data-rh="true" property="og:description" content="Dropbox saved resources by creating a superior version of a tool everyone uses"/><meta data-rh="true" name="twitter:description" content="Dropbox saved resources by creating a superior version of a tool everyone uses"/><meta data-rh="true" property="og:image" content="https://substackcdn.com/image/fetch/$s_!GAAQ!,w_1200,h_675,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png"/><meta data-rh="true" name="twitter:image" content="https://substackcdn.com/image/fetch/$s_!_zUL!,f_auto,q_auto:best,fl_progressive:steep/https%3A%2F%2Fbetterstack.substack.com%2Fapi%2Fv1%2Fpost_preview%2F152558295%2Ftwitter.jpg%3Fversion%3D4"/><meta data-rh="true" name="twitter:card" content="summary_large_image"/><meta data-rh="true" property="article:modified_time" content="2024-12-11T14:01:04.016Z"/>
<style>
@layer legacy, pencraftReset, pencraft;
</style>
<link rel="preload" as="style" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
<link rel="preload" as="style" href="https://substackcdn.com/bundle/theme/color_links.9856f2f08353fdd05959.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/11100.cee9fbee.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/86118.c25b9f67.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/93519.6640a5b2.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/42118.cba90b72.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/39522.25128f14.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/47358.668d8a3c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/94881.cb76fe04.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27386.72f00430.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/76768.f15fc605.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/41932.97d2663c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/48409.0588981c.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/32552.313415d5.css" />
<link rel="stylesheet" type="text/css" href="https://substackcdn.com/bundle/static/css/27313.92c631cb.css" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=0, viewport-fit=cover" />
<meta name="author" content="Richard Oliver Bray" />
<meta property="og:url" content="https://newsletter.betterstack.com/p/how-dropbox-saved-millions-of-dollars" />
<link rel="canonical" href="https://newsletter.betterstack.com/p/how-dropbox-saved-millions-of-dollars" />
<link rel="shortcut icon" href="https://substackcdn.com/image/fetch/$s_!Dxi5!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2665d1e9-50b4-4313-805a-dcbba920bb6b%2Ffavicon.ico">
<link rel="icon" type="image/png" sizes="16x16" href="https://substackcdn.com/image/fetch/$s_!7xYw!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2665d1e9-50b4-4313-805a-dcbba920bb6b%2Ffavicon-16x16.png">
<link rel="icon" type="image/png" sizes="32x32" href="https://substackcdn.com/image/fetch/$s_!LsWo!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2665d1e9-50b4-4313-805a-dcbba920bb6b%2Ffavicon-32x32.png">
<link rel="icon" type="image/png" sizes="48x48" href="https://substackcdn.com/image/fetch/$s_!ZvHi!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2665d1e9-50b4-4313-805a-dcbba920bb6b%2Ffavicon-48x48.png">
<link rel="apple-touch-icon" sizes="57x57" href="https://substackcdn.com/image/fetch/$s_!oHwe!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2665d1e9-50b4-4313-805a-dcbba920bb6b%2Fapple-touch-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="https://substackcdn.com/image/fetch/$s_!del3!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2665d1e9-50b4-4313-805a-dcbba920bb6b%2Fapple-touch-icon-60x60.png">
<link rel="apple-touch-icon" sizes="72x72" href="https://substackcdn.com/image/fetch/$s_!hjIz!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2665d1e9-50b4-4313-805a-dcbba920bb6b%2Fapple-touch-icon-72x72.png">
<link rel="apple-touch-icon" sizes="76x76" href="https://substackcdn.com/image/fetch/$s_!Uq50!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2665d1e9-50b4-4313-805a-dcbba920bb6b%2Fapple-touch-icon-76x76.png">
<link rel="apple-touch-icon" sizes="114x114" href="https://substackcdn.com/image/fetch/$s_!G3dS!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2665d1e9-50b4-4313-805a-dcbba920bb6b%2Fapple-touch-icon-114x114.png">
<link rel="apple-touch-icon" sizes="120x120" href="https://substackcdn.com/image/fetch/$s_!4Kz8!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2665d1e9-50b4-4313-805a-dcbba920bb6b%2Fapple-touch-icon-120x120.png">
<link rel="apple-touch-icon" sizes="144x144" href="https://substackcdn.com/image/fetch/$s_!aFgF!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2665d1e9-50b4-4313-805a-dcbba920bb6b%2Fapple-touch-icon-144x144.png">
<link rel="apple-touch-icon" sizes="152x152" href="https://substackcdn.com/image/fetch/$s_!aOvU!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2665d1e9-50b4-4313-805a-dcbba920bb6b%2Fapple-touch-icon-152x152.png">
<link rel="apple-touch-icon" sizes="167x167" href="https://substackcdn.com/image/fetch/$s_!nI4S!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2665d1e9-50b4-4313-805a-dcbba920bb6b%2Fapple-touch-icon-167x167.png">
<link rel="apple-touch-icon" sizes="180x180" href="https://substackcdn.com/image/fetch/$s_!uHBr!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2665d1e9-50b4-4313-805a-dcbba920bb6b%2Fapple-touch-icon-180x180.png">
<link rel="apple-touch-icon" sizes="1024x1024" href="https://substackcdn.com/image/fetch/$s_!QBXs!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2665d1e9-50b4-4313-805a-dcbba920bb6b%2Fapple-touch-icon-1024x1024.png">
<link rel="alternate" type="application/rss+xml" href="/feed" title="Hacking Scale by Better Stack"/>
<style data-pub-theme-prefix>:root{--color_theme_bg_pop:#5b63d3;--background_pop:#5b63d3;--cover_bg_color:#FFFFFF;--cover_bg_color_secondary:#f0f0f0;--background_pop_darken:#4750ce;--print_on_pop:#ffffff;--color_theme_bg_pop_darken:#4750ce;--color_theme_print_on_pop:#ffffff;--color_theme_bg_pop_20:rgba(91, 99, 211, 0.2);--color_theme_bg_pop_30:rgba(91, 99, 211, 0.3);--print_pop:#5b63d3;--color_theme_accent:#5b63d3;--cover_print_primary:#363737;--cover_print_secondary:#757575;--cover_print_tertiary:#b6b6b6;--cover_border_color:#5b63d3;--font_family_body_preset:'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';--font_weight_body_preset:400;--font_preset_body:sans;--home_hero:newspaper;--home_posts:grid;--web_bg_color:#ffffff;--background_contrast_1:#f0f0f0;--background_contrast_2:#dddddd;--background_contrast_3:#b7b7b7;--background_contrast_4:#929292;--background_contrast_5:#515151;--color_theme_bg_contrast_1:#f0f0f0;--color_theme_bg_contrast_2:#dddddd;--color_theme_bg_contrast_3:#b7b7b7;--color_theme_bg_contrast_4:#929292;--color_theme_bg_contrast_5:#515151;--color_theme_bg_elevated:#ffffff;--color_theme_bg_elevated_secondary:#f0f0f0;--color_theme_bg_elevated_tertiary:#dddddd;--color_theme_detail:#e6e6e6;--background_contrast_pop:rgba(91, 99, 211, 0.4);--color_theme_bg_contrast_pop:rgba(91, 99, 211, 0.4);--theme_bg_is_dark:0;--print_on_web_bg_color:#363737;--print_secondary_on_web_bg_color:#868787;--background_pop_rgb:91, 99, 211;--color_theme_bg_pop_rgb:91, 99, 211;--color_theme_accent_rgb:91, 99, 211;}</style>
<link rel="stylesheet" href="https://substackcdn.com/bundle/theme/main.06a4c4c291d4583811c7.css" />
<link rel="stylesheet" href="https://substackcdn.com/bundle/theme/color_links.9856f2f08353fdd05959.css" />
<style data-pub-theme-postfix></style>
<style id="app-theme-css" type="text/css">body {
--color-app-light-accent: rgb(255 103 25 / 1);
--color-app-light-accent-hover: rgb(255 86 0 / 1);
--color-app-light-accent-inverse: rgb(255 255 255 / 1);
--color-app-light-accent-text: rgb(255 103 25 / 1);
--color-app-dark-accent: rgb(255 103 25 / 1);
--color-app-dark-accent-hover: rgb(255 132 73 / 1);
--color-app-dark-accent-inverse: rgb(255 255 255 / 1);
--color-app-dark-accent-text: rgb(255 103 25 / 1);
--color-app-light-bg-accent: rgb(255 103 25 / 0.2);
--color-app-light-bg-accent-hover: rgb(255 103 25 / 0.3);
--color-app-dark-bg-accent: rgb(255 103 25 / 0.2);
--color-app-dark-bg-accent-hover: rgb(255 103 25 / 0.3);
--color-app-light-fg-primary: rgb(0 0 0 / 0.8);
--color-app-light-fg-secondary: rgb(0 0 0 / 0.5);
--color-app-light-fg-tertiary: rgb(0 0 0 / 0.3);
--color-app-dark-fg-primary: rgb(255 255 255 / 0.9);
--color-app-dark-fg-secondary: rgb(255 255 255 / 0.6);
--color-app-dark-fg-tertiary: rgb(255 255 255 / 0.4);
--color-app-light-bg-primary: rgb(255 255 255 / 1);
--color-app-light-bg-primary-hover: rgb(250 250 250 / 1);
--color-app-light-bg-secondary: rgb(238 238 238 / 1);
--color-app-light-bg-tertiary: rgb(219 219 219 / 1);
--color-app-light-bg-quaternary: rgb(182 182 182 / 1);
--color-app-dark-bg-primary: rgb(22 23 24 / 1);
--color-app-dark-bg-primary-hover: rgb(27 28 29 / 1);
--color-app-dark-bg-secondary: rgb(35 37 37 / 1);
--color-app-dark-bg-tertiary: rgb(54 55 55 / 1);
--color-app-dark-bg-quaternary: rgb(90 91 91 / 1);
}
</style>
<script async="true" src="https://www.googletagmanager.com/gtag/js?id=G-MWWB9F6017&l=GA4DataLayer" >
</script>
</head>
<body class="pc-root app-theme">
<div id="entry">
<div id="main" style="--post-preview-border-radius:var(--border-radius-xs);" class="main typography use-theme-bg"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div data-testid="navbar" class="main-menu"><div class="mainMenuContent-DME8DR"><div style="position:relative;height:71px;" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-20 pc-paddingRight-20 pc-justifyContent-space-between pc-alignItems-center pc-reset border-bottom-detail-k1F6C4 topBar-pIF0J1"><div style="flex-basis:0px;flex-grow:1;" class="logoContainer-p12gJb"><a href="/" native class="pencraft pc-display-contents pc-reset"><div draggable="false" class="pencraft pc-display-flex pc-position-relative pc-reset"><div style="width:40px;height:40px;" class="pencraft pc-display-flex pc-reset bg-white-ZBV5av pc-borderRadius-sm overflow-hidden-WdpwT6 sizing-border-box-DggLA4"><picture class="pencraft pc-display-contents pc-reset"><source type="image/webp" sizes="40px" srcset="https://substackcdn.com/image/fetch/$s_!_FGI!,w_40,h_40,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87b62237-9eb2-480e-abda-ac1ffc34a163_256x256.png 40w, https://substackcdn.com/image/fetch/$s_!_FGI!,w_80,h_80,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87b62237-9eb2-480e-abda-ac1ffc34a163_256x256.png 80w, https://substackcdn.com/image/fetch/$s_!_FGI!,w_120,h_120,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87b62237-9eb2-480e-abda-ac1ffc34a163_256x256.png 120w" style="display:none;"/><img style="object-fit:cover;width:40px;height:40px;" src="https://substackcdn.com/image/fetch/$s_!_FGI!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87b62237-9eb2-480e-abda-ac1ffc34a163_256x256.png" srcset="https://substackcdn.com/image/fetch/$s_!_FGI!,w_40,h_40,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87b62237-9eb2-480e-abda-ac1ffc34a163_256x256.png 40w, https://substackcdn.com/image/fetch/$s_!_FGI!,w_80,h_80,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87b62237-9eb2-480e-abda-ac1ffc34a163_256x256.png 80w, https://substackcdn.com/image/fetch/$s_!_FGI!,w_120,h_120,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87b62237-9eb2-480e-abda-ac1ffc34a163_256x256.png 120w" sizes="40px" alt="Hacking Scale by Better Stack" draggable="false" class="pencraft pc-reset"/></picture></div></div></a></div><div style="flex-grow:0;" class="titleContainer-DJYq5v"><h1 class="pencraft pc-reset font-pub-headings-FE5byy reset-IxiVJZ title-oOnUGd"><a href="/" class="pencraft pc-display-contents pc-reset">Hacking Scale by Better Stack</a></h1></div><div style="flex-basis:0px;flex-grow:1;" class="pencraft pc-display-flex pc-justifyContent-flex-end pc-alignItems-center pc-reset buttonsContainerContainer-ThaN_w"><div class="buttonsContainer-SJBuep"><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-flex-end pc-alignItems-center pc-reset navbar-buttons"><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><span data-state="closed"><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></span><button tabindex="0" type="button" aria-label="Share Publication" id="radix-P0-4" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-share"><path d="M4 12v8a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2v-8"></path><polyline points="16 6 12 2 8 6"></polyline><line x1="12" x2="12" y1="2" y2="15"></line></svg></button></div><button tabindex="0" type="button" data-testid="noncontributor-cta-button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_primary-RfbeYt size_md-gCDS3o">Subscribe</button><button tabindex="0" data-native="true" type="button" data-href="https://substack.com/sign-in?redirect=%2Fp%2Fhow-dropbox-saved-millions-of-dollars&amp;for_pub=betterstack" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_tertiary-rlke8z size_md-gCDS3o">Sign in</button></div></div></div></div></div><div style="height:72px;"></div></div></div><div><script nonce="JWtlbHixuqoee6u-MsbsjA" type="application/ld+json">{"@context":"https://schema.org","@type":"NewsArticle","url":"https://newsletter.betterstack.com/p/how-dropbox-saved-millions-of-dollars","mainEntityOfPage":"https://newsletter.betterstack.com/p/how-dropbox-saved-millions-of-dollars","headline":"How Dropbox Saved Millions of Dollars by Building a Load Balancer","description":"Dropbox saved resources by creating a superior version of a tool everyone uses","image":[{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png"}],"datePublished":"2024-12-11T14:01:03+00:00","dateModified":"2024-12-11T14:01:03+00:00","isAccessibleForFree":true,"author":[{"@type":"Person","name":"Richard Oliver Bray","url":"https://substack.com/@richardoliverbray","description":null,"identifier":"user:251681141","image":{"@type":"ImageObject","contentUrl":"https://substackcdn.com/image/fetch/$s_!yNNs!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cfb44c1-5ecd-42e3-a7e9-f08a8ad4ad51_1424x1254.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!yNNs!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cfb44c1-5ecd-42e3-a7e9-f08a8ad4ad51_1424x1254.png"}}],"publisher":{"@type":"Organization","name":"Hacking Scale by Better Stack","url":"https://newsletter.betterstack.com","description":"A weekly newsletter about building and scaling software from engineers at Better Stack","interactionStatistic":{"@type":"InteractionCounter","name":"Subscribers","interactionType":"https://schema.org/SubscribeAction","userInteractionCount":1000},"identifier":"pub:2269685","logo":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!_FGI!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87b62237-9eb2-480e-abda-ac1ffc34a163_256x256.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!_FGI!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87b62237-9eb2-480e-abda-ac1ffc34a163_256x256.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!_FGI!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87b62237-9eb2-480e-abda-ac1ffc34a163_256x256.png"},"image":{"@type":"ImageObject","url":"https://substackcdn.com/image/fetch/$s_!_FGI!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87b62237-9eb2-480e-abda-ac1ffc34a163_256x256.png","contentUrl":"https://substackcdn.com/image/fetch/$s_!_FGI!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87b62237-9eb2-480e-abda-ac1ffc34a163_256x256.png","thumbnailUrl":"https://substackcdn.com/image/fetch/$s_!_FGI!,w_128,h_128,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87b62237-9eb2-480e-abda-ac1ffc34a163_256x256.png"}},"interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":17},{"@type":"InteractionCounter","interactionType":"https://schema.org/ShareAction","userInteractionCount":1},{"@type":"InteractionCounter","interactionType":"https://schema.org/CommentAction","userInteractionCount":0}]}</script><script nonce="JWtlbHixuqoee6u-MsbsjA" type="application/ld+json">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Hacking Scale by Better Stack","item":"https://newsletter.betterstack.com"},{"@type":"ListItem","position":2,"name":"How Dropbox Saved Millions of Dollars by Building a Load Balancer","item":"https://newsletter.betterstack.com/p/how-dropbox-saved-millions-of-dollars"}]}</script><div aria-label="Post" role="main" class="single-post-container"><div class="container"><div class="single-post"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><article class="typography newsletter-post post"><div role="region" aria-label="Post header" class="post-header"><h1 dir="auto" class="post-title published title-X77sOw">How Dropbox Saved Millions of Dollars by Building a Load Balancer</h1><h3 dir="auto" class="subtitle subtitle-HEEcLo">Dropbox saved resources by creating a superior version of a tool everyone uses</h3><div aria-label="Post UFI" role="region" class="pencraft pc-display-flex pc-flexDirection-column pc-paddingBottom-16 pc-reset"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-gap-12 pc-alignItems-center pc-reset byline-wrapper"><div class="pencraft pc-display-flex pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-row pc-gap-8 pc-alignItems-center pc-justifyContent-flex-start pc-reset"><div style="--scale:36px;--offset:9px;--border-width:4.5px;" class="pencraft pc-display-flex pc-flexDirection-row pc-alignItems-center pc-justifyContent-flex-start pc-reset ltr-qDBmby"><a href="https://substack.com/@richardoliverbray" aria-label="View Richard Oliver Bray's profile" class="pencraft pc-display-contents pc-reset"><div style="--scale:36px;" tabindex="0" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA animate-XFJxE4 outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 pressable-sm-YIJFKJ showFocus-sk_vEm container-TAtrWj interactive-UkK0V6 avatar-u8q6xB last-JfNEJ_"><div style="--scale:36px;" title="Richard Oliver Bray" class="pencraft pc-display-flex pc-width-36 pc-height-36 pc-justifyContent-center pc-alignItems-center pc-position-relative pc-reset bg-secondary-UUD3_J flex-auto-j3S2WA outline-detail-vcQLyr pc-borderRadius-full overflow-hidden-WdpwT6 sizing-border-box-DggLA4 container-TAtrWj"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!yNNs!,w_36,h_36,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cfb44c1-5ecd-42e3-a7e9-f08a8ad4ad51_1424x1254.png 1x, https://substackcdn.com/image/fetch/$s_!yNNs!,w_72,h_72,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cfb44c1-5ecd-42e3-a7e9-f08a8ad4ad51_1424x1254.png 2x, https://substackcdn.com/image/fetch/$s_!yNNs!,w_108,h_108,c_fill,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cfb44c1-5ecd-42e3-a7e9-f08a8ad4ad51_1424x1254.png 3x"/><img src="https://substackcdn.com/image/fetch/$s_!yNNs!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cfb44c1-5ecd-42e3-a7e9-f08a8ad4ad51_1424x1254.png" alt="Richard Oliver Bray's avatar" srcset="https://substackcdn.com/image/fetch/$s_!yNNs!,w_36,h_36,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cfb44c1-5ecd-42e3-a7e9-f08a8ad4ad51_1424x1254.png 1x, https://substackcdn.com/image/fetch/$s_!yNNs!,w_72,h_72,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cfb44c1-5ecd-42e3-a7e9-f08a8ad4ad51_1424x1254.png 2x, https://substackcdn.com/image/fetch/$s_!yNNs!,w_108,h_108,c_fill,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cfb44c1-5ecd-42e3-a7e9-f08a8ad4ad51_1424x1254.png 3x" width="36" height="36" draggable="false" class="img-OACg1c object-fit-cover-u4ReeV pencraft pc-reset"/></picture></div></div></a></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset"><div class="pencraft pc-reset color-primary-zABazT line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><span style="min-width:0;" data-state="closed"><a href="https://substack.com/@richardoliverbray" class="pencraft pc-reset decoration-hover-underline-ClDVRM reset-IxiVJZ">Richard Oliver Bray</a></span></div><div class="pencraft pc-display-flex pc-gap-4 pc-reset"><div class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-meta-MWBumP size-11-NuY2Zx weight-medium-fw81nC transform-uppercase-yKDgcq reset-IxiVJZ meta-EgzBVA"><time datetime="2024-12-11T14:01:03.923Z" title="2024-12-11T14:01:03.923Z">Dec 11, 2024</time></div></div></div></div></div><div class="pencraft pc-display-flex pc-gap-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (17)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">17</div></button></div><button tabindex="0" type="button" id="radix-P0-27" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg><div class="label">1</div></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div></div><div class="visibility-check"></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset flex-grow-rzmknG"><div class="pencraft pc-display-flex pc-width-72 pc-height-72 pc-justifyContent-center pc-alignItems-center pc-reset flex-auto-j3S2WA border-detail-EGrm7T pc-borderRadius-sm"><svg aria-hidden="true" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke-width="1.8" stroke="var(--color-fg-secondary)" xmlns="http://www.w3.org/2000/svg"><g><path d="M11.8518 3.31518C12.2459 4.26329 13.551 7.10985 14.4542 8.3467C15.3764 9.60967 12.8285 9.95184 12 10.5" stroke-width="1.5" stroke-linecap="round"></path><path d="M12 10.5C12 10.5 12.4662 11.2332 12.7627 11.7326C13.0592 12.232 13 12.5 11.5 13C10.7772 13.2409 9.5 13.5 9.5 13.5C10.3447 13.9272 12.2664 14.1393 12.4041 14.3167C12.763 14.779 12.3241 15.4618 11.8617 15.8208C11.3994 16.1797 12.0724 18.7353 10.5 19.5C8.9276 20.2647 5.70772 19.432 3 18" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path><path d="M18.5 6.5L20.3544 4.8526" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path><path d="M18.5 15.5L20.3544 17.2545" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path><path d="M19.5 11.0326L22 11.0326" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"></path></g></svg></div><div aria-label="Voiceover player" role="region" class="pencraft pc-display-flex pc-alignItems-center pc-gap-12 pc-padding-12 pc-reset bg-primary-zk6FDl flex-grow-rzmknG userSelect-none-oDUy26 border-detail-EGrm7T pc-borderRadius-sm sizing-border-box-DggLA4 embed-gCEqUi"><button tabindex="0" type="button" aria-label="Play" aria-pressed="false" data-state="off" class="pencraft pc-reset pencraft toggle-qPLOMm animate-Rwg7DP button-yhxrk8 iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_lg-A_bUNK priority_secondary-S63h9o"><div class="pencraft pc-display-flex pc-justifyContent-center pc-alignItems-center pc-reset content-PO04C_ on-oX6ZaQ"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 20 20" fill="var(--color-fg-primary)" stroke-width="1.8" stroke="#000" xmlns="http://www.w3.org/2000/svg"><g><path stroke="none" d="M5.29883 17.9082H7.52539C8.375 17.9082 8.82422 17.459 8.82422 16.5996V3.29883C8.82422 2.41016 8.375 2 7.52539 2H5.29883C4.44922 2 4 2.44922 4 3.29883V16.5996C4 17.459 4.44922 17.9082 5.29883 17.9082ZM12.3984 17.9082H14.6152C15.4746 17.9082 15.9141 17.459 15.9141 16.5996V3.29883C15.9141 2.41016 15.4746 2 14.6152 2H12.3984C11.5391 2 11.0898 2.44922 11.0898 3.29883V16.5996C11.0898 17.459 11.5391 17.9082 12.3984 17.9082Z"></path></g></svg></div><div class="pencraft pc-display-flex pc-justifyContent-center pc-alignItems-center pc-reset content-PO04C_ off-Pr96yj"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 20 20" fill="var(--color-fg-primary)" stroke-width="1.8" stroke="#000" xmlns="http://www.w3.org/2000/svg"><g><path stroke="none" d="M5.04688 18.5527C5.4375 18.5527 5.76953 18.3965 6.16016 18.1719L17.5469 11.5898C18.3574 11.1113 18.6406 10.7988 18.6406 10.2812C18.6406 9.76367 18.3574 9.45117 17.5469 8.98242L6.16016 2.39063C5.76953 2.16602 5.4375 2.01953 5.04688 2.01953C4.32422 2.01953 3.875 2.56641 3.875 3.41602V17.1465C3.875 17.9961 4.32422 18.5527 5.04688 18.5527Z"></path></g></svg></div></button><div class="pencraft pc-display-flex pc-flexDirection-column pc-gap-6 pc-reset flex-grow-rzmknG"><div class="pencraft pc-reset color-secondary-ls1g8s line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-medium-fw81nC reset-IxiVJZ">Article voiceover</div><div aria-valuemin="0" aria-valuemax="100" aria-valuenow="0" class="pencraft pc-display-flex pc-alignItems-center pc-gap-8 pc-reset flex-grow-rzmknG"><div class="pencraft pc-minWidth-40 pc-reset color-secondary-ls1g8s align-left-PENAI6 line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-regular-mUq6Gb reset-IxiVJZ">0:00</div><div style="--progress:0;" class="pencraft pc-display-flex pc-height-12 pc-alignItems-center pc-position-relative pc-reset flex-grow-rzmknG cursor-pointer-LYORKw container-AQs2b4"><div class="pencraft pc-display-flex pc-height-4 pc-position-relative pc-reset bg-tertiary-CPDeXV flex-grow-rzmknG pc-borderRadius-full overflow-hidden-WdpwT6"><div class="pencraft pc-opacity-90 pc-position-absolute pc-reset fill-xiDTGC"></div></div><div class="pencraft pc-width-12 pc-height-12 pc-boxShadow-xs pc-position-absolute pc-reset pc-borderRadius-full playhead-jk5E3b"></div></div><div class="pencraft pc-minWidth-40 pc-reset color-secondary-ls1g8s align-right-VJbKw5 line-height-20-t4M0El font-text-qe4AeH size-13-hZTUKr weight-regular-mUq6Gb reset-IxiVJZ">-8:00</div></div></div><button tabindex="0" type="button" aria-label="Gauge" id="radix-P0-41" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_quaternary-kpMibu"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-gauge"><path d="m12 14 4-4"></path><path d="M3.34 19a10 10 0 1 1 17.32 0"></path></svg></button></div></div><audio data-testid="audio-element" src="/api/v1/audio/upload/ef91a656-c97e-4643-8189-5761abaa8be2/src" preload="none">Audio playback is not supported on your browser. Please upgrade.</audio><div><div class="dt-post-body"><div class="available-content"><div dir="auto" class="body markup"><p><strong>Dropbox</strong><span> is a </span><strong>cloud-based storage service</strong><span> that is ridiculously easy to use.</span></p><p>Download the app and drag your files into the newly created folder. That's it; your files are in the cloud and can be accessed from anywhere.</p><p>It sounds like a simple idea, but back in 2007, when it was released, there wasn't anything like it.</p><p><span>Today, Dropbox has around </span><strong>700 million users</strong><span> and stores over </span><strong>550 billion files</strong><span>.</span></p><p><span>All these files need to be </span><strong>organized, backed up, and accessible from anywhere</strong><span>. Dropbox uses </span><strong>virtual servers</strong><span> for this. But they often got </span><strong>overloaded</strong><span> and sometimes crashed.</span></p><p><span>So, the team at Dropbox built a solution to </span><strong>manage server loads</strong><span>.</span></p><p>Here's how they did it.</p><p><em>Estimated reading time: 5 minutes 15 seconds</em></p><div><hr/></div><div class="subscription-widget-wrap"><div class="subscription-widget show-subscribe"><div class="preamble"><p>We do tech deep dives like this <strong>every 2 weeks</strong>. Get the next one sent straight to your inbox.</p></div><div data-component-name="SubscribeWidget" class="subscribe-widget"><div class="pencraft pc-display-flex pc-justifyContent-center pc-reset"><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe-widget-preamble"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><input type="hidden" name="source_post_id" value="152558295"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div></div></div><div><hr/></div><h2 class="header-anchor-post">Why Dropbox Servers Were Overloaded<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§why-dropbox-servers-were-overloaded" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://newsletter.betterstack.com/i/152558295/why-dropbox-servers-were-overloaded" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p><span>Before Dropbox grew in scale, they </span><strong>used a traditional system</strong><span> to balance load.</span></p><p><span>This likely used a </span><strong>round-robin algorithm with fixed weights</strong><span>.</span></p><p>So, a user or client would upload a file. The load balancer would forward the upload request to a server. Then, that server would upload the file and store it correctly.</p><div><hr/></div><h4 class="header-anchor-post"><em>Sidenote: Weighted Round Robin</em><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§sidenote-weighted-round-robin" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://newsletter.betterstack.com/i/152558295/sidenote-weighted-round-robin" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h4><p><em><span>A round-robin is a </span><strong>simple load-balancing algorithm</strong><span>. It works by cycling requests to different servers so they get an equal share of the load.</span></em></p><p><em>If there are three servers, A, B, C, and three requests come in. A gets the first, B gets the second, and C gets the third.</em></p><p><em><strong>Weighted round robin</strong><span> is a level up from round robin. Each server is given a </span><strong>weight based on its processing power and capacity</strong><span>.</span></em></p><p><em><strong>Static weights</strong><span> are assigned manually by a network admin. </span><strong>Dynamic weights</strong><span> are adjusted in </span><strong>real time</strong><span> by a load balancer. </span></em></p><p><em>The higher the weight, the more load the server gets.</em></p><p><em>So if A has a weight of 3, B has 2, C has 1, and there were 12 requests. A would get 6, B would get 4, and C would get 2.</em></p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!eN06!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!eN06!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png 424w, https://substackcdn.com/image/fetch/$s_!eN06!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png 848w, https://substackcdn.com/image/fetch/$s_!eN06!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png 1272w, https://substackcdn.com/image/fetch/$s_!eN06!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!eN06!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png" width="639" height="470.8997429305913" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:860,&quot;width&quot;:1167,&quot;resizeWidth&quot;:639,&quot;bytes&quot;:113057,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!eN06!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png 424w, https://substackcdn.com/image/fetch/$s_!eN06!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png 848w, https://substackcdn.com/image/fetch/$s_!eN06!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png 1272w, https://substackcdn.com/image/fetch/$s_!eN06!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><div><hr/></div><p>But there was an issue with their traditional load balancing approach.</p><p><span>Dropbox had </span><strong>many virtual servers</strong><span> with vastly </span><strong>different hardware</strong><span>. This made it </span><strong>difficult to distribute the load evenly</strong><span> between them with static weights.</span></p><p>This difference in hardware could have been caused by Dropbox using more powerful servers as it grew.</p><p>They may have started with an average server. As it grew, the team acquired more powerful servers. As it grew more, they acquired even more powerful ones.</p><p><span>At the time, there was </span><strong>no off-the-shelf load-balancing solution</strong><span> that could help. Especially one that used a dynamic weighted round-robin </span><strong>with gRPC support</strong><span>.</span></p><p><span>So, they </span><strong>built their own</strong><span>, which they called </span><strong>Robinhood</strong><span>.</span></p><div><hr/></div><h4 class="header-anchor-post"><em>Sidenote: gRPC</em><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§sidenote-grpc" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://newsletter.betterstack.com/i/152558295/sidenote-grpc" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h4><p><em><strong>Google Remote Procedure Call</strong><span> (gRPC) is a way for different programs to talk to each other. It's based on RPC, which allows a client to </span><strong>run a function on the server simply by calling it</strong><span>.</span></em></p><p><em><span>This is </span><strong>different from REST</strong><span>, which requires communication via a URL. REST also focuses on the resource being accessed instead of the action that needs to be taken.</span></em></p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!A6Ug!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png" data-component-name="Image2ToDOM" class="image-link image2 can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!A6Ug!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png 424w, https://substackcdn.com/image/fetch/$s_!A6Ug!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png 848w, https://substackcdn.com/image/fetch/$s_!A6Ug!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png 1272w, https://substackcdn.com/image/fetch/$s_!A6Ug!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!A6Ug!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png" width="536" height="158.6122448979592" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/f8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:232,&quot;width&quot;:784,&quot;resizeWidth&quot;:536,&quot;bytes&quot;:42039,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!A6Ug!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png 424w, https://substackcdn.com/image/fetch/$s_!A6Ug!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png 848w, https://substackcdn.com/image/fetch/$s_!A6Ug!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png 1272w, https://substackcdn.com/image/fetch/$s_!A6Ug!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div></div></div></a></figure></div><p><em>But gRPC has many more differences between REST and regular RPC.</em></p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!VnYj!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!VnYj!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png 424w, https://substackcdn.com/image/fetch/$s_!VnYj!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png 848w, https://substackcdn.com/image/fetch/$s_!VnYj!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png 1272w, https://substackcdn.com/image/fetch/$s_!VnYj!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!VnYj!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png" width="617" height="284.0733137829912" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:471,&quot;width&quot;:1023,&quot;resizeWidth&quot;:617,&quot;bytes&quot;:88880,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!VnYj!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png 424w, https://substackcdn.com/image/fetch/$s_!VnYj!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png 848w, https://substackcdn.com/image/fetch/$s_!VnYj!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png 1272w, https://substackcdn.com/image/fetch/$s_!VnYj!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><p><em><span>The biggest one is the </span><strong>use of protobufs</strong><span>. This file format </span><a href="https://protobuf.dev/">developed by Google</a><span> is used </span><strong>to store and send data</strong><span>.</span></em></p><p><em>It works by encoding structured data into a binary format for fast transmission. The recipient then decodes it back to structured data. This format is also much smaller than something like JSON.</em></p><p><em><span>Protobufs are what make </span><strong>gRPC fast</strong><span>, but also more difficult to set up since the client and server need to support it.</span></em></p><p><em><span>gRPC isn't supported natively by browsers. So, it's </span><strong>commonly</strong><span> </span><strong>used for internal server communication</strong><span>.</span></em></p><div><hr/></div><h2 class="header-anchor-post">The Custom Load Balancer<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§the-custom-load-balancer" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://newsletter.betterstack.com/i/152558295/the-custom-load-balancer" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p><span>The </span><strong>main component of RobinHood</strong><span> is the load balancing service or </span><strong>LBS</strong><span>. This </span><strong>manages how requests are distributed</strong><span> to different servers.</span></p><p><span>It does this by </span><strong>continuously collecting data</strong><span> from all the servers. It uses this data to figure out the average optimal resource usage for all the servers.</span></p><p><span>Each server is given a </span><strong>PID controller</strong><span>, a piece of code to help with resource regulation. This has an </span><strong>upper and lower server resource limit</strong><span> close to the average.</span></p><p>Say the average CPU limit is 70%. The upper limit could be 75%, and the lower limit could be 65%. If a server hits 75%, it is given fewer requests to deal with, and if it goes below 65%, it is given more.</p><p><span>This is </span><strong>how the LBS gives weights</strong><span> to each server. Because the LBS uses dynamic weights, a server that previously weighted 5 could become 1 if its resources go above the average.</span></p><p><span>In addition to the LBS, </span><strong>Robinhood had two other components</strong><span>: the </span><strong>proxy</strong><span> and the </span><strong>routing database</strong><span>.</span></p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!GAAQ!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!GAAQ!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png 424w, https://substackcdn.com/image/fetch/$s_!GAAQ!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png 848w, https://substackcdn.com/image/fetch/$s_!GAAQ!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png 1272w, https://substackcdn.com/image/fetch/$s_!GAAQ!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!GAAQ!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png" width="539" height="654.034103410341" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:1103,&quot;width&quot;:909,&quot;resizeWidth&quot;:539,&quot;bytes&quot;:123449,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!GAAQ!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png 424w, https://substackcdn.com/image/fetch/$s_!GAAQ!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png 848w, https://substackcdn.com/image/fetch/$s_!GAAQ!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png 1272w, https://substackcdn.com/image/fetch/$s_!GAAQ!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><p><span>The </span><strong>proxy sends server load data to the LBS</strong><span> via gRPC. </span></p><p>Why doesn't the LBS collect this itself? Well, the LBS is already doing a lot.</p><p>Imagine there could be thousands of servers. It would need to scale up just to collect metrics from all of them.</p><p>So, the proxy has the sole responsibility of collecting server data to reduce the load on the LBS.</p><p><span>The </span><strong>routing database stores server information</strong><span>. Things like weights generated by the LBS, IP addresses, hostname, etc.</span></p><p><span>Although the LBS stores some data in memory for quick access, </span><strong>an</strong><span> </span><strong>LBS itself can come in and out of existence</strong><span>; sometimes, it crashes and needs to restart.</span></p><p><span>The </span><strong>routing database keeps data for a long time</strong><span>, so new or existing LBS instances can access it.</span></p><p>Routing databases can either be Zookeeper or etcd based. The decision to choose one or the other may be to support legacy systems.</p><div><hr/></div><h4 class="header-anchor-post"><em>Sidenote: Zookeeper vs etcd</em><div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§sidenote-zookeeper-vs-etcd" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://newsletter.betterstack.com/i/152558295/sidenote-zookeeper-vs-etcd" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h4><p><em><span>Both Zookeeper and etcd are what's called a </span><strong>distributed coordination service</strong><span>.</span></em></p><p><em><span>They are designed to be the </span><strong>central place where config and state data is stored</strong><span> in a distributed system.</span></em></p><p><em><span>They also make sure that </span><strong>each node</strong><span> in the system has the most </span><strong>up-to-date version of this data</strong><span>.</span></em></p><p><em>These services contain multiple servers and elect a single server, called a leader, that takes all the writes.</em></p><p><em>This server copies the data to other servers, which then distribute the data to the relevant clients. In this case, a client could be an LBS instance.</em></p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!bXRy!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!bXRy!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png 424w, https://substackcdn.com/image/fetch/$s_!bXRy!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png 848w, https://substackcdn.com/image/fetch/$s_!bXRy!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png 1272w, https://substackcdn.com/image/fetch/$s_!bXRy!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!bXRy!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png" width="666" height="320.64972527472526" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/af11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:701,&quot;width&quot;:1456,&quot;resizeWidth&quot;:666,&quot;bytes&quot;:94238,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!bXRy!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png 424w, https://substackcdn.com/image/fetch/$s_!bXRy!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png 848w, https://substackcdn.com/image/fetch/$s_!bXRy!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png 1272w, https://substackcdn.com/image/fetch/$s_!bXRy!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><p><em>So, if a new LBS instance joins the cluster, it knows the exact state of all the servers and the average that needs to be achieved.</em></p><p><em>There are a few differences between Zookeeper and etcd.</em></p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!T1n9!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!T1n9!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png 424w, https://substackcdn.com/image/fetch/$s_!T1n9!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png 848w, https://substackcdn.com/image/fetch/$s_!T1n9!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png 1272w, https://substackcdn.com/image/fetch/$s_!T1n9!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!T1n9!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png" width="627" height="289.33756097560973" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/d4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:473,&quot;width&quot;:1025,&quot;resizeWidth&quot;:627,&quot;bytes&quot;:82294,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!T1n9!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png 424w, https://substackcdn.com/image/fetch/$s_!T1n9!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png 848w, https://substackcdn.com/image/fetch/$s_!T1n9!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png 1272w, https://substackcdn.com/image/fetch/$s_!T1n9!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><div><hr/></div><p><span>After </span><strong>Dropbox deployed RobinHood to all their data centers</strong><span>, here is the difference it made.</span></p><div class="captioned-image-container"><figure><a target="_blank" href="https://substackcdn.com/image/fetch/$s_!wzfl!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp" data-component-name="Image2ToDOM" class="image-link image2 is-viewable-img can-restack"><div class="image2-inset"><picture><source type="image/webp" srcset="https://substackcdn.com/image/fetch/$s_!wzfl!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp 424w, https://substackcdn.com/image/fetch/$s_!wzfl!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp 848w, https://substackcdn.com/image/fetch/$s_!wzfl!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp 1272w, https://substackcdn.com/image/fetch/$s_!wzfl!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp 1456w" sizes="100vw"/><img src="https://substackcdn.com/image/fetch/$s_!wzfl!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp" width="691" height="436.6736111111111" data-attrs="{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:910,&quot;width&quot;:1440,&quot;resizeWidth&quot;:691,&quot;bytes&quot;:21304,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/webp&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}" alt srcset="https://substackcdn.com/image/fetch/$s_!wzfl!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp 424w, https://substackcdn.com/image/fetch/$s_!wzfl!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp 848w, https://substackcdn.com/image/fetch/$s_!wzfl!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp 1272w, https://substackcdn.com/image/fetch/$s_!wzfl!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp 1456w" sizes="100vw" loading="lazy" class="sizing-normal"/></picture><div class="image-link-expand"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container restack-image"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="1.5" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg></button><button tabindex="0" type="button" class="pencraft pc-reset pencraft icon-container view-image"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-maximize2 lucide-maximize-2"><polyline points="15 3 21 3 21 9"></polyline><polyline points="9 21 3 21 3 15"></polyline><line x1="21" x2="14" y1="3" y2="10"></line><line x1="3" x2="10" y1="21" y2="14"></line></svg></button></div></div></div></a></figure></div><p>The X axis shows date in MM/DD and the Y axis shows the ratio of CPU usage compared to the average. So, a value of 1.5 means CPU usage was 1.5 times higher than the average.</p><p>You can see that at the start, 95% of CPUs were operating at around 1.17 above the average.</p><p><span>It takes a </span><strong>few days for RobinHood to regulate everything</strong><span>, but after 11/01, the usage is stabilized, and most CPUs are operating at the average. </span></p><p><span>This shows a </span><strong>massive reduction in CPU workload</strong><span>, which indicates a </span><strong>better-balanced load</strong><span>.</span></p><p><span>In fact, after using Robinhood in production for a few years, the team at Dropbox has been able to </span><strong>reduce their server size by 25%</strong><span>. This massively reduced their costs.</span></p><p>It isn't stated that Dropbox saved millions annually from this change. But, based on the cost and resource savings they mentioned from implementing Robinhood, as well as their size.</p><p>It can be inferred that they saved a lot of money, most likely millions from this change.</p><h2 class="header-anchor-post">Wrapping Things Up<div class="pencraft pc-display-flex pc-alignItems-center pc-position-absolute pc-reset header-anchor-parent"><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div id="§wrapping-things-up" class="pencraft pc-reset header-anchor offset-top"></div><button tabindex="0" type="button" aria-label="Link" data-href="https://newsletter.betterstack.com/i/152558295/wrapping-things-up" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_sm-G3LciD priority_secondary-S63h9o"><svg xmlns="http://www.w3.org/2000/svg" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-link"><path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71"></path><path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71"></path></svg></button></div></div></h2><p>It's amazing everything that goes on behind the scenes when someone uploads a file to Dropbox. I will never look at the app in the same way again.</p><p><span>I hope you enjoyed reading this as much as I enjoyed writing it. If you want more details, you can check out the </span><a href="https://dropbox.tech/infrastructure/robinhood-in-house-load-balancing-service#footnote-two">original article</a><span>.</span></p><p>And as usual, be sure to subscribe to get the next article sent straight to your inbox.</p><div><hr/></div><p><strong>Newsletter Spotlight</strong></p><p><span>If you want to become a more productive tech professional, check out the </span><a href="https://techproductivity.co/">Tech Productivity</a><span> newsletter. </span></p><p>It’s full of links and tools you wish you had known about earlier.</p><div><hr/></div><p><em><span>PS: Enjoyed this newsletter? Please forward it to a pal or follow us on socials (</span><a href="https://www.linkedin.com/company/betterstack/">LinkedIn</a><span>, </span><a href="https://x.com/BetterStackHQ">Twitter</a><span>, </span><a href="https://www.youtube.com/@betterstack">YouTube</a><span>, </span><a href="https://www.instagram.com/betterstackhq/">Instagram</a><span>). It only takes 10 seconds. Making this one took 20 hours.</span></em></p></div></div><div class="visibility-check"></div></div><div class="post-footer"><div class="pencraft pc-display-flex pc-gap-16 pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset flex-grow-rzmknG border-top-detail-themed-k9TZAY border-bottom-detail-themed-Ua9186 post-ufi"><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><div class="like-button-container post-ufi-button style-button"><button tabindex="0" type="button" aria-label="Like (17)" aria-pressed="false" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" style="height:20px;width:20px;" width="20" height="20" viewBox="0 0 24 24" fill="#000000" stroke-width="2" stroke="#000" xmlns="http://www.w3.org/2000/svg" class="icon"><g><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-heart"><path d="M19 14c1.49-1.46 3-3.21 3-5.5A5.5 5.5 0 0 0 16.5 3c-1.76 0-3 .5-4.5 2-1.5-1.5-2.74-2-4.5-2A5.5 5.5 0 0 0 2 8.5c0 2.3 1.5 4.05 3 5.5l7 7Z"></path></svg></g></svg><div class="label">17</div></button></div><button tabindex="0" type="button" id="radix-P0-58" aria-haspopup="menu" aria-expanded="false" data-state="closed" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><svg aria-hidden="true" width="20" height="20" viewBox="0 0 20 20" fill="none" stroke-width="2" stroke="var(--color-fg-primary)" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg" class="icon"><g><path d="M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882"></path></g></svg><div class="label">1</div></button></div><div class="pencraft pc-display-flex pc-gap-8 pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft post-ufi-button style-button has-label with-border"><div class="label">Share</div></button></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="pencraft pc-display-flex pc-paddingTop-16 pc-paddingBottom-16 pc-justifyContent-space-between pc-reset"><button tabindex="0" type="button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_secondary-S63h9o size_md-gCDS3o"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-left"><path d="m12 19-7-7 7-7"></path><path d="M19 12H5"></path></svg>Previous</button><button tabindex="0" type="button" class="pencraft pc-reset pencraft buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_secondary-S63h9o size_md-gCDS3o">Next<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-arrow-right"><path d="M5 12h14"></path><path d="m12 5 7 7-7 7"></path></svg></button></div></div></div></article></div></div></div><div class="pencraft pc-display-contents pc-reset pubTheme-yiXxQA"><div class="single-post-section"><div class="container"><div class="visibility-check"></div><div style="margin-left:-8px;margin-right:-8px;" aria-label="Top Posts Footer" role="region" class="pencraft pc-paddingTop-24 pc-paddingBottom-24 pc-reset"><div class="portable-archive empty-list"><div aria-label="Archive sort tabs" role="navigation" class="pencraft pc-display-flex pc-gap-12 pc-paddingLeft-8 pc-paddingRight-8 pc-paddingBottom-16 pc-justifyContent-space-between pc-alignItems-center pc-reset"><div class="pencraft pc-display-flex pc-flexDirection-column pc-position-relative pc-minWidth-0 pc-reset bg-primary-zk6FDl outline-detail-vcQLyr pc-borderRadius-sm overflow-hidden-WdpwT6"><div dir="ltr" data-orientation="horizontal" class="pencraft pc-display-flex pc-flexDirection-column pc-reset flex-grow-rzmknG"><div style="outline:none;" tabindex="-1" aria-label="Tabs" role="tablist" aria-orientation="horizontal" data-orientation="horizontal" class="pencraft pc-display-flex pc-gap-4 pc-padding-4 pc-position-relative pc-reset cursor-default-flE2S1 pc-borderRadius-sm overflow-auto-7WTsTi scrollBar-hidden-HcAIpI"><button tabindex="-1" type="button" role="tab" aria-selected="true" aria-controls="radix-P0-72-content-top" data-state="active" id="radix-P0-72-trigger-top" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Top</button><button tabindex="-1" type="button" role="tab" aria-selected="false" aria-controls="radix-P0-72-content-new" data-state="inactive" id="radix-P0-72-trigger-new" data-orientation="horizontal" data-radix-collection-item class="pencraft pc-reset flex-auto-j3S2WA pencraft segment-j4TeZ4 buttonBase-GK1x3M buttonText-X0uSmG buttonStyle-r7yGCK priority_quaternary-kpMibu size_sm-G3LciD">Latest</button><div class="pencraft pc-position-absolute pc-height-32 pc-reset bg-secondary-UUD3_J pc-borderRadius-xs sizing-border-box-DggLA4 highlight-U002IP"></div></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH left-Tg8vqp"><div class="overlay-zrMCxn primary-lv_sOW"></div></div><div class="pencraft pc-display-flex pc-alignItems-center pc-reset arrowButtonContainer-O4uSiH arrowButtonOverlaidContainer-t10AyH right-i3oWGi"><div class="overlay-zrMCxn primary-lv_sOW"></div></div></div><button tabindex="0" type="button" aria-label="Search" class="pencraft pc-reset pencraft iconButton-mq_Et5 iconButtonBase-dJGHgN buttonBase-GK1x3M buttonStyle-r7yGCK size_md-gCDS3o priority_tertiary-rlke8z"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="lucide lucide-search"><circle cx="11" cy="11" r="8"></circle><path d="m21 21-4.3-4.3"></path></svg></button></div><div class="portable-archive-list"><p class="portable-archive-empty dt-archive-empty-state">No posts</p></div></div></div></div></div><div class="visibility-check"></div><div class="pencraft pc-display-contents pc-reset pubInvertedTheme-U483dz"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-justifyContent-center pc-padding-48 pc-mobile-padding-24 pc-reset bg-primary-zk6FDl container-jsOc9L"><div class="pencraft pc-display-flex pc-flexDirection-column pc-alignItems-center pc-gap-24 pc-flexWrap-wrap pc-reset content-jLbYeh"><h3 class="pencraft pc-reset color-primary-zABazT align-center-y7ZD4w line-height-28-s562kJ font-display-nhmvtD size-24-lFU3ly weight-semibold-uqA4FV reset-IxiVJZ">Ready for more?</h3><div class="container-IpPqBD"><form action="/api/v1/free?nojs=true" method="post" novalidate class="form form-M5sC90"><input type="hidden" name="first_url" value/><input type="hidden" name="first_referrer" value/><input type="hidden" name="current_url"/><input type="hidden" name="current_referrer"/><input type="hidden" name="first_session_url" value/><input type="hidden" name="first_session_referrer" value/><input type="hidden" name="referral_code"/><input type="hidden" name="source" value="subscribe_footer"/><input type="hidden" name="referring_pub_id"/><input type="hidden" name="additional_referring_pub_ids"/><div class="sideBySideWrap-vGXrwP"><div class="emailInputWrapper-QlA86j"><div class="pencraft pc-display-flex pc-minWidth-0 pc-position-relative pc-reset flex-auto-j3S2WA"><input name="email" placeholder="Type your email..." type="email" class="pencraft emailInput-OkIMeB emailInputOnAccentBackground-TfaCGr input-y4v6N4 inputText-pV_yWb"/></div></div><button tabindex="0" type="submit" disabled class="pencraft pc-reset pencraft rightButton primary subscribe-btn button-VFSdkv buttonOnAccentBackground-vmEt94 buttonBase-GK1x3M"><span class="button-text ">Subscribe</span></button></div><div id="error-container"></div></form></div></div></div></div></div></div></div><div class="pencraft pc-display-flex pc-flexDirection-column pc-reset container-K5XPov"></div><div class="footer-wrap publication-footer"><div class="visibility-check"></div><div class="footer themed-background"><div class="container"><div class="footer-terms"><span>© 2026 Better Stack</span><span> · </span><a href="https://substack.com/privacy" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Privacy</a><span> ∙ </span><a href="https://substack.com/tos" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Terms</a><span> ∙ </span><a href="https://substack.com/ccpa#personal-data-collected" target="_blank" rel="noopener" class="pencraft pc-reset decoration-underline-ClTkYc">Collection notice</a></div><div class="pencraft pc-display-flex pc-gap-8 pc-justifyContent-center pc-alignItems-center pc-reset footerButtons-ap9Sk7"><a data-native="true" href="https://substack.com/signup?utm_source=substack&amp;utm_medium=web&amp;utm_content=footer" class="footerSubstackCta-v5HWfj"><svg aria-hidden="true" width="1000" height="1000" viewBox="0 0 1000 1000" fill="#ff6719" stroke-width="1.8" stroke="none" xmlns="http://www.w3.org/2000/svg"><g><path d="M764.166 348.371H236.319V419.402H764.166V348.371Z"></path><path d="M236.319 483.752V813.999L500.231 666.512L764.19 813.999V483.752H236.319Z"></path><path d="M764.166 213H236.319V284.019H764.166V213Z"></path></g></svg> Start your Substack</a><a data-native="true" href="https://substack.com/app/app-store-redirect?utm_campaign=app-marketing&amp;utm_content=web-footer-button" class="footerSubstackCta-v5HWfj getTheApp-Yk3w1O noIcon-z7v9D8">Get the app</a></div><div translated class="pencraft pc-reset reset-IxiVJZ footer-slogan-blurb"><a href="https://substack.com" data-native="true">Substack</a> is the home for great culture</div></div></div></div></div><div role="region" aria-label="Notifications (F8)" tabindex="-1" style="pointer-events:none;"><ol tabindex="-1" style="--offset:0px;z-index:1000;" class="viewport-_BM4Bg"></ol></div><div></div><script suppressHydrationWarning nonce="JWtlbHixuqoee6u-MsbsjA">window.__staticRouterHydrationData = JSON.parse("{\"loaderData\":{},\"actionData\":null,\"errors\":null}");</script>
</div>
<script src="https://js.sentry-cdn.com/6c2ff3e3828e4017b7faf7b63e24cdf8.min.js" crossorigin="anonymous"></script>
<script nonce="JWtlbHixuqoee6u-MsbsjA">
window.Sentry && window.Sentry.onLoad(function() {
window.Sentry.init({
environment: window._preloads.sentry_environment,
dsn: window._preloads.sentry_dsn,
})
})
</script>
<script nonce="JWtlbHixuqoee6u-MsbsjA">window._preloads = JSON.parse("{\"cspNonce\":\"JWtlbHixuqoee6u-MsbsjA\",\"isEU\":false,\"language\":\"en\",\"country\":\"US\",\"leaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"enabledLeaderboardCountries\":[\"ES\",\"FR\",\"GB\",\"IT\",\"NL\"],\"userLocale\":{\"language\":\"en\",\"region\":\"US\",\"source\":\"accept-language\"},\"base_url\":\"https://newsletter.betterstack.com\",\"stripe_publishable_key\":\"pk_live_51QfnARLDSWi1i85FBpvw6YxfQHljOpWXw8IKi5qFWEzvW8HvoD8cqTulR9UWguYbYweLvA16P7LN6WZsGdZKrNkE00uGbFaOE3\",\"captcha_site_key\":\"6LeI15YsAAAAAPXyDcvuVqipba_jEFQCjz1PFQoz\",\"pub\":{\"apple_pay_disabled\":false,\"apex_domain\":null,\"author_id\":131972456,\"byline_images_enabled\":true,\"bylines_enabled\":true,\"chartable_token\":null,\"community_enabled\":true,\"copyright\":\"Better Stack\",\"cover_photo_url\":null,\"created_at\":\"2024-01-18T12:37:43.322Z\",\"custom_domain_optional\":false,\"custom_domain\":\"newsletter.betterstack.com\",\"default_comment_sort\":\"best_first\",\"default_coupon\":null,\"default_group_coupon\":null,\"default_show_guest_bios\":true,\"email_banner_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/599d67dc-9563-4691-9a1f-f08c461a4f33_1100x220.png\",\"email_from_name\":\"Hacking Scale\",\"email_from\":null,\"embed_tracking_disabled\":false,\"explicit\":false,\"expose_paywall_content_to_search_engines\":true,\"fb_pixel_id\":\"1062799887416376\",\"fb_site_verification_token\":null,\"flagged_as_spam\":false,\"founding_subscription_benefits\":null,\"free_subscription_benefits\":null,\"ga_pixel_id\":null,\"google_site_verification_token\":null,\"google_tag_manager_token\":null,\"hero_image\":null,\"hero_text\":\"A weekly newsletter about building and scaling software from engineers at Better Stack\",\"hide_intro_subtitle\":null,\"hide_intro_title\":null,\"hide_podcast_feed_link\":false,\"homepage_type\":\"newspaper\",\"id\":2269685,\"image_thumbnails_always_enabled\":false,\"invite_only\":false,\"hide_podcast_from_pub_listings\":false,\"hide_podcast_from_pub_listings_prior_to\":null,\"language\":\"en\",\"logo_url_wide\":null,\"logo_url\":\"https://substackcdn.com/image/fetch/$s_!_FGI!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F87b62237-9eb2-480e-abda-ac1ffc34a163_256x256.png\",\"minimum_group_size\":2,\"moderation_enabled\":true,\"name\":\"Hacking Scale by Better Stack\",\"paid_subscription_benefits\":null,\"parsely_pixel_id\":null,\"chartbeat_domain\":null,\"payments_state\":\"disabled\",\"paywall_free_trial_enabled\":false,\"podcast_art_url\":null,\"paid_podcast_episode_art_url\":null,\"podcast_byline\":null,\"podcast_description\":null,\"podcast_enabled\":false,\"podcast_feed_url\":null,\"podcast_title\":null,\"post_preview_limit\":null,\"primary_user_id\":131972456,\"require_clickthrough\":false,\"show_pub_podcast_tab\":false,\"show_recs_on_homepage\":true,\"subdomain\":\"betterstack\",\"subscriber_invites\":0,\"support_email\":null,\"theme_var_background_pop\":\"#FF81CD\",\"theme_var_color_links\":true,\"theme_var_cover_bg_color\":null,\"trial_end_override\":null,\"twitter_pixel_id\":\"o7hjd\",\"type\":\"newsletter\",\"post_reaction_faces_enabled\":true,\"is_personal_mode\":false,\"plans\":null,\"stripe_user_id\":null,\"stripe_country\":null,\"stripe_publishable_key\":null,\"stripe_platform_account\":null,\"automatic_tax_enabled\":null,\"author_name\":\"Better Stack\",\"author_handle\":\"betterstack\",\"author_photo_url\":\"https://substackcdn.com/image/fetch/$s_!pJy8!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F40a082fe-9310-4b0a-824f-c48109eb31d1_144x144.png\",\"author_bio\":null,\"has_custom_tos\":false,\"has_custom_privacy\":false,\"theme\":{\"background_pop_color\":\"#5b63d3\",\"web_bg_color\":\"#ffffff\",\"cover_bg_color\":null,\"publication_id\":2269685,\"color_links\":null,\"font_preset_heading\":null,\"font_preset_body\":\"sans\",\"font_family_headings\":null,\"font_family_body\":null,\"font_family_ui\":null,\"font_size_body_desktop\":null,\"print_secondary\":null,\"custom_css_web\":null,\"custom_css_email\":null,\"home_hero\":\"newspaper\",\"home_posts\":\"grid\",\"home_show_top_posts\":false,\"hide_images_from_list\":false,\"home_hero_alignment\":\"left\",\"home_hero_show_podcast_links\":true,\"default_post_header_variant\":null,\"custom_header\":null,\"custom_footer\":null,\"social_media_links\":null,\"font_options\":null,\"section_template\":null,\"custom_subscribe\":null,\"design_template\":null,\"design_template_options\":null},\"threads_v2_settings\":{\"first_thread_email_sent_at\":null,\"activated_at\":null,\"reader_thread_notifications_enabled\":true,\"create_thread_minimum_role\":\"free_subscriber\",\"photo_replies_enabled\":true,\"boost_free_subscriber_chat_preview_enabled\":false,\"push_suppression_enabled\":false},\"default_group_coupon_percent_off\":null,\"default_group_coupon_include_founding\":null,\"pause_return_date\":null,\"has_posts\":true,\"has_recommendations\":true,\"first_post_date\":\"2024-07-31T13:54:05.177Z\",\"has_podcast\":false,\"has_free_podcast\":false,\"has_subscriber_only_podcast\":false,\"has_community_content\":false,\"rankingDetail\":\"Launched 2 years ago\",\"rankingDetailFreeIncluded\":\"Thousands of subscribers\",\"rankingDetailOrderOfMagnitude\":0,\"rankingDetailFreeIncludedOrderOfMagnitude\":1000,\"rankingDetailFreeSubscriberCount\":\"Over 8,000 subscribers\",\"rankingDetailByLanguage\":{\"ar\":{\"rankingDetail\":\"\u062A\u0645 \u0627\u0644\u0625\u0637\u0644\u0627\u0642 2 years ago\"},\"ca\":{\"rankingDetail\":\"S\u2019ha llan\u00E7at fa 2 anys\"},\"da\":{\"rankingDetail\":\"Lancering 2 \u00E5r\"},\"de\":{\"rankingDetail\":\"vor 2 Jahren gelauncht\"},\"es\":{\"rankingDetail\":\"Lanzado hace 2 a\u00F1os\"},\"fr\":{\"rankingDetail\":\"Lanc\u00E9 il y a 2 ann\u00E9es\"},\"id\":{\"rankingDetail\":\"Diluncurkan 2 tahun yang lalu\"},\"ja\":{\"rankingDetail\":\"\u958B\u59CB\u65E5 2\u5E74\u524D\"},\"ko\":{\"rankingDetail\":\"2\uB144 \uC804\uC5D0 \uCD9C\uC2DC\"},\"nb\":{\"rankingDetail\":\"Lansert 2 \u00E5r\"},\"nl\":{\"rankingDetail\":\"Gelanceerd 2 jaar geleden\"},\"pl\":{\"rankingDetail\":\"Uruchomiono 2 lat temu\"},\"pt\":{\"rankingDetail\":\"Lan\u00E7ado 2 anos\"},\"pt-br\":{\"rankingDetail\":\"Lan\u00E7ado 2 anos\"},\"en-gb\":{\"rankingDetail\":\"Launched 2 years ago\"},\"it\":{\"rankingDetail\":\"Lanciato 2 anni\"},\"tr\":{\"rankingDetail\":\"2 y\u0131l ba\u015Flat\u0131ld\u0131\"},\"sv\":{\"rankingDetail\":\"Lanserad 2 \u00E5r sedan\"},\"fi\":{\"rankingDetail\":\"Launched 2 vuotta\"},\"is\":{\"rankingDetail\":\"Launched 2 \u00E1r\"},\"zh-hans\":{\"rankingDetail\":\"\u4E8E2 \u5E74\u524D\u4E0A\u7EBF\"},\"zh-hant\":{\"rankingDetail\":\"\u65BC 2 \u5E74\u524D \u63A8\u51FA\"},\"en\":{\"rankingDetail\":\"Launched 2 years ago\"}},\"freeSubscriberCount\":\"8,000\",\"freeSubscriberCountOrderOfMagnitude\":\"8.2K+\",\"author_bestseller_tier\":0,\"author_badge\":null,\"disable_monthly_subscriptions\":false,\"disable_annual_subscriptions\":false,\"hide_post_restacks\":false,\"notes_feed_enabled\":true,\"showIntroModule\":false,\"isPortraitLayout\":false,\"last_chat_post_at\":null,\"primary_profile_name\":\"Better Stack\",\"primary_profile_photo_url\":\"https://substackcdn.com/image/fetch/$s_!pJy8!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F40a082fe-9310-4b0a-824f-c48109eb31d1_144x144.png\",\"no_follow\":false,\"sponsorshipCampaigns\":{},\"paywall_chat\":\"free\",\"sections\":[],\"podcastTabInfo\":null,\"multipub_migration\":null,\"navigationBarItems\":[{\"id\":\"f57d0274-9d4e-41d3-9f42-dfda420ee5b0\",\"publication_id\":2269685,\"sibling_rank\":0,\"link_title\":null,\"link_url\":null,\"section_id\":null,\"post_id\":null,\"is_hidden\":false,\"standard_key\":\"leaderboard\",\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"post\":null,\"section\":null,\"postTag\":null,\"children\":[]},{\"id\":\"f8f279f5-128c-4fcd-8d01-05adf7503678\",\"publication_id\":2269685,\"sibling_rank\":0,\"link_title\":null,\"link_url\":null,\"section_id\":null,\"post_id\":null,\"is_hidden\":true,\"standard_key\":\"about\",\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"post\":null,\"section\":null,\"postTag\":null,\"children\":[]},{\"id\":\"7986aab3-2e1e-4b8c-a17a-6873b0d892ec\",\"publication_id\":2269685,\"sibling_rank\":0,\"link_title\":null,\"link_url\":null,\"section_id\":null,\"post_id\":null,\"is_hidden\":false,\"standard_key\":\"archive\",\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"post\":null,\"section\":null,\"postTag\":null,\"children\":[]},{\"id\":\"7f1ffc4b-447d-44b3-8418-5a9e87791359\",\"publication_id\":2269685,\"sibling_rank\":3,\"link_title\":\"Made by Better Stack\",\"link_url\":\"https://betterstack.com\",\"section_id\":null,\"post_id\":null,\"is_hidden\":null,\"standard_key\":null,\"post_tag_id\":null,\"parent_id\":null,\"is_group\":false,\"post\":null,\"section\":null,\"postTag\":null,\"children\":[]}],\"has_active_perks\":false,\"contributors\":[{\"name\":\"Better Stack\",\"handle\":\"betterstack\",\"role\":\"admin\",\"owner\":true,\"user_id\":131972456,\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/40a082fe-9310-4b0a-824f-c48109eb31d1_144x144.png\",\"bio\":null,\"status\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null}},{\"name\":\"Richard Oliver Bray\",\"handle\":\"richardoliverbray\",\"role\":\"admin\",\"owner\":false,\"user_id\":251681141,\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/2cfb44c1-5ecd-42e3-a7e9-f08a8ad4ad51_1424x1254.png\",\"bio\":null,\"status\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null}}],\"threads_v2_enabled\":true,\"viralGiftsConfig\":null,\"no_index\":false,\"tier\":2,\"can_set_google_site_verification\":true,\"can_have_sitemap\":true,\"founding_plan_name_english\":\"Founding Member\",\"base_url\":\"https://newsletter.betterstack.com\",\"hostname\":\"newsletter.betterstack.com\",\"is_on_substack\":false,\"spotify_podcast_settings\":null,\"unified_podcast_settings\":null,\"podcastPalette\":{\"DarkMuted\":{\"population\":72,\"rgb\":[73,153,137]},\"DarkVibrant\":{\"population\":6013,\"rgb\":[4,100,84]},\"LightMuted\":{\"population\":7,\"rgb\":[142,198,186]},\"LightVibrant\":{\"population\":3,\"rgb\":[166,214,206]},\"Muted\":{\"population\":6,\"rgb\":[92,164,156]},\"Vibrant\":{\"population\":5,\"rgb\":[76,164,146]}},\"pageThemes\":{\"podcast\":null},\"supports_ip_content_unlock\":false,\"appTheme\":{\"colors\":{\"accent\":{\"name\":\"#5b63d3\",\"primary\":{\"r\":91,\"g\":99,\"b\":211,\"a\":1},\"primary_hover\":{\"r\":70,\"g\":82,\"b\":191,\"a\":1},\"primary_elevated\":{\"r\":70,\"g\":82,\"b\":191,\"a\":1},\"secondary\":{\"r\":91,\"g\":99,\"b\":211,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":91,\"g\":99,\"b\":211,\"a\":0.2},\"bg_hover\":{\"r\":91,\"g\":99,\"b\":211,\"a\":0.3},\"dark\":{\"primary\":{\"r\":91,\"g\":99,\"b\":211,\"a\":1},\"primary_hover\":{\"r\":111,\"g\":116,\"b\":231,\"a\":1},\"primary_elevated\":{\"r\":111,\"g\":116,\"b\":231,\"a\":1},\"secondary\":{\"r\":91,\"g\":99,\"b\":211,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":91,\"g\":99,\"b\":211,\"a\":0.2},\"bg_hover\":{\"r\":91,\"g\":99,\"b\":211,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.8},\"secondary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.6},\"tertiary\":{\"r\":0,\"g\":0,\"b\":0,\"a\":0.4},\"accent\":{\"r\":91,\"g\":99,\"b\":211,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":111,\"g\":116,\"b\":231,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"primary_elevated\":{\"r\":250,\"g\":250,\"b\":250,\"a\":1},\"secondary\":{\"r\":238,\"g\":238,\"b\":238,\"a\":1},\"secondary_elevated\":{\"r\":206.90096477355226,\"g\":206.90096477355175,\"b\":206.9009647735519,\"a\":1},\"tertiary\":{\"r\":219,\"g\":219,\"b\":219,\"a\":1},\"quaternary\":{\"r\":182,\"g\":182,\"b\":182,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}}}},\"portalAppTheme\":{\"colors\":{\"accent\":{\"name\":\"#5b63d3\",\"primary\":{\"r\":91,\"g\":99,\"b\":211,\"a\":1},\"primary_hover\":{\"r\":71,\"g\":80,\"b\":206,\"a\":1},\"primary_elevated\":{\"r\":91,\"g\":99,\"b\":211,\"a\":1},\"secondary\":{\"r\":91,\"g\":99,\"b\":211,\"a\":1},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.2},\"bg_hover\":{\"r\":255,\"g\":103,\"b\":25,\"a\":0.3},\"dark\":{\"primary\":{\"r\":91,\"g\":99,\"b\":211,\"a\":1},\"primary_hover\":{\"r\":111,\"g\":116,\"b\":231,\"a\":1},\"primary_elevated\":{\"r\":111,\"g\":116,\"b\":231,\"a\":1},\"secondary\":{\"r\":91,\"g\":99,\"b\":211,\"a\":0.2},\"contrast\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"bg\":{\"r\":91,\"g\":99,\"b\":211,\"a\":0.2},\"bg_hover\":{\"r\":91,\"g\":99,\"b\":211,\"a\":0.3}}},\"fg\":{\"primary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"secondary\":{\"r\":134,\"g\":135,\"b\":135,\"a\":1},\"tertiary\":{\"r\":146,\"g\":146,\"b\":146,\"a\":1},\"accent\":{\"r\":91,\"g\":99,\"b\":211,\"a\":1},\"dark\":{\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.9},\"secondary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.6},\"tertiary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":0.4},\"accent\":{\"r\":111,\"g\":116,\"b\":231,\"a\":1}}},\"bg\":{\"name\":\"#ffffff\",\"hue\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"tint\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"primary_hover\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"primary_elevated\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"secondary\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"secondary_elevated\":{\"r\":240,\"g\":240,\"b\":240,\"a\":1},\"tertiary\":{\"r\":221,\"g\":221,\"b\":221,\"a\":1},\"quaternary\":{\"r\":183,\"g\":183,\"b\":183,\"a\":1},\"dark\":{\"primary\":{\"r\":22,\"g\":23,\"b\":24,\"a\":1},\"primary_hover\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"primary_elevated\":{\"r\":27,\"g\":28,\"b\":29,\"a\":1},\"secondary\":{\"r\":35,\"g\":37,\"b\":37,\"a\":1},\"secondary_elevated\":{\"r\":41.35899397549579,\"g\":43.405356429195315,\"b\":43.40489285041963,\"a\":1},\"tertiary\":{\"r\":54,\"g\":55,\"b\":55,\"a\":1},\"quaternary\":{\"r\":90,\"g\":91,\"b\":91,\"a\":1}}},\"wordmark_bg\":{\"r\":255,\"g\":255,\"b\":255,\"a\":1},\"is_dark\":false},\"fonts\":{\"body\":\"sans\"}},\"logoPalette\":{\"Vibrant\":{\"rgb\":[5,245,246],\"population\":210},\"DarkVibrant\":{\"rgb\":[9,105,111],\"population\":91},\"LightVibrant\":{\"rgb\":[100,248,248],\"population\":92},\"Muted\":{\"rgb\":[105,165,168],\"population\":13},\"DarkMuted\":{\"rgb\":[94,54,54],\"population\":3},\"LightMuted\":{\"rgb\":[230,198,198],\"population\":4}}},\"confirmedLogin\":false,\"hide_intro_popup\":false,\"block_auto_login\":false,\"domainInfo\":{\"isSubstack\":false,\"customDomain\":\"newsletter.betterstack.com\"},\"experimentFeatures\":{\"web_onboarding_content_type_preferences\":\"control\"},\"experimentExposures\":{},\"siteConfigs\":{\"score_upsell_email\":\"control\",\"first_chat_email_enabled\":true,\"new_commenter_approval\":false,\"pub_update_opennode_api_key\":false,\"notes_video_max_duration_minutes\":15,\"show_content_label_age_gating_in_feed\":false,\"zendesk_automation_cancellations\":false,\"mfa_action_box_enabled\":false,\"publication_max_bylines\":35,\"no_contest_charge_disputes\":false,\"scheduled_post_share_assets_enabled\":false,\"publication_tabs_reorder\":false,\"comp_expiry_email_new_copy\":\"NONE\",\"free_unlock_required\":false,\"enable_section_group_pages\":false,\"enable_post_summarization\":false,\"live_stream_host_warning_message\":\"\",\"bitcoin_enabled\":false,\"minimum_ios_os_version\":\"17.0.0\",\"show_entire_square_image\":false,\"hide_subscriber_count\":false,\"fit_in_live_stream_player\":false,\"publication_author_display_override\":\"\",\"enable_saved_segments\":true,\"generate_pdf_tax_report\":false,\"hide_post_sidebar\":false,\"show_generic_post_importer\":false,\"enable_pledges_modal\":true,\"community_automod_non_english_enabled\":true,\"audio_clips_enabled\":false,\"enable_videos_page\":false,\"exempt_from_gtm_filter\":false,\"group_sections_and_podcasts_in_menu\":false,\"boost_optin_modal_enabled\":true,\"standards_and_enforcement_features_enabled\":false,\"pub_creation_captcha_behavior\":\"risky_pubs_or_rate_limit\",\"post_blogspot_importer\":false,\"enable_high_res_background_uploading\":false,\"pub_tts_override\":\"default\",\"disable_monthly_subscriptions\":false,\"skip_welcome_email\":false,\"chat_reader_thread_notification_default\":false,\"scheduled_pinned_posts\":false,\"disable_redirect_outbound_utm_params\":false,\"reader_gift_referrals_enabled\":true,\"dont_show_guest_byline\":false,\"like_comments_enabled\":true,\"enable_theme_editor_deferred_saves\":false,\"enable_sponsorship_campaigns_advanced\":false,\"temporal_livestream_ended_draft\":true,\"enable_pangram_ai_detection\":true,\"fallback_to_archive_search_on_section_pages\":false,\"livekit_track_egress_custom_base_url\":\"http://livekit-egress-custom-recorder-participant-test.s3-website-us-east-1.amazonaws.com\",\"welcome_screen_blurb_override\":\"\",\"like_posts_enabled\":true,\"twitter_player_card_enabled\":true,\"feed_promoted_user\":false,\"show_note_stats_for_all_notes\":false,\"section_specific_csv_imports_enabled\":false,\"disable_podcast_feed_description_cta\":false,\"bypass_profile_substack_logo_detection\":false,\"use_preloaded_player_sources\":false,\"list_pruning_enabled\":false,\"facebook_connect\":false,\"opt_in_to_sections_during_subscribe\":false,\"underlined_colored_links\":false,\"enable_efficient_digest_embed\":false,\"enable_aligned_images\":false,\"max_image_upload_mb\":64,\"threads_suggested_ios_version\":null,\"pledges_disabled\":false,\"threads_minimum_ios_version\":812,\"hide_podcast_email_setup_link\":false,\"subscribe_captcha_behavior\":\"default\",\"publication_ban_sample_rate\":0,\"custom_themes_substack_subscribe_modal\":false,\"ios_post_share_assets_screenshot_trigger\":\"control\",\"opt_in_to_sections_during_subscribe_include_main_pub_newsletter\":false,\"continue_support_cta_in_newsletter_emails\":false,\"bloomberg_syndication_enabled\":false,\"allow_document_freeze\":false,\"test_age_gate_user\":false,\"podcast_main_feed_is_firehose\":false,\"pub_app_incentive_gift\":\"\",\"no_embed_redirect\":false,\"customized_email_from_name_for_new_follow_emails\":\"treatment\",\"spotify_open_access_sandbox_mode\":false,\"disable_custom_nav_menu\":false,\"enable_livestream_name_cards\":false,\"fullstory_enabled\":false,\"chat_reply_poll_interval\":3,\"thefp_enable_email_upsell_banner\":false,\"force_pub_links_to_use_subdomain\":false,\"always_show_cookie_banner\":false,\"hide_media_download_option\":false,\"hide_post_restacks\":false,\"feed_item_source_debug_mode\":false,\"twitter_embed_video_playback_enabled\":false,\"publication_homepage_title_display_override\":\"\",\"enable_email_automation_stats\":false,\"post_preview_highlight_byline\":false,\"4k_video\":false,\"enable_islands_section_intent_screen\":false,\"post_metering_enabled\":false,\"notifications_disabled\":\"\",\"cross_post_notification_threshold\":1000,\"facebook_connect_prod_app\":true,\"force_into_pymk_ranking\":false,\"minimum_android_version\":756,\"group_size_change_notification_email_enabled\":false,\"enable_transcription_translations\":false,\"use_og_image_as_twitter_image_for_post_previews\":false,\"always_use_podcast_channel_art_as_episode_art_in_rss\":false,\"seo_tier_override\":\"NONE\",\"editor_role_enabled\":false,\"no_follow_links\":false,\"publisher_api_enabled\":false,\"zendesk_support_priority\":\"default\",\"enable_post_clips_stats\":false,\"enable_subscriber_referrals_awards\":true,\"ios_profile_themes_feed_permalink_enabled\":false,\"reader_survey_during_subscribe_enabled\":true,\"use_publication_language_for_transcription\":false,\"show_substack_funded_gifts_tooltip\":true,\"disable_ai_transcription\":false,\"per_pub_notification_settings\":false,\"live_stream_creation_enabled\":true,\"disable_card_element_in_europe\":false,\"web_growth_item_promotion_threshold\":0,\"bundle_subscribe_enabled\":false,\"web_vitals_sample_rate\":0,\"allow_live_stream_auto_takedown\":\"true\",\"mobile_publication_attachments_enabled\":false,\"enable_design_templates_events_page\":false,\"enable_survey_response_crm_filters\":false,\"disable_personal_substack_initialization\":false,\"section_specific_welcome_pages\":false,\"local_payment_methods\":\"control\",\"publisher_api_cancel_comp\":false,\"posts_in_rss_feed\":20,\"publisher_dashboard_section_selector\":false,\"reader_surveys_platform_question_order\":\"36,1,4,2,3,5,6,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35\",\"login_guard_app_link_in_email\":true,\"community_moderators_enabled\":false,\"enable_custom_theme\":false,\"monthly_sub_is_one_off\":false,\"unread_notes_activity_digest\":\"control\",\"display_cookie_settings\":false,\"welcome_page_query_params\":false,\"enable_free_podcast_urls\":false,\"comp_expiry_emails_disabled\":false,\"enable_description_on_polls\":false,\"use_microlink_for_instagram_embeds\":false,\"free_signup_confirmation_behavior\":\"with_email_validation\",\"ios_post_stats_for_admins\":false,\"enable_design_templates\":false,\"use_livestream_post_media_composition\":true,\"section_specific_preambles\":false,\"pub_export_temp_disable\":false,\"show_menu_on_posts\":false,\"ios_post_subscribe_web_routing\":true,\"enable_pangram_detection_settings\":false,\"freewalls_enabled_web\":false,\"app_onboarding_survey_email\":false,\"republishing_enabled\":false,\"app_mode\":false,\"show_phone_banner\":false,\"minimum_ios_version\":2200,\"enable_author_pages\":false,\"enable_decagon_chat\":true,\"first_month_upsell\":\"control\",\"new_user_checklist_enabled\":\"use_follower_count\",\"disable_high_res_recording\":false,\"latex_upgraded_inline\":false,\"anonymous_post_metering_enabled\":false,\"rss_verification_code\":\"\",\"notification_post_emails\":\"experiment\",\"chat_suppress_contributor_push_option_enabled\":false,\"enable_subscriber_tags\":true,\"media_feed_app_prepend_inbox_limit\":0,\"export_hooks_enabled\":false,\"audio_encoding_bitrate\":null,\"extra_seats_coupon_type\":false,\"post_subdomain_universal_links\":false,\"post_import_max_file_size\":26214400,\"feed_promoted_video_publication\":false,\"livekit_reconnect_slate_url\":\"https://mux-livestream-assets.s3.us-east-1.amazonaws.com/custom-disconnect-slate-tall.png\",\"exclude_from_pymk_suggestions\":false,\"publication_ranking_variant\":\"experiment\",\"disable_annual_subscriptions\":false,\"hack_jane_manchun_wong\":true,\"android_enable_auto_gain_control\":true,\"allow_coupons_on_upgrade\":false,\"test_au_age_gate_user\":false,\"pub_auto_moderation_enabled\":false,\"disable_live_stream_ai_trimming_by_default\":false,\"disable_deletion\":false,\"livekit_egress_custom_base_url\":\"http://livekit-egress-custom-recorder.s3-website-us-east-1.amazonaws.com\",\"clip_focused_video_upload_flow\":false,\"live_stream_max_guest_users\":2,\"android_upgrade_alert_dialog_reincarnated\":true,\"enable_video_seo_data\":false,\"can_reimport_unsubscribed_users_with_2x_optin\":false,\"founding_upgrade_during_gift_disabled\":false,\"thefp_enable_section_groups\":false,\"review_incoming_email\":\"default\",\"enable_founding_gifts\":false,\"enable_sponsorship_campaigns\":false,\"thefp_forum_polls_enabled\":false,\"thefp_enable_embed_media_links\":false,\"sort_modal_search_results\":false,\"default_thumbnail_time\":10,\"load_test_unichat\":false,\"live_stream_head_alignment_guide\":false,\"free_press_combo_subscribe_flow_enabled\":false,\"enable_code_embed_toc\":false,\"enable_publication_tax_settings\":false,\"gifts_from_substack_feature_available\":true,\"disable_ai_clips\":false,\"enable_elevenlabs_voiceovers\":false,\"thefp_enable_transcripts\":false,\"show_simple_post_editor\":false,\"instacart_integration_enabled\":false,\"enable_publication_podcasts_page\":false,\"enable_direct_message_request_bypass\":false,\"enable_apple_news_sync\":false,\"live_stream_in_trending_topic_overrides\":\"\",\"free_press_newsletter_promo_enabled\":false,\"account_based_post_metering_enabled\":false,\"disable_live_stream_reactions\":false,\"instacart_partner_id\":\"\",\"enable_adhoc_email_scheduling\":true,\"media_feed_prepend_inbox_limit\":35,\"welcome_page_no_opt_out\":false,\"enable_caption_translation_tools\":false,\"section_specific_postscripts\":false,\"edit_profile_theme_colors\":false,\"disable_clipping_for_readers\":false,\"enable_apple_podcast_delivery_self_serve_pem_files\":false,\"enable_press_kit_preview_modal\":false,\"enable_rnc_page\":true,\"live_stream_video_degradation_preference\":\"maintainFramerate\",\"enable_high_follower_dm\":true,\"pause_app_badges\":false,\"profile_feed_expanded_inventory\":false,\"livekit_mux_latency_mode\":\"low\",\"feed_juiced_user\":0,\"global_iap_plan_setup_countries\":\"AE,AG,AL,AM,AT,AU,BA,BG,BH,BJ,BN,BS,BT,BW,CH,CI,CR,CY,CZ,DK,DZ,EC,EE,ET,FI,GM,GT,GY,HK,HR,HU,ID,IL,IN,IS,JM,JO,KE,KW,LC,LI,LK,LT,LU,LV,MA,MD,MG,MK,MN,MT,MU,MX,MY,MZ,NA,NO,NZ,OM,PA,PE,PH,PK,QA,RO,RS,RW,SE,SG,SI,SK,SN,SV,TH,TN,TR,TT,TW,TZ,UZ,VN,ZA\",\"enable_collapsed_content\":false,\"enable_polymarket_expandable_embeds\":true,\"publisher_dashboard_group_subs_sidebar\":false,\"use_advanced_commerce_api_for_iap\":false,\"skip_free_preview_language_in_podcast_notes\":false,\"larger_wordmark_on_publication_homepage\":false,\"enable_mobile_stats_for_admins\":false,\"enable_livestream_youtube_simulcast\":false,\"enable_persona_sandbox_environment\":false,\"allowed_email_domains\":\"mozmail.com\",\"create_nav_item_from_tag\":false,\"bypass_single_unlock_token_limit\":false,\"enable_code_embeds\":false,\"add_section_and_tag_metadata\":false,\"daily_promoted_notes_enabled\":true,\"enable_segments_ui_revamp\":true,\"enable_islands_cms\":false,\"enable_livestream_combined_stats\":false,\"redirect_to_pub_after_welcome_signup\":false,\"chartbeat_video_enabled\":false,\"enable_drip_campaigns\":false,\"enable_filtered_posts_on_homepage\":false,\"post_management_search_engine\":\"elasticsearch\",\"new_bestseller_leaderboard_feed_item_enabled\":false,\"feed_main_disabled\":false,\"enable_account_settings_revamp\":false,\"thefp_enable_fp_recirc_block\":false,\"top_search_variant\":\"control\",\"enable_debug_logs_ios\":false,\"show_pub_content_on_profile_for_pub_id\":0,\"show_pub_content_on_profile\":false,\"livekit_track_egress\":true,\"onboarding_suggestions_search\":\"experiment\",\"feed_tuner_enabled\":false,\"livekit_mux_latency_mode_rtmp\":\"low\",\"google_login_enabled\":false,\"livekit_high_quality_egress\":false,\"iap_announcement_blog_url\":\"\",\"disable_code_embed_chrome\":false,\"ios_livestream_feedback\":false,\"founding_plan_upgrade_warning\":false,\"ios_mediaplayer_reply_bar_v2\":false,\"enable_notification_email_batching\":true,\"note_author_poll_results_visible_by_default\":true,\"ios_hide_portal_tab_bar\":false,\"follow_upsell_rollout_percentage\":30,\"gift_article_enabled\":false,\"live_stream_invite_ttl_seconds\":900000,\"include_founding_plans_coupon_option\":false,\"thefp_enable_cancellation_discount_offer\":false,\"android_enable_edit_profile_theme\":false,\"disable_audio_enhancement\":false,\"pub_search_variant\":\"control\",\"ignore_video_in_notes_length_limit\":false,\"substack_gtm_enabled\":false,\"allow_long_videos\":true,\"enable_automation_filters\":false,\"use_enhanced_video_embed_player\":true,\"founding_upgrade_preview\":false,\"enable_viewing_all_livestream_viewers\":false,\"send_subscription_canceled_email\":false,\"enable_clip_prompt_variant_filtering\":true,\"chartbeat_enabled\":false,\"dpn_ranking_enabled\":true,\"enable_custom_email_css\":false,\"thefp_enable_shows_page\":false,\"enable_apple_podcast_auto_publish\":false,\"linkedin_profile_search_enabled\":false,\"publication_has_own_app\":false,\"suggested_minimum_ios_version\":0,\"thebulwark_enable_footer_actions\":false,\"enable_posts_to_segments\":true,\"trending_topics_module_long_term_experiment\":\"control\",\"phone_verification_fallback_to_twilio\":false,\"enable_suggested_searches\":true,\"enable_subscription_notification_email_batching\":true,\"feed_controls\":\"control\",\"thefp_enable_plans_modal_v2\":false,\"a24_redemption_link\":\"\",\"design_templates_launch_phase\":\"off\",\"post_custom_email_from_name_enabled\":false,\"ios_live_stream_auto_gain_enabled\":true,\"enable_publication_tts_player\":false,\"enable_group_direct_messages\":true,\"enable_notes_admins\":false,\"thefp_show_pub_app_callout_on_post\":false,\"search_ranker_variant\":\"experiment\",\"forced_featured_topic_id\":\"\",\"ios_audio_captions_disabled\":false,\"web_podcasts_tab\":false,\"related_posts_enabled\":false,\"thefp_enable_homepage_secondary_nav\":false,\"ios_live_stream_pip_dismiss_v4\":\"control\",\"reply_rate_limit_max_distinct_users_daily\":110,\"web_post_metering_enabled\":true,\"search_retrieval_variant\":\"experiment\",\"mobile_user_attachments_enabled\":false,\"web_profile_publication_recommendations_carousel\":true,\"publisher_banner\":\"\",\"enable_sponsorship_profile\":false,\"gift_article_button_enabled\":false,\"reply_rate_limit_max_distinct_users_monthly\":600,\"desktop_live_stream_screen_share_audio_enabled\":false,\"creator_match\":true,\"ai_image_generation_enabled\":true,\"ios_default_coupon_enabled\":true,\"automod_admin_hidden_comments_tab\":true,\"video_clip_suggestions_enabled\":false,\"publisher_dashboard_home_layout_experiment\":\"control\",\"android_polymarket_embed_search\":false,\"audience_section_editor_help_link\":\"https://support.substack.com/hc/en-us/articles/50558240901268-How-do-I-add-audience-specific-content-to-a-post-on-Substack\",\"portal_ranking_variant\":\"experiment\",\"permalink_reply_ranking_variant\":\"experiment\",\"allow_feed_category_filtering\":false,\"enable_dynamic_content\":true,\"enable_thumbnail_editor_media_settings_integration\":false,\"subscriber_profile_survey_responses_enabled\":true,\"private_live_streaming_enabled\":true,\"desktop_live_stream_safe_framing\":0.8,\"translate_posts_on_standalone_sites\":true,\"bundle_subscribe_default_to_regular_plan\":false,\"enable_standardized_sponsorship_formats\":false,\"enable_pencraft_signin_form\":true,\"thefp_enable_secondary_section_redesign\":false,\"bignight_page_enabled\":true,\"thefp_paywall_annual_plans_experiment\":\"control\",\"enable_launch_assistant\":false,\"enable_youtube_exploration\":true,\"publisher_dashboard_nav_refresh\":false,\"thefp_enable_bundle_post_follow\":false,\"web_notes_tiptap_v3_experiment\":\"treatment_tiptap_v3\",\"recipe_rich_text_enabled\":true,\"publisher_dashboard_nav_refresh_experiment\":\"treatment\",\"chat_channel_creation_enabled\":false,\"web_onboarding_content_type_preferences\":\"experiment\",\"web_mobile_asset_handoff\":false},\"trackingTokens\":{\"fbPixelId\":\"1062799887416376\",\"gaPixelId\":null,\"twitterPixelId\":\"o7hjd\",\"parselyPixelId\":null,\"googleTagManagerToken\":null,\"googleAnalytics4Token\":\"G-MWWB9F6017\",\"twitterPixelSignupEventId\":null,\"twitterPixelSubscribeEventId\":null},\"publicationSettings\":{\"block_ai_crawlers\":false,\"credit_token_enabled\":false,\"custom_tos_and_privacy\":false,\"did_identity\":null,\"disable_optimistic_bank_payments\":false,\"display_welcome_page_details\":true,\"payment_pledges_enabled\":false,\"enable_drop_caps\":false,\"enable_post_page_conversion\":true,\"enable_prev_next_nav\":true,\"enable_restacking\":true,\"founding_group_subscriptions_enabled\":false,\"gifts_from_substack_disabled\":false,\"google_analytics_4_token\":\"G-MWWB9F6017\",\"group_sections_and_podcasts_in_menu_enabled\":false,\"live_stream_homepage_visibility\":\"contributorsAndAdmins\",\"live_stream_homepage_style\":\"autoPlay\",\"live_stream_replay_enabled\":true,\"medium_length_description\":\"\",\"notes_feed_enabled\":true,\"paywall_unlock_tokens\":false,\"post_preview_crop_gravity\":\"auto\",\"post_preview_radius\":\"xs\",\"reader_referrals_enabled\":true,\"reader_referrals_leaderboard_enabled\":true,\"seen_coming_soon_explainer\":false,\"seen_google_analytics_migration_modal\":false,\"local_currency_modal_seen\":false,\"local_payment_methods_modal_seen\":false,\"twitter_pixel_signup_event_id\":null,\"twitter_pixel_subscribe_event_id\":null,\"use_inclusive_sales_tax\":false,\"use_local_currency\":false,\"welcome_page_opt_out_text\":\"No thanks\",\"cookie_settings\":\"\",\"show_restacks_below_posts\":true,\"holiday_gifting_post_header\":false,\"homepage_message_text\":\"\",\"homepage_message_link\":\"\",\"about_us_author_ids\":\"\",\"archived_section_ids\":\"\",\"column_section_ids\":\"\",\"fp_primary_column_section_ids\":\"\",\"event_section_ids\":\"\",\"podcasts_metadata\":\"\",\"video_section_ids\":\"\",\"post_metering_enabled\":false,\"account_based_post_metering_enabled\":false,\"use_custom_theme\":false,\"reply_rules\":null,\"automatic_moderation_enabled\":true,\"auto_translate_enabled\":true,\"additional_post_languages_enabled\":false,\"high_res_recording_beta\":false},\"publicationUserSettings\":null,\"userSettings\":{\"user_id\":null,\"activity_likes_enabled\":true,\"dashboard_nav_refresh_enabled\":false,\"is_guest_post_enabled\":true,\"invite_friends_nux_dismissed_at\":null,\"suggestions_feed_item_last_shown_at\":null,\"last_notification_alert_shown_at\":null,\"disable_reply_hiding\":false,\"newest_seen_chat_item_published_at\":null,\"explicitContentEnabled\":false,\"contactMatchingEnabled\":false,\"messageRequestLevel\":\"everyone\",\"liveStreamAcceptableInviteLevel\":\"everyone\",\"liveStreamAcceptableChatLevel\":\"everyone\",\"creditTokensTreatmentExposed\":false,\"appBadgeIncludesChat\":false,\"autoPlayVideo\":true,\"autoAdvanceVideo\":true,\"smart_delivery_enabled\":false,\"chatbotTermsLastAcceptedAt\":null,\"has_seen_notes_post_app_upsell\":false,\"first_note_id\":null,\"show_concurrent_live_stream_viewers\":false,\"edit_profile_feed_item_dismissed_at\":null,\"mobile_permalink_app_upsell_seen_at\":null,\"new_user_checklist_enabled\":false,\"has_seen_youtube_shorts_auto_publish_announcement\":false,\"has_seen_publish_youtube_connect_upsell\":false,\"notificationQualityFilterEnabled\":true,\"hasSeenOnboardingNewslettersScreen\":false,\"bestsellerBadgeEnabled\":true,\"hasSelfIdentifiedAsCreator\":false,\"autoTranslateEnabled\":true,\"autoTranslateBlocklist\":[],\"fineTuneFeedItemSeenAt\":null,\"fineTuneFeedItemDismissedAt\":null,\"aiGeneratedTextMode\":\"show\",\"feedVideoPreference\":5,\"feedAudioPreference\":5,\"feedLongerPostsPreference\":5,\"feedChatPreference\":5},\"mux_env_key\":\"u42pci814i6011qg3segrcpp9\",\"persona_environment_id\":\"env_o1Lbk4JhpY4PmvNkwaBdYwe5Fzkt\",\"sentry_environment\":\"production\",\"launchWelcomePage\":false,\"pendingInviteForActiveLiveStream\":null,\"isEligibleForLiveStreamCreation\":true,\"webviewPlatform\":null,\"noIndex\":false,\"post\":{\"audience\":\"everyone\",\"audience_before_archived\":null,\"canonical_url\":\"https://newsletter.betterstack.com/p/how-dropbox-saved-millions-of-dollars\",\"default_comment_sort\":null,\"editor_v2\":false,\"exempt_from_archive_paywall\":false,\"free_unlock_required\":false,\"id\":152558295,\"podcast_art_url\":null,\"podcast_duration\":null,\"podcast_preview_upload_id\":null,\"podcast_upload_id\":null,\"podcast_url\":null,\"post_date\":\"2024-12-11T14:01:03.923Z\",\"updated_at\":\"2024-12-11T14:01:04.016Z\",\"publication_id\":2269685,\"search_engine_description\":null,\"search_engine_title\":null,\"section_id\":null,\"should_send_free_preview\":false,\"show_guest_bios\":true,\"slug\":\"how-dropbox-saved-millions-of-dollars\",\"social_title\":null,\"subtitle\":\"Dropbox saved resources by creating a superior version of a tool everyone uses\",\"teaser_post_eligible\":true,\"title\":\"How Dropbox Saved Millions of Dollars by Building a Load Balancer\",\"type\":\"newsletter\",\"video_upload_id\":null,\"write_comment_permissions\":\"none\",\"meter_type\":\"none\",\"live_stream_id\":null,\"is_published\":true,\"restacks\":1,\"reactions\":{\"\u2764\":17},\"top_exclusions\":[],\"pins\":[],\"section_pins\":[],\"has_shareable_clips\":false,\"previous_post_slug\":\"how-stripe-processed-1-trillion-in\",\"next_post_slug\":\"how-doordash-improved-redis-to-handle\",\"cover_image\":\"https://substackcdn.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png\",\"cover_image_is_square\":false,\"cover_image_is_explicit\":false,\"videoUpload\":null,\"podcastFields\":{\"post_id\":152558295,\"podcast_episode_number\":null,\"podcast_season_number\":null,\"podcast_episode_type\":null,\"should_syndicate_to_other_feed\":null,\"syndicate_to_section_id\":null,\"hide_from_feed\":false,\"free_podcast_url\":null,\"free_podcast_duration\":null,\"preview_contains_ad\":false,\"was_imported_self_serve_sync\":false,\"draft_free_podcast_url\":null,\"draft_free_podcast_duration\":null},\"podcastUpload\":null,\"podcastPreviewUpload\":null,\"voiceover_upload_id\":\"ef91a656-c97e-4643-8189-5761abaa8be2\",\"voiceoverUpload\":{\"id\":\"ef91a656-c97e-4643-8189-5761abaa8be2\",\"name\":\"rh.mp3\",\"created_at\":\"2024-12-11T13:11:27.643Z\",\"uploaded_at\":\"2024-12-11T13:11:34.962Z\",\"publication_id\":2269685,\"state\":\"transcoded\",\"post_id\":152558295,\"user_id\":251681141,\"duration\":480.41797,\"height\":null,\"width\":null,\"thumbnail_id\":1,\"preview_start\":null,\"preview_duration\":null,\"media_type\":\"audio\",\"primary_file_size\":5765374,\"is_mux\":false,\"mux_asset_id\":null,\"mux_playback_id\":null,\"mux_preview_asset_id\":null,\"mux_preview_playback_id\":null,\"mux_rendition_quality\":null,\"mux_preview_rendition_quality\":null,\"explicit\":false,\"copyright_infringement\":null,\"src_media_upload_id\":null,\"live_stream_id\":null},\"has_voiceover\":true,\"description\":\"Dropbox saved resources by creating a superior version of a tool everyone uses\",\"body_html\":\"<p><strong>Dropbox</strong> is a <strong>cloud-based storage service</strong> that is ridiculously easy to use.</p><p>Download the app and drag your files into the newly created folder. That's it; your files are in the cloud and can be accessed from anywhere.</p><p>It sounds like a simple idea, but back in 2007, when it was released, there wasn't anything like it.</p><p>Today, Dropbox has around <strong>700 million users</strong> and stores over <strong>550 billion files</strong>.</p><p>All these files need to be <strong>organized, backed up, and accessible from anywhere</strong>. Dropbox uses <strong>virtual servers</strong> for this. But they often got <strong>overloaded</strong> and sometimes crashed.</p><p>So, the team at Dropbox built a solution to <strong>manage server loads</strong>.</p><p>Here's how they did it.</p><p><em>Estimated reading time: 5 minutes 15 seconds</em></p><div><hr></div><div class=\\\"subscription-widget-wrap-editor\\\" data-attrs=\\\"{&quot;url&quot;:&quot;https://newsletter.betterstack.com/subscribe?&quot;,&quot;text&quot;:&quot;Subscribe&quot;,&quot;language&quot;:&quot;en&quot;}\\\" data-component-name=\\\"SubscribeWidgetToDOM\\\"><div class=\\\"subscription-widget show-subscribe\\\"><div class=\\\"preamble\\\"><p class=\\\"cta-caption\\\">We do tech deep dives like this <strong>every 2 weeks</strong>. Get the next one sent straight to your inbox.</p></div><form class=\\\"subscription-widget-subscribe\\\"><input type=\\\"email\\\" class=\\\"email-input\\\" name=\\\"email\\\" placeholder=\\\"Type your email\u2026\\\" tabindex=\\\"-1\\\"><input type=\\\"submit\\\" class=\\\"button primary\\\" value=\\\"Subscribe\\\"><div class=\\\"fake-input-wrapper\\\"><div class=\\\"fake-input\\\"></div><div class=\\\"fake-button\\\"></div></div></form></div></div><div><hr></div><h2>Why Dropbox Servers Were Overloaded</h2><p>Before Dropbox grew in scale, they <strong>used a traditional system</strong> to balance load.</p><p>This likely used a <strong>round-robin algorithm with fixed weights</strong>.</p><p>So, a user or client would upload a file. The load balancer would forward the upload request to a server. Then, that server would upload the file and store it correctly.</p><div><hr></div><h4><em>Sidenote: Weighted Round Robin</em></h4><p><em>A round-robin is a <strong>simple load-balancing algorithm</strong>. It works by cycling requests to different servers so they get an equal share of the load.</em></p><p><em>If there are three servers, A, B, C, and three requests come in. A gets the first, B gets the second, and C gets the third.</em></p><p><em><strong>Weighted round robin</strong> is a level up from round robin. Each server is given a <strong>weight based on its processing power and capacity</strong>.</em></p><p><em><strong>Static weights</strong> are assigned manually by a network admin. <strong>Dynamic weights</strong> are adjusted in <strong>real time</strong> by a load balancer. </em></p><p><em>The higher the weight, the more load the server gets.</em></p><p><em>So if A has a weight of 3, B has 2, C has 1, and there were 12 requests. A would get 6, B would get 4, and C would get 2.</em></p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!eN06!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!eN06!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png 424w, https://substackcdn.com/image/fetch/$s_!eN06!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png 848w, https://substackcdn.com/image/fetch/$s_!eN06!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png 1272w, https://substackcdn.com/image/fetch/$s_!eN06!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!eN06!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png\\\" width=\\\"639\\\" height=\\\"470.8997429305913\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:860,&quot;width&quot;:1167,&quot;resizeWidth&quot;:639,&quot;bytes&quot;:113057,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!eN06!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png 424w, https://substackcdn.com/image/fetch/$s_!eN06!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png 848w, https://substackcdn.com/image/fetch/$s_!eN06!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png 1272w, https://substackcdn.com/image/fetch/$s_!eN06!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F2cdf30e7-17a2-437d-842d-9c75b052fff6_1167x860.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><div><hr></div><p>But there was an issue with their traditional load balancing approach.</p><p>Dropbox had <strong>many virtual servers</strong> with vastly <strong>different hardware</strong>. This made it <strong>difficult to distribute the load evenly</strong> between them with static weights.</p><p>This difference in hardware could have been caused by Dropbox using more powerful servers as it grew.</p><p>They may have started with an average server. As it grew, the team acquired more powerful servers. As it grew more, they acquired even more powerful ones.</p><p>At the time, there was <strong>no off-the-shelf load-balancing solution</strong> that could help. Especially one that used a dynamic weighted round-robin <strong>with gRPC support</strong>.</p><p>So, they <strong>built their own</strong>, which they called <strong>Robinhood</strong>.</p><div><hr></div><h4><em>Sidenote: gRPC</em></h4><p><em><strong>Google Remote Procedure Call</strong> (gRPC) is a way for different programs to talk to each other. It's based on RPC, which allows a client to <strong>run a function on the server simply by calling it</strong>.</em></p><p><em>This is <strong>different from REST</strong>, which requires communication via a URL. REST also focuses on the resource being accessed instead of the action that needs to be taken.</em></p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!A6Ug!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!A6Ug!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png 424w, https://substackcdn.com/image/fetch/$s_!A6Ug!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png 848w, https://substackcdn.com/image/fetch/$s_!A6Ug!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png 1272w, https://substackcdn.com/image/fetch/$s_!A6Ug!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!A6Ug!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png\\\" width=\\\"536\\\" height=\\\"158.6122448979592\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/f8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:232,&quot;width&quot;:784,&quot;resizeWidth&quot;:536,&quot;bytes&quot;:42039,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!A6Ug!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png 424w, https://substackcdn.com/image/fetch/$s_!A6Ug!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png 848w, https://substackcdn.com/image/fetch/$s_!A6Ug!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png 1272w, https://substackcdn.com/image/fetch/$s_!A6Ug!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Ff8c9a35b-00f2-4f88-bfe4-48557886807c_784x232.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div></div></div></a></figure></div><p><em>But gRPC has many more differences between REST and regular RPC.</em></p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!VnYj!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!VnYj!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png 424w, https://substackcdn.com/image/fetch/$s_!VnYj!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png 848w, https://substackcdn.com/image/fetch/$s_!VnYj!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png 1272w, https://substackcdn.com/image/fetch/$s_!VnYj!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!VnYj!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png\\\" width=\\\"617\\\" height=\\\"284.0733137829912\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:471,&quot;width&quot;:1023,&quot;resizeWidth&quot;:617,&quot;bytes&quot;:88880,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!VnYj!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png 424w, https://substackcdn.com/image/fetch/$s_!VnYj!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png 848w, https://substackcdn.com/image/fetch/$s_!VnYj!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png 1272w, https://substackcdn.com/image/fetch/$s_!VnYj!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F9853703d-f470-4c45-8c43-7db3b5a86b5f_1023x471.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><p><em>The biggest one is the <strong>use of protobufs</strong>. This file format <a href=\\\"https://protobuf.dev/\\\">developed by Google</a> is used <strong>to store and send data</strong>.</em></p><p><em>It works by encoding structured data into a binary format for fast transmission. The recipient then decodes it back to structured data. This format is also much smaller than something like JSON.</em></p><p><em>Protobufs are what make <strong>gRPC fast</strong>, but also more difficult to set up since the client and server need to support it.</em></p><p><em>gRPC isn't supported natively by browsers. So, it's <strong>commonly</strong> <strong>used for internal server communication</strong>.</em></p><div><hr></div><h2>The Custom Load Balancer</h2><p>The <strong>main component of RobinHood</strong> is the load balancing service or <strong>LBS</strong>. This <strong>manages how requests are distributed</strong> to different servers.</p><p>It does this by <strong>continuously collecting data</strong> from all the servers. It uses this data to figure out the average optimal resource usage for all the servers.</p><p>Each server is given a <strong>PID controller</strong>, a piece of code to help with resource regulation. This has an <strong>upper and lower server resource limit</strong> close to the average.</p><p>Say the average CPU limit is 70%. The upper limit could be 75%, and the lower limit could be 65%. If a server hits 75%, it is given fewer requests to deal with, and if it goes below 65%, it is given more.</p><p>This is <strong>how the LBS gives weights</strong> to each server. Because the LBS uses dynamic weights, a server that previously weighted 5 could become 1 if its resources go above the average.</p><p>In addition to the LBS, <strong>Robinhood had two other components</strong>: the <strong>proxy</strong> and the <strong>routing database</strong>.</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!GAAQ!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!GAAQ!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png 424w, https://substackcdn.com/image/fetch/$s_!GAAQ!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png 848w, https://substackcdn.com/image/fetch/$s_!GAAQ!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png 1272w, https://substackcdn.com/image/fetch/$s_!GAAQ!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!GAAQ!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png\\\" width=\\\"539\\\" height=\\\"654.034103410341\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:1103,&quot;width&quot;:909,&quot;resizeWidth&quot;:539,&quot;bytes&quot;:123449,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!GAAQ!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png 424w, https://substackcdn.com/image/fetch/$s_!GAAQ!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png 848w, https://substackcdn.com/image/fetch/$s_!GAAQ!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png 1272w, https://substackcdn.com/image/fetch/$s_!GAAQ!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F83f9f178-43b3-4026-b6db-1f20a1e7b0e2_909x1103.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><p>The <strong>proxy sends server load data to the LBS</strong> via gRPC. </p><p>Why doesn't the LBS collect this itself? Well, the LBS is already doing a lot.</p><p>Imagine there could be thousands of servers. It would need to scale up just to collect metrics from all of them.</p><p>So, the proxy has the sole responsibility of collecting server data to reduce the load on the LBS.</p><p>The <strong>routing database stores server information</strong>. Things like weights generated by the LBS, IP addresses, hostname, etc.</p><p>Although the LBS stores some data in memory for quick access, <strong>an</strong> <strong>LBS itself can come in and out of existence</strong>; sometimes, it crashes and needs to restart.</p><p>The <strong>routing database keeps data for a long time</strong>, so new or existing LBS instances can access it.</p><p>Routing databases can either be Zookeeper or etcd based. The decision to choose one or the other may be to support legacy systems.</p><div><hr></div><h4><em>Sidenote: Zookeeper vs etcd</em></h4><p><em>Both Zookeeper and etcd are what's called a <strong>distributed coordination service</strong>.</em></p><p><em>They are designed to be the <strong>central place where config and state data is stored</strong> in a distributed system.</em></p><p><em>They also make sure that <strong>each node</strong> in the system has the most <strong>up-to-date version of this data</strong>.</em></p><p><em>These services contain multiple servers and elect a single server, called a leader, that takes all the writes.</em></p><p><em>This server copies the data to other servers, which then distribute the data to the relevant clients. In this case, a client could be an LBS instance.</em></p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!bXRy!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!bXRy!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png 424w, https://substackcdn.com/image/fetch/$s_!bXRy!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png 848w, https://substackcdn.com/image/fetch/$s_!bXRy!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png 1272w, https://substackcdn.com/image/fetch/$s_!bXRy!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!bXRy!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png\\\" width=\\\"666\\\" height=\\\"320.64972527472526\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/af11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:701,&quot;width&quot;:1456,&quot;resizeWidth&quot;:666,&quot;bytes&quot;:94238,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!bXRy!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png 424w, https://substackcdn.com/image/fetch/$s_!bXRy!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png 848w, https://substackcdn.com/image/fetch/$s_!bXRy!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png 1272w, https://substackcdn.com/image/fetch/$s_!bXRy!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Faf11fe7b-1dc9-49c7-96ec-c5b6ea0adb38_1536x739.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><p><em>So, if a new LBS instance joins the cluster, it knows the exact state of all the servers and the average that needs to be achieved.</em></p><p><em>There are a few differences between Zookeeper and etcd.</em></p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!T1n9!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!T1n9!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png 424w, https://substackcdn.com/image/fetch/$s_!T1n9!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png 848w, https://substackcdn.com/image/fetch/$s_!T1n9!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png 1272w, https://substackcdn.com/image/fetch/$s_!T1n9!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!T1n9!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png\\\" width=\\\"627\\\" height=\\\"289.33756097560973\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/d4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:473,&quot;width&quot;:1025,&quot;resizeWidth&quot;:627,&quot;bytes&quot;:82294,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/png&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!T1n9!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png 424w, https://substackcdn.com/image/fetch/$s_!T1n9!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png 848w, https://substackcdn.com/image/fetch/$s_!T1n9!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png 1272w, https://substackcdn.com/image/fetch/$s_!T1n9!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fd4e63a16-6002-4a30-946f-93ee5bd1bb91_1025x473.png 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><div><hr></div><p>After <strong>Dropbox deployed RobinHood to all their data centers</strong>, here is the difference it made.</p><div class=\\\"captioned-image-container\\\"><figure><a class=\\\"image-link image2 is-viewable-img\\\" target=\\\"_blank\\\" href=\\\"https://substackcdn.com/image/fetch/$s_!wzfl!,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp\\\" data-component-name=\\\"Image2ToDOM\\\"><div class=\\\"image2-inset\\\"><picture><source type=\\\"image/webp\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!wzfl!,w_424,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp 424w, https://substackcdn.com/image/fetch/$s_!wzfl!,w_848,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp 848w, https://substackcdn.com/image/fetch/$s_!wzfl!,w_1272,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp 1272w, https://substackcdn.com/image/fetch/$s_!wzfl!,w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp 1456w\\\" sizes=\\\"100vw\\\"><img src=\\\"https://substackcdn.com/image/fetch/$s_!wzfl!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp\\\" width=\\\"691\\\" height=\\\"436.6736111111111\\\" data-attrs=\\\"{&quot;src&quot;:&quot;https://substack-post-media.s3.amazonaws.com/public/images/395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp&quot;,&quot;srcNoWatermark&quot;:null,&quot;fullscreen&quot;:null,&quot;imageSize&quot;:null,&quot;height&quot;:910,&quot;width&quot;:1440,&quot;resizeWidth&quot;:691,&quot;bytes&quot;:21304,&quot;alt&quot;:null,&quot;title&quot;:null,&quot;type&quot;:&quot;image/webp&quot;,&quot;href&quot;:null,&quot;belowTheFold&quot;:true,&quot;topImage&quot;:false,&quot;internalRedirect&quot;:null,&quot;isProcessing&quot;:false,&quot;align&quot;:null,&quot;offset&quot;:false}\\\" class=\\\"sizing-normal\\\" alt=\\\"\\\" srcset=\\\"https://substackcdn.com/image/fetch/$s_!wzfl!,w_424,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp 424w, https://substackcdn.com/image/fetch/$s_!wzfl!,w_848,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp 848w, https://substackcdn.com/image/fetch/$s_!wzfl!,w_1272,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp 1272w, https://substackcdn.com/image/fetch/$s_!wzfl!,w_1456,c_limit,f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395f629e-dadf-4560-97cd-f018529203b7_1440x910.webp 1456w\\\" sizes=\\\"100vw\\\" loading=\\\"lazy\\\"></picture><div class=\\\"image-link-expand\\\"><div class=\\\"pencraft pc-display-flex pc-gap-8 pc-reset\\\"><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container restack-image\\\"><svg aria-hidden=\\\"true\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 20 20\\\" fill=\\\"none\\\" stroke-width=\\\"1.5\\\" stroke=\\\"var(--color-fg-primary)\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" xmlns=\\\"http://www.w3.org/2000/svg\\\"><g><path d=\\\"M2.53001 7.81595C3.49179 4.73911 6.43281 2.5 9.91173 2.5C13.1684 2.5 15.9537 4.46214 17.0852 7.23684L17.6179 8.67647M17.6179 8.67647L18.5002 4.26471M17.6179 8.67647L13.6473 6.91176M17.4995 12.1841C16.5378 15.2609 13.5967 17.5 10.1178 17.5C6.86118 17.5 4.07589 15.5379 2.94432 12.7632L2.41165 11.3235M2.41165 11.3235L1.5293 15.7353M2.41165 11.3235L6.38224 13.0882\\\"></path></g></svg></button><button tabindex=\\\"0\\\" type=\\\"button\\\" class=\\\"pencraft pc-reset pencraft icon-container view-image\\\"><svg xmlns=\\\"http://www.w3.org/2000/svg\\\" width=\\\"20\\\" height=\\\"20\\\" viewBox=\\\"0 0 24 24\\\" fill=\\\"none\\\" stroke=\\\"currentColor\\\" stroke-width=\\\"2\\\" stroke-linecap=\\\"round\\\" stroke-linejoin=\\\"round\\\" class=\\\"lucide lucide-maximize2 lucide-maximize-2\\\"><polyline points=\\\"15 3 21 3 21 9\\\"></polyline><polyline points=\\\"9 21 3 21 3 15\\\"></polyline><line x1=\\\"21\\\" x2=\\\"14\\\" y1=\\\"3\\\" y2=\\\"10\\\"></line><line x1=\\\"3\\\" x2=\\\"10\\\" y1=\\\"21\\\" y2=\\\"14\\\"></line></svg></button></div></div></div></a></figure></div><p>The X axis shows date in MM/DD and the Y axis shows the ratio of CPU usage compared to the average. So, a value of 1.5 means CPU usage was 1.5 times higher than the average.</p><p>You can see that at the start, 95% of CPUs were operating at around 1.17 above the average.</p><p>It takes a <strong>few days for RobinHood to regulate everything</strong>, but after 11/01, the usage is stabilized, and most CPUs are operating at the average. </p><p>This shows a <strong>massive reduction in CPU workload</strong>, which indicates a <strong>better-balanced load</strong>.</p><p>In fact, after using Robinhood in production for a few years, the team at Dropbox has been able to <strong>reduce their server size by 25%</strong>. This massively reduced their costs.</p><p>It isn't stated that Dropbox saved millions annually from this change. But, based on the cost and resource savings they mentioned from implementing Robinhood, as well as their size.</p><p>It can be inferred that they saved a lot of money, most likely millions from this change.</p><h2>Wrapping Things Up</h2><p>It's amazing everything that goes on behind the scenes when someone uploads a file to Dropbox. I will never look at the app in the same way again.</p><p>I hope you enjoyed reading this as much as I enjoyed writing it. If you want more details, you can check out the <a href=\\\"https://dropbox.tech/infrastructure/robinhood-in-house-load-balancing-service#footnote-two\\\">original article</a>.</p><p>And as usual, be sure to subscribe to get the next article sent straight to your inbox.</p><div><hr></div><p><strong>Newsletter Spotlight</strong></p><p>If you want to become a more productive tech professional, check out the <a href=\\\"https://techproductivity.co/\\\">Tech Productivity</a> newsletter. </p><p>It\u2019s full of links and tools you wish you had known about earlier.</p><div><hr></div><p><em>PS: Enjoyed this newsletter? Please forward it to a pal or follow us on socials (<a href=\\\"https://www.linkedin.com/company/betterstack/\\\">LinkedIn</a>, <a href=\\\"https://x.com/BetterStackHQ\\\">Twitter</a>, <a href=\\\"https://www.youtube.com/@betterstack\\\">YouTube</a>, <a href=\\\"https://www.instagram.com/betterstackhq/\\\">Instagram</a>). It only takes 10 seconds. Making this one took 20 hours.</em></p>\",\"has_dynamic_content\":false,\"truncated_body_text\":\"Dropbox is a cloud-based storage service that is ridiculously easy to use.\",\"wordcount\":1397,\"post_preview_limit\":null,\"language\":\"en\",\"postTags\":[{\"id\":\"1f46f500-2b38-4dc8-8cf9-87167a8b1407\",\"publication_id\":2269685,\"name\":\"tech\",\"slug\":\"tech\",\"hidden\":false},{\"id\":\"9e41186d-f235-415c-901d-9ad19cfec8d0\",\"publication_id\":2269685,\"name\":\"data\",\"slug\":\"data\",\"hidden\":false},{\"id\":\"c07c9770-89f2-473d-b196-72efc717f63b\",\"publication_id\":2269685,\"name\":\"servers\",\"slug\":\"servers\",\"hidden\":false},{\"id\":\"cbbc48be-39cf-4793-8f22-eb32f0b17280\",\"publication_id\":2269685,\"name\":\"big tech\",\"slug\":\"big-tech\",\"hidden\":false},{\"id\":\"f52af370-5bf1-4228-a314-9824aa93e1c7\",\"publication_id\":2269685,\"name\":\"dropbox\",\"slug\":\"dropbox\",\"hidden\":false}],\"postCountryBlocks\":[],\"headlineTest\":null,\"coverImagePalette\":{\"Vibrant\":{\"rgb\":[151,125,230],\"population\":23},\"DarkVibrant\":{\"rgb\":[43.629677419354884,21.387096774193562,111.21290322580644],\"population\":0},\"LightVibrant\":{\"rgb\":[178,158,234],\"population\":61},\"Muted\":{\"rgb\":[118,128,136],\"population\":53},\"DarkMuted\":{\"rgb\":[50.34193548387102,24.677419354838715,128.32258064516128],\"population\":0},\"LightMuted\":{\"rgb\":[159,167,188],\"population\":91}},\"publishedBylines\":[{\"id\":251681141,\"name\":\"Richard Oliver Bray\",\"handle\":\"richardoliverbray\",\"photo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/2cfb44c1-5ecd-42e3-a7e9-f08a8ad4ad51_1424x1254.png\",\"bio\":null,\"profile_set_up_at\":\"2024-07-10T08:54:13.868Z\",\"reader_installed_at\":null,\"publicationUsers\":[{\"id\":2817996,\"user_id\":251681141,\"publication_id\":2269685,\"role\":\"admin\",\"public\":true,\"is_primary\":false,\"publication\":{\"id\":2269685,\"name\":\"Hacking Scale by Better Stack\",\"subdomain\":\"betterstack\",\"custom_domain\":\"newsletter.betterstack.com\",\"custom_domain_optional\":false,\"hero_text\":\"A weekly newsletter about building and scaling software from engineers at Better Stack\",\"logo_url\":\"https://substack-post-media.s3.amazonaws.com/public/images/87b62237-9eb2-480e-abda-ac1ffc34a163_256x256.png\",\"author_id\":131972456,\"primary_user_id\":131972456,\"theme_var_background_pop\":\"#FF81CD\",\"created_at\":\"2024-01-18T12:37:43.322Z\",\"email_from_name\":\"Hacking Scale\",\"copyright\":\"Better Stack\",\"founding_plan_name\":null,\"community_enabled\":true,\"invite_only\":false,\"payments_state\":\"disabled\",\"language\":null,\"explicit\":false,\"homepage_type\":\"newspaper\",\"is_personal_mode\":false,\"logo_url_wide\":null}}],\"is_guest\":false,\"bestseller_tier\":null,\"status\":{\"bestsellerTier\":null,\"subscriberTier\":null,\"leaderboard\":null,\"vip\":false,\"badge\":null,\"subscriber\":null}}],\"reaction\":null,\"reaction_count\":17,\"comment_count\":0,\"child_comment_count\":0,\"is_geoblocked\":false,\"audio_items\":[{\"post_id\":152558295,\"voice_id\":\"voiceover\",\"audio_url\":\"https://api.substack.com/api/v1/audio/upload/ef91a656-c97e-4643-8189-5761abaa8be2/src\",\"type\":\"voiceover\",\"status\":\"completed\",\"duration\":480.41797}],\"hasCashtag\":false},\"comments\":[],\"canonicalUrl\":\"https://newsletter.betterstack.com/p/how-dropbox-saved-millions-of-dollars\",\"inlineComments\":false,\"readerIsSearchCrawler\":false,\"ogUrl\":\"https://newsletter.betterstack.com/p/how-dropbox-saved-millions-of-dollars\",\"bannedFromNotes\":false,\"themeVariables\":{\"color_theme_bg_pop\":\"#5b63d3\",\"background_pop\":\"#5b63d3\",\"color_theme_bg_web\":null,\"cover_bg_color\":\"#FFFFFF\",\"cover_bg_color_secondary\":\"#f0f0f0\",\"background_pop_darken\":\"#4750ce\",\"print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_darken\":\"#4750ce\",\"color_theme_print_on_pop\":\"#ffffff\",\"color_theme_bg_pop_20\":\"rgba(91, 99, 211, 0.2)\",\"color_theme_bg_pop_30\":\"rgba(91, 99, 211, 0.3)\",\"print_pop\":\"#5b63d3\",\"color_theme_accent\":\"#5b63d3\",\"cover_print_primary\":\"#363737\",\"cover_print_secondary\":\"#757575\",\"cover_print_tertiary\":\"#b6b6b6\",\"cover_border_color\":\"#5b63d3\",\"font_family_body_preset\":\"'SF Pro Display', -apple-system, system-ui, BlinkMacSystemFont, 'Inter', 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'\",\"font_weight_body_preset\":400,\"font_preset_body\":\"sans\",\"home_hero\":\"newspaper\",\"home_posts\":\"grid\",\"web_bg_color\":\"#ffffff\",\"background_contrast_1\":\"#f0f0f0\",\"background_contrast_2\":\"#dddddd\",\"background_contrast_3\":\"#b7b7b7\",\"background_contrast_4\":\"#929292\",\"background_contrast_5\":\"#515151\",\"color_theme_bg_contrast_1\":\"#f0f0f0\",\"color_theme_bg_contrast_2\":\"#dddddd\",\"color_theme_bg_contrast_3\":\"#b7b7b7\",\"color_theme_bg_contrast_4\":\"#929292\",\"color_theme_bg_contrast_5\":\"#515151\",\"color_theme_bg_elevated\":\"#ffffff\",\"color_theme_bg_elevated_secondary\":\"#f0f0f0\",\"color_theme_bg_elevated_tertiary\":\"#dddddd\",\"color_theme_detail\":\"#e6e6e6\",\"background_contrast_pop\":\"rgba(91, 99, 211, 0.4)\",\"color_theme_bg_contrast_pop\":\"rgba(91, 99, 211, 0.4)\",\"theme_bg_is_dark\":\"0\",\"print_on_web_bg_color\":\"#363737\",\"print_secondary_on_web_bg_color\":\"#868787\",\"background_pop_rgb\":\"91, 99, 211\",\"color_theme_bg_pop_rgb\":\"91, 99, 211\",\"color_theme_accent_rgb\":\"91, 99, 211\"},\"recentEpisodes\":null,\"trackFrontendVisit\":true,\"activeLiveStream\":null,\"freeTrialCoupon\":null,\"defaultCoupon\":null,\"isChatActive\":false,\"publicationPageAlert\":null,\"features\":{},\"browser\":{\"name\":\"Chrome\",\"version\":\"125.0\",\"major\":\"125\"},\"showCookieBanner\":false,\"disabledCookies\":[],\"dd_env\":\"prod\",\"dd_ti\":true}")</script>
<script nonce="JWtlbHixuqoee6u-MsbsjA">window._analyticsConfig = JSON.parse("{\"properties\":{\"subdomain\":\"betterstack\",\"publication_id\":2269685,\"has_plans\":false,\"pub_community_enabled\":true,\"is_personal_publication\":false,\"is_subscribed\":false,\"is_free_subscribed\":false,\"is_author\":false,\"is_contributor\":false,\"is_admin\":false,\"is_founding\":false},\"adwordsAccountId\":\"AW-316245675\",\"adwordsEventSendTo\":\"Tf76CKqcyL4DEKuN5pYB\"}")</script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/lib-router.7371871e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/55188.571db0c6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/62357.a0731d37.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53754.d196682b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/33485.2686bb5d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/2237.ae8857e4.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37707.4614ce58.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/9351.41564edd.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80821.b2140fcd.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44292.f3dc2b43.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70291.7aa24e85.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17499.1820773d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57975.765aca5a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/19714.2f1bdde5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48320.4e726ad5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93000.f03f24de.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/44014.f579d62b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/43745.80c75c78.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/60892.92b9fc9c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/40646.258e49f2.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/87906.d493457b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/36479.ce61e623.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/17706.015b0f71.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93977.f1207089.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/37606.b91ab2ef.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94311.b93f6845.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89356.66161d1e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/21262.211a7337.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47925.24a01d36.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/80220.03bd4c41.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71505.3dc43e33.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11100.5d148f2c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/3663.86d95e8c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18725.a5a682c3.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/86118.ff1ec843.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/93519.7fe128ac.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/84828.c78aabc1.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/5075.d6e9f8ec.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/29231.a7af5911.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56725.2b32bbb6.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/42336.f1c17ded.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/89879.1d0af2b9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/70793.951c11cb.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/39522.58e078b0.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53164.dfde36c8.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74584.94c6b3f1.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/71199.a327e9f4.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63266.8bb0d59c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/47358.8f5c5991.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/4492.cb326b80.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/88089.680833b5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/97348.df6a856d.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/52109.cf3bcc2c.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/56256.8a1bc5dd.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/77410.3f20fb36.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/25058.bdbc4e21.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/58483.3adddb8a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/94881.0a97506f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27386.09a9082f.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/10174.7f0face7.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/92610.a4fc1daf.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91214.cb10b4b8.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/11244.b723f409.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99260.28b0ed6a.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48445.7d5c5ba3.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/76768.9a2f2bae.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/41932.6c688f22.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/12114.979fb38e.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/48409.9732c6f9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/26252.4f5ae3be.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/53871.2923c5c9.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/18494.588a06d1.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/99068.bd2e1fba.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91850.79d3f415.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/49047.c7f428fb.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/16075.098bcbcf.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/28856.2eb51db5.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/95396.72d1d844.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/63175.bbf1c283.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/74628.2e8ed950.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/32552.889dd584.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/27313.c287f2d8.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/mainReactRouter.6c265246.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/91507.9b9761ae.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/54770.b40bf629.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/30054.07cd629b.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78118.35b48b09.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/57051.f37fb930.js" charset="utf-8"></script>
<script defer type="module" src="https://substackcdn.com/bundle/static/js/78191.24075434.js" charset="utf-8"></script>
<script nonce="JWtlbHixuqoee6u-MsbsjA" nomodule>
(function() {
var message = 'Your browser does not support modern JavaScript modules. Please upgrade your browser for the best experience.';
var warningDiv = document.createElement('div');
warningDiv.style.color = 'red';
warningDiv.style.padding = '10px';
warningDiv.style.margin = '10px 0';
warningDiv.style.border = '1px solid red';
warningDiv.style.backgroundColor = 'lightyellow';
warningDiv.innerText = message;
document.body.prepend(warningDiv);
})();
</script>
<!-- Datadog Analytics -->
<script nonce="JWtlbHixuqoee6u-MsbsjA">
(function(h,o,u,n,d) {
h=h[d]=h[d]||{q:[],onReady:function(c){h.q.push(c)}}
d=o.createElement(u);d.async=1;d.src=n
n=o.getElementsByTagName(u)[0];n.parentNode.insertBefore(d,n)
})(window,document,'script','https://www.datadoghq-browser-agent.com/us1/v5/datadog-rum.js','DD_RUM')
window.DD_RUM.onReady(function() {
window.DD_RUM.init({
clientToken: 'puba71073f072643721169b68f352438710',
applicationId: '2e321b35-c76b-4073-8d04-cc9a10461793',
site: 'datadoghq.com',
service: 'web',
env: window._preloads.dd_env,
version: '8b4c00716b2bc0f7ca1fded553014f7e37b5b07a',
sessionSampleRate: 1,
sessionReplaySampleRate: 10,
trackUserInteractions: window._preloads.dd_ti,
trackResources: true,
trackLongTasks: true,
defaultPrivacyLevel: 'mask-user-input',
allowedTracingUrls: [/https?:\/\/(.+\/.)?substack(cdn)?\.com/]
});
})
</script>
<!-- End Datadog Analytics -->
<!-- Cloudflare Web Analytics -->
<script defer src="https://static.cloudflareinsights.com/beacon.min.js" data-cf-beacon='{"token": "216309cffb464db4b0e02daf0b8e8060"}'></script>
<!-- End Cloudflare Web Analytics -->
<!-- Fallback tracking pixels -->
<noscript><img height="1" width="1" style="display:none" alt="" src="https://www.facebook.com/tr?id=1062799887416376&ev=PageView&noscript=1"/></noscript>
<noscript>
<img height="1" width="1" style="display:none" alt="" src="https://analytics.twitter.com/i/adsct?p_id=Twitter&p_user_id=0&txn_id=o7hjd&events=%5B%5B%22pageview%22%2Cnull%5D%5D&tw_sale_amount=0&tw_order_quantity=0&tw_iframe_status=0"/>
<img height="1" width="1" style="display:none" alt="" src="https://t.co/i/adsct?p_id=Twitter&p_user_id=0&txn_id=o7hjd&events=%5B%5B%22pageview%22%2Cnull%5D%5D&tw_sale_amount=0&tw_order_quantity=0&tw_iframe_status=0"/>
</noscript>
<script nonce="JWtlbHixuqoee6u-MsbsjA">
!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
},s.version='1.1',s.queue=[],u=t.createElement(n),u.async=!0,u.src='https://static.ads-twitter.com/uwt.js',
a=t.getElementsByTagName(n)[0],a.parentNode.insertBefore(u,a))}(window,document,'script');
twq('config','o7hjd');
</script>
<noscript>
<style>
#nojs-banner {
position: fixed;
bottom: 0;
left: 0;
padding: 16px 16px 16px 32px;
width: 100%;
box-sizing: border-box;
background: red;
color: white;
font-family: -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
font-size: 13px;
line-height: 13px;
}
#nojs-banner a {
color: inherit;
text-decoration: underline;
}
</style>
<div id="nojs-banner">
This site requires JavaScript to run correctly. Please <a href="https://enable-javascript.com/" target="_blank">turn on JavaScript</a> or unblock scripts
</div>
</noscript>
<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" nonce="JWtlbHixuqoee6u-MsbsjA" data-cf-beacon='{"version":"2024.11.0","token":"68cfe66b5c4749e2ba64d4d9640c04c0","spa":2}' crossorigin="anonymous"></script>
</body>
</html>