Files
nexus/sreweekly/articles/270/02-the-downtime-project.html
2026-09-12 17:23:01 +08:00

2094 lines
148 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html itemscope="itemscope" itemtype="http://schema.org/WebPage" lang="en-US">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" />
<link rel="profile" href="http://gmpg.org/xfn/11" />
<link rel="pingback" href="https://downtimeproject.com/xmlrpc.php" />
<!--[if lt IE 9]>
<script src="https://downtimeproject.com/wp-content/themes/independent-publisher/js/html5.js" type="text/javascript"></script>
<![endif]-->
<title>The Downtime Project</title>
<meta name='robots' content='max-image-preview:large' />
<link rel="alternate" type="application/rss+xml" title="The Downtime Project &raquo; Feed" href="https://downtimeproject.com/feed/" />
<link rel="alternate" type="application/rss+xml" title="The Downtime Project &raquo; Comments Feed" href="https://downtimeproject.com/comments/feed/" />
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=https://downtimeproject.com/wp-content/plugins/gutenberg/build/styles/block-library/common.min.css */
</style>
<style id="wp-block-list-inline-css">
ol,ul{box-sizing:border-box}:root :where(.wp-block-list.has-background){padding:1.25em 2.375em}
/*# sourceURL=https://downtimeproject.com/wp-content/plugins/gutenberg/build/styles/block-library/list/style.min.css */
</style>
<style id="wp-block-paragraph-inline-css">
.is-small-text{font-size:.875em}.is-regular-text{font-size:1em}.is-large-text{font-size:2.25em}.is-larger-text{font-size:3em}.has-drop-cap:not(:focus):first-letter{float:left;font-size:8.4em;font-style:normal;font-weight:100;line-height:.68;margin:.05em .1em 0 0;text-transform:uppercase}body.rtl .has-drop-cap:not(:focus):first-letter{float:none;margin-left:.1em}p.has-drop-cap.has-background{overflow:hidden}:root :where(p.has-background){padding:1.25em 2.375em}:where(p.has-text-color:not(.has-link-color)) a{color:inherit}p.has-text-align-left[style*="writing-mode:vertical-lr"],p.has-text-align-right[style*="writing-mode:vertical-rl"]{rotate:180deg}
/*# sourceURL=https://downtimeproject.com/wp-content/plugins/gutenberg/build/styles/block-library/paragraph/style.min.css */
</style>
<style id="classic-theme-styles-inline-css">
.wp-block-button__link{background-color:#32373c;border-radius:9999px;box-shadow:none;color:#fff;font-size:1.125em;padding:calc(.667em + 2px) calc(1.333em + 2px);text-decoration:none}.wp-block-file__button{background:#32373c;color:#fff}.wp-block-accordion-heading{margin:0}.wp-block-accordion-heading__toggle{background-color:inherit!important;color:inherit!important}.wp-block-accordion-heading__toggle:not(:focus-visible){outline:none}.wp-block-accordion-heading__toggle:focus,.wp-block-accordion-heading__toggle:hover{background-color:inherit!important;border:none;box-shadow:none;color:inherit;padding:var(--wp--preset--spacing--20,1em) 0;text-decoration:none}.wp-block-accordion-heading__toggle:focus-visible{outline:auto;outline-offset:0}.wp-block-tab:not(.has-text-color){color:inherit!important}.wp-block-tab:not(.has-background){background-color:inherit!important}.wp-block-tab:focus,.wp-block-tab:hover{text-decoration:none}.wp-block-tab:focus-visible{outline:auto;outline-offset:0}
/*# sourceURL=https://downtimeproject.com/wp-content/plugins/gutenberg/build/styles/block-library/classic.min.css */
</style>
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
/*# sourceURL=global-styles-inline-css */
</style>
<link rel='stylesheet' id='genericons-css' href='https://downtimeproject.com/wp-content/themes/independent-publisher/fonts/genericons/genericons.css?ver=3.1' media='all' />
<link rel='stylesheet' id='independent-publisher-style-css' href='https://downtimeproject.com/wp-content/themes/independent-publisher/style.css?ver=7.1' media='all' />
<link rel='stylesheet' id='customizer-css' href='https://downtimeproject.com/wp-admin/admin-ajax.php?action=independent_publisher_customizer_css&#038;ver=1.7' media='all' />
<script id="jquery-core-js" src="https://downtimeproject.com/wp-includes/js/jquery/jquery.min.js?ver=3.7.1"></script>
<script id="jquery-migrate-js" src="https://downtimeproject.com/wp-includes/js/jquery/jquery-migrate.min.js?ver=3.4.1"></script>
<script id="enhanced-comment-form-js-js" src="https://downtimeproject.com/wp-content/themes/independent-publisher/js/enhanced-comment-form.js?ver=1.0"></script>
<link rel="https://api.w.org/" href="https://downtimeproject.com/wp-json/" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://downtimeproject.com/xmlrpc.php?rsd" />
<meta name="generator" content="WordPress 7.1" />
<meta name="generator" content="Seriously Simple Podcasting 3.17.0" />
<link href="https://micro.blog/beaucollins" rel="me" /><link rel="alternate" type="application/feed+json" title="The Downtime Project &raquo; JSON Feed" href="https://downtimeproject.com/feed/json/" />
<link rel="alternate" type="application/rss+xml" title="Podcast RSS feed" href="https://downtimeproject.com/feed/podcast" />
<meta property="fediverse:creator" name="fediverse:creator" content="episodes@downtimeproject.com" />
<!-- Analytics by WP Statistics - https://wp-statistics.com -->
<link rel="alternate" title="ActivityPub (JSON)" type="application/activity+json" href="https://downtimeproject.com/?author=0" />
<link rel="icon" href="https://downtimeproject.com/files/2021/03/downtime-150x150.png" sizes="32x32" />
<link rel="icon" href="https://downtimeproject.com/files/2021/03/downtime-300x300.png" sizes="192x192" />
<link rel="apple-touch-icon" href="https://downtimeproject.com/files/2021/03/downtime-300x300.png" />
<meta name="msapplication-TileImage" content="https://downtimeproject.com/files/2021/03/downtime-300x300.png" />
<link rel='stylesheet' id='ssp-castos-player-css' href='https://downtimeproject.com/wp-content/plugins/seriously-simple-podcasting/assets/css/castos-player.min.css?ver=3.17.0' media='all' />
</head>
<body class="home blog wp-theme-independent-publisher no-post-excerpts hfeed h-feed" itemscope="itemscope" itemtype="http://schema.org/WebPage">
<div id="page" class="site">
<header id="masthead" class="site-header" role="banner" itemscope itemtype="http://schema.org/WPHeader">
<div class="site-header-info">
<a class="site-logo" href="https://downtimeproject.com/" title="The Downtime Project" rel="home">
<img class="no-grav" src="https://secure.gravatar.com/avatar/15fa3819332b0c803deea11086587304/?s=100&#038;d=mm" height="100" width="100" alt="The Downtime Project" />
</a>
<div class="site-title">
<a href="https://downtimeproject.com/" title="The Downtime Project" rel="home">The Downtime Project</a>
</div>
<div class="site-description"></div>
<div id="menu-social" class="menu"><ul id="menu-social-items" class="menu-items"><li id="menu-item-92" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-92"><a href="https://twitter.com/sevreview"><span class="screen-reader-text">Twitter</span></a></li>
</ul></div> </div>
<nav role="navigation" class="site-navigation main-navigation">
<a class="screen-reader-text skip-link" href="#content" title="Skip to content">Skip to content</a>
<div class="menu"></div>
</nav><!-- .site-navigation .main-navigation -->
</header>
<!-- #masthead .site-header -->
<div id="main" class="site-main">
<div id="primary" class="content-area">
<main id="content" class="site-content" role="main">
<article id="post-100" class="first-post post-100 podcast type-podcast status-publish series-the-downtime-project h-entry hentry">
<header class="entry-header">
<h1 class="entry-title p-name">
<a href="https://downtimeproject.com/podcast/7-lessons-from-10-outages/" title="Permalink to 7 Lessons From 10 Outages" rel="bookmark">7 Lessons From 10 Outages</a>
</h1>
</header>
<!-- .entry-header -->
<div class="entry-content e-content">
<div class="podcast_player"><div id="2022605842" class="castos-player dark-mode " role="region" aria-label="Podcast player" data-episode="100" data-player_id="2022605842">
<div class="player">
<div class="player__main">
<div class="player__artwork player__artwork-100">
<img decoding="async" src="https://downtimeproject.com/files/2021/03/dp_logo-150x150.png"
alt="The Downtime Project"
title="The Downtime Project">
</div>
<div class="player__body">
<div class="currently-playing">
<div class="show player__podcast-title">
The Downtime Project </div>
<div class="episode-title player__episode-title">7 Lessons From 10 Outages</div>
</div>
<div class="play-progress">
<div class="play-pause-controls">
<button title="Play" aria-label="Play Episode" aria-pressed="false" class="play-btn">
<span class="screen-reader-text">Play Episode</span>
</button>
<button title="Pause" aria-label="Pause Episode" aria-pressed="false" class="pause-btn hide">
<span class="screen-reader-text">Pause Episode</span>
</button>
<img decoding="async" src="https://downtimeproject.com/wp-content/plugins/seriously-simple-podcasting/assets/css/images/player/images/icon-loader.svg" alt="Loading" class="ssp-loader hide"/>
</div>
<div>
<audio preload="none" class="clip clip-100">
<source src="https://downtimeproject.com/files/2021/06/DP-012-Mix1.mp3">
</audio>
<div class="ssp-progress" role="progressbar" title="Seek" aria-valuenow="0" aria-valuemin="0" aria-valuemax="2780">
<span class="progress__filled"></span>
</div>
<div class="ssp-playback playback">
<div class="playback__controls">
<button class="player-btn player-btn__volume" title="Mute/Unmute" aria-label="Mute audio" aria-pressed="false">
</button>
<button data-skip="-10" class="player-btn player-btn__rwd" title="Rewind 10 seconds" aria-label="Rewind 10 seconds">
</button>
<button data-speed="1" class="player-btn player-btn__speed" title="Playback Speed" aria-label="Playback speed, currently 1x">1x</button>
<button data-skip="30" class="player-btn player-btn__fwd" title="Fast Forward 30 seconds" aria-label="Fast forward 30 seconds">
</button>
</div>
<div class="playback__timers">
<time class="ssp-timer" aria-live="off">00:00</time>
<span>/</span>
<!-- We need actual duration here from the server -->
<time class="ssp-duration" datetime="PT0H46M20S">46:20</time>
</div>
</div>
</div>
</div>
<nav class="player-panels-nav">
<button class="subscribe-btn" id="subscribe-btn-100" title="Subscribe" aria-expanded="false">Subscribe</button>
<button class="share-btn" id="share-btn-100" title="Share" aria-expanded="false">Share</button>
</nav>
</div>
</div>
</div>
<div class="player-panels player-panels-100">
<div class="subscribe player-panel subscribe-100">
<button class="close-btn close-btn-100" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="panel__inner">
<div class="subscribe-icons">
<a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" rel="noopener noreferrer"
class="apple_podcasts"
title="Subscribe on Apple Podcasts">
<span></span>
Apple Podcasts </a>
<a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" rel="noopener noreferrer"
class="stitcher"
title="Subscribe on Stitcher">
<span></span>
Stitcher </a>
<a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" rel="noopener noreferrer"
class="google_podcasts"
title="Subscribe on Google Podcasts">
<span></span>
Google Podcasts </a>
<a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" rel="noopener noreferrer"
class="spotify"
title="Subscribe on Spotify">
<span></span>
Spotify </a>
</div>
<div class="player-panel-row" role="group" aria-label="RSS Feed URL">
<div class="title">RSS Feed</div>
<div>
<input value="https://downtimeproject.com/feed/podcast/the-downtime-project" class="input-rss input-rss-100" title="RSS Feed URL" readonly />
</div>
<button class="copy-rss copy-rss-100" title="Copy RSS Feed URL" aria-label="Copy RSS Feed URL"></button>
</div>
</div>
</div>
<div class="share share-100 player-panel">
<button class="close-btn close-btn-100" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="player-panel-row">
<div class="title">
Share </div>
<div class="icons-holder">
<a href="https://www.facebook.com/sharer/sharer.php?u=https://downtimeproject.com/podcast/7-lessons-from-10-outages/&t=7 Lessons From 10 Outages"
target="_blank" rel="noopener noreferrer" class="share-icon facebook" title="Share on Facebook" aria-label="Share on Facebook">
<span></span>
</a>
<a href="https://twitter.com/intent/tweet?text=https://downtimeproject.com/podcast/7-lessons-from-10-outages/&url=7 Lessons From 10 Outages"
target="_blank" rel="noopener noreferrer" class="share-icon twitter" title="Share on Twitter" aria-label="Share on Twitter">
<span></span>
</a>
<a href="https://downtimeproject.com/files/2021/06/DP-012-Mix1.mp3"
target="_blank" rel="noopener noreferrer" class="share-icon download" title="Download" aria-label="Download" download>
<span></span>
</a>
</div>
</div>
<div class="player-panel-row">
<div class="title">
Link </div>
<div>
<input value="https://downtimeproject.com/podcast/7-lessons-from-10-outages/" class="input-link input-link-100" title="Episode URL" readonly />
</div>
<button class="copy-link copy-link-100" title="Copy Episode URL" aria-label="Copy Episode URL"></button>
</div>
<div class="player-panel-row">
<div class="title">
Embed </div>
<div>
<textarea rows="1"
title="Embed Code"
class="input-embed input-embed-100" readonly>&lt;blockquote class=&quot;wp-embedded-content&quot; data-secret=&quot;0DygCo45X0&quot;&gt;&lt;a href=&quot;https://downtimeproject.com/podcast/7-lessons-from-10-outages/&quot;&gt;7 Lessons From 10 Outages&lt;/a&gt;&lt;/blockquote&gt;&lt;iframe sandbox=&quot;allow-scripts&quot; security=&quot;restricted&quot; src=&quot;https://downtimeproject.com/podcast/7-lessons-from-10-outages/embed/#?secret=0DygCo45X0&quot; width=&quot;500&quot; height=&quot;350&quot; title=&quot;&amp;#8220;7 Lessons From 10 Outages&amp;#8221; &amp;#8212; The Downtime Project&quot; data-secret=&quot;0DygCo45X0&quot; frameborder=&quot;0&quot; marginwidth=&quot;0&quot; marginheight=&quot;0&quot; scrolling=&quot;no&quot; class=&quot;wp-embedded-content&quot;&gt;&lt;/iframe&gt;&lt;script&gt;
/*! This file is auto-generated */
!function(d,l){&quot;use strict&quot;;l.querySelector&amp;&amp;d.addEventListener&amp;&amp;&quot;undefined&quot;!=typeof URL&amp;&amp;(d.wp=d.wp||{},d.wp.receiveEmbedMessage||(d.wp.receiveEmbedMessage=function(e){var t=e.data;if((t||t.secret||t.message||t.value)&amp;&amp;!/[^a-zA-Z0-9]/.test(t.secret)){for(var s,r,n,a=l.querySelectorAll(&#039;iframe[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),o=l.querySelectorAll(&#039;blockquote[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),c=new RegExp(&quot;^https?:$&quot;,&quot;i&quot;),i=0;i&lt;o.length;i++)o[i].style.display=&quot;none&quot;;for(i=0;i&lt;a.length;i++)s=a[i],e.source===s.contentWindow&amp;&amp;(s.removeAttribute(&quot;style&quot;),&quot;height&quot;===t.message?(1e3&lt;(r=parseInt(t.value,10))?r=1e3:~~r&lt;200&amp;&amp;(r=200),s.height=r):&quot;link&quot;===t.message&amp;&amp;(r=new URL(s.getAttribute(&quot;src&quot;)),n=new URL(t.value),c.test(n.protocol))&amp;&amp;n.host===r.host&amp;&amp;l.activeElement===s&amp;&amp;(d.top.location.href=t.value))}},d.addEventListener(&quot;message&quot;,d.wp.receiveEmbedMessage,!1),l.addEventListener(&quot;DOMContentLoaded&quot;,function(){for(var e,t,s=l.querySelectorAll(&quot;iframe.wp-embedded-content&quot;),r=0;r&lt;s.length;r++)(t=(e=s[r]).getAttribute(&quot;data-secret&quot;))||(t=Math.random().toString(36).substring(2,12),e.src+=&quot;#?secret=&quot;+t,e.setAttribute(&quot;data-secret&quot;,t)),e.contentWindow.postMessage({message:&quot;ready&quot;,secret:t},&quot;*&quot;)},!1)))}(window,document);
//# sourceURL=https://downtimeproject.com/wp-includes/js/wp-embed.min.js
&lt;/script&gt;
</textarea>
</div>
<button class="copy-embed copy-embed-100" title="Copy Embed Code" aria-label="Copy Embed Code"></button>
</div>
</div>
</div>
</div>
<div class="podcast_meta"><aside><p><a href="https://downtimeproject.com/podcast-download/100/7-lessons-from-10-outages?ref=download" title="7 Lessons From 10 Outages " class="podcast-meta-download">Download file</a> | <a href="https://downtimeproject.com/podcast-download/100/7-lessons-from-10-outages?ref=new_window" target="_blank" title="7 Lessons From 10 Outages " class="podcast-meta-new-window">Play in new window</a> | <span class="podcast-meta-duration">Duration: 46:20</span> | <span class="podcast-meta-date">Recorded on June 17, 2021</span></p><p>Subscribe: <a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" title="Apple Podcasts" class="podcast-meta-itunes">Apple Podcasts</a> | <a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" title="Stitcher" class="podcast-meta-itunes">Stitcher</a> | <a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" title="Google Podcasts" class="podcast-meta-itunes">Google Podcasts</a> | <a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" title="Spotify" class="podcast-meta-itunes">Spotify</a></p></aside></div></div>
<p class="wp-block-paragraph">After 10 post-mortems in their first season, Tom and Jamie reflect on the common issues they&#8217;ve seen. Click through for details!</p>
<p><a href="https://downtimeproject.com/podcast/7-lessons-from-10-outages/#more-100" class="more-link">Continue reading <span class="meta-nav">&rarr;</span></a></p>
</div>
<!-- .entry-content -->
<footer class="entry-meta">
<span>2,416 Words</span> <span class="sep"> | </span>
<span class="comments-link"><a href="https://downtimeproject.com/podcast/7-lessons-from-10-outages/#comments">1 Comment</a></span>
</footer>
<!-- .entry-meta -->
</article><!-- #post-100 -->
<article id="post-96" class="post-96 podcast type-podcast status-publish series-the-downtime-project h-entry hentry">
<header class="entry-header">
<h1 class="entry-title p-name">
<a href="https://downtimeproject.com/podcast/salesforce-publishes-a-controversial-postmortem-and-breaks-their-dns/" title="Permalink to Salesforce Publishes a Controversial Postmortem (and breaks their DNS)" rel="bookmark">Salesforce Publishes a Controversial Postmortem (and breaks their DNS)</a>
</h1>
</header>
<!-- .entry-header -->
<div class="entry-content e-content">
<div class="podcast_player"><div id="527562663" class="castos-player dark-mode " role="region" aria-label="Podcast player" data-episode="96" data-player_id="527562663">
<div class="player">
<div class="player__main">
<div class="player__artwork player__artwork-96">
<img decoding="async" src="https://downtimeproject.com/files/2021/03/dp_logo-150x150.png"
alt="The Downtime Project"
title="The Downtime Project">
</div>
<div class="player__body">
<div class="currently-playing">
<div class="show player__podcast-title">
The Downtime Project </div>
<div class="episode-title player__episode-title">Salesforce Publishes a Controversial Postmortem (and breaks their DNS)</div>
</div>
<div class="play-progress">
<div class="play-pause-controls">
<button title="Play" aria-label="Play Episode" aria-pressed="false" class="play-btn">
<span class="screen-reader-text">Play Episode</span>
</button>
<button title="Pause" aria-label="Pause Episode" aria-pressed="false" class="pause-btn hide">
<span class="screen-reader-text">Pause Episode</span>
</button>
<img decoding="async" src="https://downtimeproject.com/wp-content/plugins/seriously-simple-podcasting/assets/css/images/player/images/icon-loader.svg" alt="Loading" class="ssp-loader hide"/>
</div>
<div>
<audio preload="none" class="clip clip-96">
<source src="https://downtimeproject.com/files/2021/05/DP-011-Mix1.mp3">
</audio>
<div class="ssp-progress" role="progressbar" title="Seek" aria-valuenow="0" aria-valuemin="0" aria-valuemax="2451">
<span class="progress__filled"></span>
</div>
<div class="ssp-playback playback">
<div class="playback__controls">
<button class="player-btn player-btn__volume" title="Mute/Unmute" aria-label="Mute audio" aria-pressed="false">
</button>
<button data-skip="-10" class="player-btn player-btn__rwd" title="Rewind 10 seconds" aria-label="Rewind 10 seconds">
</button>
<button data-speed="1" class="player-btn player-btn__speed" title="Playback Speed" aria-label="Playback speed, currently 1x">1x</button>
<button data-skip="30" class="player-btn player-btn__fwd" title="Fast Forward 30 seconds" aria-label="Fast forward 30 seconds">
</button>
</div>
<div class="playback__timers">
<time class="ssp-timer" aria-live="off">00:00</time>
<span>/</span>
<!-- We need actual duration here from the server -->
<time class="ssp-duration" datetime="PT0H40M51S">40:51</time>
</div>
</div>
</div>
</div>
<nav class="player-panels-nav">
<button class="subscribe-btn" id="subscribe-btn-96" title="Subscribe" aria-expanded="false">Subscribe</button>
<button class="share-btn" id="share-btn-96" title="Share" aria-expanded="false">Share</button>
</nav>
</div>
</div>
</div>
<div class="player-panels player-panels-96">
<div class="subscribe player-panel subscribe-96">
<button class="close-btn close-btn-96" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="panel__inner">
<div class="subscribe-icons">
<a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" rel="noopener noreferrer"
class="apple_podcasts"
title="Subscribe on Apple Podcasts">
<span></span>
Apple Podcasts </a>
<a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" rel="noopener noreferrer"
class="stitcher"
title="Subscribe on Stitcher">
<span></span>
Stitcher </a>
<a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" rel="noopener noreferrer"
class="google_podcasts"
title="Subscribe on Google Podcasts">
<span></span>
Google Podcasts </a>
<a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" rel="noopener noreferrer"
class="spotify"
title="Subscribe on Spotify">
<span></span>
Spotify </a>
</div>
<div class="player-panel-row" role="group" aria-label="RSS Feed URL">
<div class="title">RSS Feed</div>
<div>
<input value="https://downtimeproject.com/feed/podcast/the-downtime-project" class="input-rss input-rss-96" title="RSS Feed URL" readonly />
</div>
<button class="copy-rss copy-rss-96" title="Copy RSS Feed URL" aria-label="Copy RSS Feed URL"></button>
</div>
</div>
</div>
<div class="share share-96 player-panel">
<button class="close-btn close-btn-96" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="player-panel-row">
<div class="title">
Share </div>
<div class="icons-holder">
<a href="https://www.facebook.com/sharer/sharer.php?u=https://downtimeproject.com/podcast/salesforce-publishes-a-controversial-postmortem-and-breaks-their-dns/&t=Salesforce Publishes a Controversial Postmortem (and breaks their DNS)"
target="_blank" rel="noopener noreferrer" class="share-icon facebook" title="Share on Facebook" aria-label="Share on Facebook">
<span></span>
</a>
<a href="https://twitter.com/intent/tweet?text=https://downtimeproject.com/podcast/salesforce-publishes-a-controversial-postmortem-and-breaks-their-dns/&url=Salesforce Publishes a Controversial Postmortem (and breaks their DNS)"
target="_blank" rel="noopener noreferrer" class="share-icon twitter" title="Share on Twitter" aria-label="Share on Twitter">
<span></span>
</a>
<a href="https://downtimeproject.com/files/2021/05/DP-011-Mix1.mp3"
target="_blank" rel="noopener noreferrer" class="share-icon download" title="Download" aria-label="Download" download>
<span></span>
</a>
</div>
</div>
<div class="player-panel-row">
<div class="title">
Link </div>
<div>
<input value="https://downtimeproject.com/podcast/salesforce-publishes-a-controversial-postmortem-and-breaks-their-dns/" class="input-link input-link-96" title="Episode URL" readonly />
</div>
<button class="copy-link copy-link-96" title="Copy Episode URL" aria-label="Copy Episode URL"></button>
</div>
<div class="player-panel-row">
<div class="title">
Embed </div>
<div>
<textarea rows="1"
title="Embed Code"
class="input-embed input-embed-96" readonly>&lt;blockquote class=&quot;wp-embedded-content&quot; data-secret=&quot;ye3OsxMBYQ&quot;&gt;&lt;a href=&quot;https://downtimeproject.com/podcast/salesforce-publishes-a-controversial-postmortem-and-breaks-their-dns/&quot;&gt;Salesforce Publishes a Controversial Postmortem (and breaks their DNS)&lt;/a&gt;&lt;/blockquote&gt;&lt;iframe sandbox=&quot;allow-scripts&quot; security=&quot;restricted&quot; src=&quot;https://downtimeproject.com/podcast/salesforce-publishes-a-controversial-postmortem-and-breaks-their-dns/embed/#?secret=ye3OsxMBYQ&quot; width=&quot;500&quot; height=&quot;350&quot; title=&quot;&amp;#8220;Salesforce Publishes a Controversial Postmortem (and breaks their DNS)&amp;#8221; &amp;#8212; The Downtime Project&quot; data-secret=&quot;ye3OsxMBYQ&quot; frameborder=&quot;0&quot; marginwidth=&quot;0&quot; marginheight=&quot;0&quot; scrolling=&quot;no&quot; class=&quot;wp-embedded-content&quot;&gt;&lt;/iframe&gt;&lt;script&gt;
/*! This file is auto-generated */
!function(d,l){&quot;use strict&quot;;l.querySelector&amp;&amp;d.addEventListener&amp;&amp;&quot;undefined&quot;!=typeof URL&amp;&amp;(d.wp=d.wp||{},d.wp.receiveEmbedMessage||(d.wp.receiveEmbedMessage=function(e){var t=e.data;if((t||t.secret||t.message||t.value)&amp;&amp;!/[^a-zA-Z0-9]/.test(t.secret)){for(var s,r,n,a=l.querySelectorAll(&#039;iframe[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),o=l.querySelectorAll(&#039;blockquote[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),c=new RegExp(&quot;^https?:$&quot;,&quot;i&quot;),i=0;i&lt;o.length;i++)o[i].style.display=&quot;none&quot;;for(i=0;i&lt;a.length;i++)s=a[i],e.source===s.contentWindow&amp;&amp;(s.removeAttribute(&quot;style&quot;),&quot;height&quot;===t.message?(1e3&lt;(r=parseInt(t.value,10))?r=1e3:~~r&lt;200&amp;&amp;(r=200),s.height=r):&quot;link&quot;===t.message&amp;&amp;(r=new URL(s.getAttribute(&quot;src&quot;)),n=new URL(t.value),c.test(n.protocol))&amp;&amp;n.host===r.host&amp;&amp;l.activeElement===s&amp;&amp;(d.top.location.href=t.value))}},d.addEventListener(&quot;message&quot;,d.wp.receiveEmbedMessage,!1),l.addEventListener(&quot;DOMContentLoaded&quot;,function(){for(var e,t,s=l.querySelectorAll(&quot;iframe.wp-embedded-content&quot;),r=0;r&lt;s.length;r++)(t=(e=s[r]).getAttribute(&quot;data-secret&quot;))||(t=Math.random().toString(36).substring(2,12),e.src+=&quot;#?secret=&quot;+t,e.setAttribute(&quot;data-secret&quot;,t)),e.contentWindow.postMessage({message:&quot;ready&quot;,secret:t},&quot;*&quot;)},!1)))}(window,document);
//# sourceURL=https://downtimeproject.com/wp-includes/js/wp-embed.min.js
&lt;/script&gt;
</textarea>
</div>
<button class="copy-embed copy-embed-96" title="Copy Embed Code" aria-label="Copy Embed Code"></button>
</div>
</div>
</div>
</div>
<div class="podcast_meta"><aside><p><a href="https://downtimeproject.com/podcast-download/96/salesforce-publishes-a-controversial-postmortem-and-breaks-their-dns?ref=download" title="Salesforce Publishes a Controversial Postmortem (and breaks their DNS) " class="podcast-meta-download">Download file</a> | <a href="https://downtimeproject.com/podcast-download/96/salesforce-publishes-a-controversial-postmortem-and-breaks-their-dns?ref=new_window" target="_blank" title="Salesforce Publishes a Controversial Postmortem (and breaks their DNS) " class="podcast-meta-new-window">Play in new window</a> | <span class="podcast-meta-duration">Duration: 40:51</span></p><p>Subscribe: <a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" title="Apple Podcasts" class="podcast-meta-itunes">Apple Podcasts</a> | <a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" title="Stitcher" class="podcast-meta-itunes">Stitcher</a> | <a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" title="Google Podcasts" class="podcast-meta-itunes">Google Podcasts</a> | <a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" title="Spotify" class="podcast-meta-itunes">Spotify</a></p></aside></div></div>
<p class="wp-block-paragraph">On May 11, 2021, Salesforce had a multi hour outage that affected numerous services.&nbsp; Their <a href="https://help.salesforce.com/articleView?id=000358392&amp;mode=1&amp;type=1">public writeup</a> was somewhat controversial &#8212; it’s the first one we’ve done on this show that called out the actions of a single individual in a negative light. The latest <a href="https://sreweekly.com/sre-weekly-issue-272/">SRE Weekly</a> has a good list of some different articles on the subject.&nbsp;</p>
<p class="wp-block-paragraph">In this episode, Tom and Jamie talk through the outage and all the different ways that losing DNS can break things, as well as weighing in on why this post-mortem is not a good example of how the industry should treat outages.&nbsp;</p>
<p><a href="https://downtimeproject.com/podcast/salesforce-publishes-a-controversial-postmortem-and-breaks-their-dns/#more-96" class="more-link">FULL TRANSCRIPT</a></p>
</div>
<!-- .entry-content -->
<footer class="entry-meta">
<span>6,809 Words</span> <span class="sep"> | </span>
<span class="comments-link"><a href="https://downtimeproject.com/podcast/salesforce-publishes-a-controversial-postmortem-and-breaks-their-dns/#respond">Comment</a></span>
</footer>
<!-- .entry-meta -->
</article><!-- #post-96 -->
<article id="post-93" class="post-93 podcast type-podcast status-publish series-the-downtime-project h-entry hentry">
<header class="entry-header">
<h1 class="entry-title p-name">
<a href="https://downtimeproject.com/podcast/kinesis-hits-the-thread-limit/" title="Permalink to Kinesis Hits the Thread Limit" rel="bookmark">Kinesis Hits the Thread Limit</a>
</h1>
</header>
<!-- .entry-header -->
<div class="entry-content e-content">
<div class="podcast_player"><div id="1465968040" class="castos-player dark-mode " role="region" aria-label="Podcast player" data-episode="93" data-player_id="1465968040">
<div class="player">
<div class="player__main">
<div class="player__artwork player__artwork-93">
<img decoding="async" src="https://downtimeproject.com/files/2021/03/dp_logo-150x150.png"
alt="The Downtime Project"
title="The Downtime Project">
</div>
<div class="player__body">
<div class="currently-playing">
<div class="show player__podcast-title">
The Downtime Project </div>
<div class="episode-title player__episode-title">Kinesis Hits the Thread Limit</div>
</div>
<div class="play-progress">
<div class="play-pause-controls">
<button title="Play" aria-label="Play Episode" aria-pressed="false" class="play-btn">
<span class="screen-reader-text">Play Episode</span>
</button>
<button title="Pause" aria-label="Pause Episode" aria-pressed="false" class="pause-btn hide">
<span class="screen-reader-text">Pause Episode</span>
</button>
<img decoding="async" src="https://downtimeproject.com/wp-content/plugins/seriously-simple-podcasting/assets/css/images/player/images/icon-loader.svg" alt="Loading" class="ssp-loader hide"/>
</div>
<div>
<audio preload="none" class="clip clip-93">
<source src="https://downtimeproject.com/files/2021/05/DP-010-Mix1.mp3">
</audio>
<div class="ssp-progress" role="progressbar" title="Seek" aria-valuenow="0" aria-valuemin="0" aria-valuemax="2682">
<span class="progress__filled"></span>
</div>
<div class="ssp-playback playback">
<div class="playback__controls">
<button class="player-btn player-btn__volume" title="Mute/Unmute" aria-label="Mute audio" aria-pressed="false">
</button>
<button data-skip="-10" class="player-btn player-btn__rwd" title="Rewind 10 seconds" aria-label="Rewind 10 seconds">
</button>
<button data-speed="1" class="player-btn player-btn__speed" title="Playback Speed" aria-label="Playback speed, currently 1x">1x</button>
<button data-skip="30" class="player-btn player-btn__fwd" title="Fast Forward 30 seconds" aria-label="Fast forward 30 seconds">
</button>
</div>
<div class="playback__timers">
<time class="ssp-timer" aria-live="off">00:00</time>
<span>/</span>
<!-- We need actual duration here from the server -->
<time class="ssp-duration" datetime="PT0H44M42S">44:42</time>
</div>
</div>
</div>
</div>
<nav class="player-panels-nav">
<button class="subscribe-btn" id="subscribe-btn-93" title="Subscribe" aria-expanded="false">Subscribe</button>
<button class="share-btn" id="share-btn-93" title="Share" aria-expanded="false">Share</button>
</nav>
</div>
</div>
</div>
<div class="player-panels player-panels-93">
<div class="subscribe player-panel subscribe-93">
<button class="close-btn close-btn-93" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="panel__inner">
<div class="subscribe-icons">
<a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" rel="noopener noreferrer"
class="apple_podcasts"
title="Subscribe on Apple Podcasts">
<span></span>
Apple Podcasts </a>
<a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" rel="noopener noreferrer"
class="stitcher"
title="Subscribe on Stitcher">
<span></span>
Stitcher </a>
<a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" rel="noopener noreferrer"
class="google_podcasts"
title="Subscribe on Google Podcasts">
<span></span>
Google Podcasts </a>
<a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" rel="noopener noreferrer"
class="spotify"
title="Subscribe on Spotify">
<span></span>
Spotify </a>
</div>
<div class="player-panel-row" role="group" aria-label="RSS Feed URL">
<div class="title">RSS Feed</div>
<div>
<input value="https://downtimeproject.com/feed/podcast/the-downtime-project" class="input-rss input-rss-93" title="RSS Feed URL" readonly />
</div>
<button class="copy-rss copy-rss-93" title="Copy RSS Feed URL" aria-label="Copy RSS Feed URL"></button>
</div>
</div>
</div>
<div class="share share-93 player-panel">
<button class="close-btn close-btn-93" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="player-panel-row">
<div class="title">
Share </div>
<div class="icons-holder">
<a href="https://www.facebook.com/sharer/sharer.php?u=https://downtimeproject.com/podcast/kinesis-hits-the-thread-limit/&t=Kinesis Hits the Thread Limit"
target="_blank" rel="noopener noreferrer" class="share-icon facebook" title="Share on Facebook" aria-label="Share on Facebook">
<span></span>
</a>
<a href="https://twitter.com/intent/tweet?text=https://downtimeproject.com/podcast/kinesis-hits-the-thread-limit/&url=Kinesis Hits the Thread Limit"
target="_blank" rel="noopener noreferrer" class="share-icon twitter" title="Share on Twitter" aria-label="Share on Twitter">
<span></span>
</a>
<a href="https://downtimeproject.com/files/2021/05/DP-010-Mix1.mp3"
target="_blank" rel="noopener noreferrer" class="share-icon download" title="Download" aria-label="Download" download>
<span></span>
</a>
</div>
</div>
<div class="player-panel-row">
<div class="title">
Link </div>
<div>
<input value="https://downtimeproject.com/podcast/kinesis-hits-the-thread-limit/" class="input-link input-link-93" title="Episode URL" readonly />
</div>
<button class="copy-link copy-link-93" title="Copy Episode URL" aria-label="Copy Episode URL"></button>
</div>
<div class="player-panel-row">
<div class="title">
Embed </div>
<div>
<textarea rows="1"
title="Embed Code"
class="input-embed input-embed-93" readonly>&lt;blockquote class=&quot;wp-embedded-content&quot; data-secret=&quot;X9WUZhlsu7&quot;&gt;&lt;a href=&quot;https://downtimeproject.com/podcast/kinesis-hits-the-thread-limit/&quot;&gt;Kinesis Hits the Thread Limit&lt;/a&gt;&lt;/blockquote&gt;&lt;iframe sandbox=&quot;allow-scripts&quot; security=&quot;restricted&quot; src=&quot;https://downtimeproject.com/podcast/kinesis-hits-the-thread-limit/embed/#?secret=X9WUZhlsu7&quot; width=&quot;500&quot; height=&quot;350&quot; title=&quot;&amp;#8220;Kinesis Hits the Thread Limit&amp;#8221; &amp;#8212; The Downtime Project&quot; data-secret=&quot;X9WUZhlsu7&quot; frameborder=&quot;0&quot; marginwidth=&quot;0&quot; marginheight=&quot;0&quot; scrolling=&quot;no&quot; class=&quot;wp-embedded-content&quot;&gt;&lt;/iframe&gt;&lt;script&gt;
/*! This file is auto-generated */
!function(d,l){&quot;use strict&quot;;l.querySelector&amp;&amp;d.addEventListener&amp;&amp;&quot;undefined&quot;!=typeof URL&amp;&amp;(d.wp=d.wp||{},d.wp.receiveEmbedMessage||(d.wp.receiveEmbedMessage=function(e){var t=e.data;if((t||t.secret||t.message||t.value)&amp;&amp;!/[^a-zA-Z0-9]/.test(t.secret)){for(var s,r,n,a=l.querySelectorAll(&#039;iframe[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),o=l.querySelectorAll(&#039;blockquote[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),c=new RegExp(&quot;^https?:$&quot;,&quot;i&quot;),i=0;i&lt;o.length;i++)o[i].style.display=&quot;none&quot;;for(i=0;i&lt;a.length;i++)s=a[i],e.source===s.contentWindow&amp;&amp;(s.removeAttribute(&quot;style&quot;),&quot;height&quot;===t.message?(1e3&lt;(r=parseInt(t.value,10))?r=1e3:~~r&lt;200&amp;&amp;(r=200),s.height=r):&quot;link&quot;===t.message&amp;&amp;(r=new URL(s.getAttribute(&quot;src&quot;)),n=new URL(t.value),c.test(n.protocol))&amp;&amp;n.host===r.host&amp;&amp;l.activeElement===s&amp;&amp;(d.top.location.href=t.value))}},d.addEventListener(&quot;message&quot;,d.wp.receiveEmbedMessage,!1),l.addEventListener(&quot;DOMContentLoaded&quot;,function(){for(var e,t,s=l.querySelectorAll(&quot;iframe.wp-embedded-content&quot;),r=0;r&lt;s.length;r++)(t=(e=s[r]).getAttribute(&quot;data-secret&quot;))||(t=Math.random().toString(36).substring(2,12),e.src+=&quot;#?secret=&quot;+t,e.setAttribute(&quot;data-secret&quot;,t)),e.contentWindow.postMessage({message:&quot;ready&quot;,secret:t},&quot;*&quot;)},!1)))}(window,document);
//# sourceURL=https://downtimeproject.com/wp-includes/js/wp-embed.min.js
&lt;/script&gt;
</textarea>
</div>
<button class="copy-embed copy-embed-93" title="Copy Embed Code" aria-label="Copy Embed Code"></button>
</div>
</div>
</div>
</div>
<div class="podcast_meta"><aside><p><a href="https://downtimeproject.com/podcast-download/93/kinesis-hits-the-thread-limit?ref=download" title="Kinesis Hits the Thread Limit " class="podcast-meta-download">Download file</a> | <a href="https://downtimeproject.com/podcast-download/93/kinesis-hits-the-thread-limit?ref=new_window" target="_blank" title="Kinesis Hits the Thread Limit " class="podcast-meta-new-window">Play in new window</a> | <span class="podcast-meta-duration">Duration: 44:42</span> | <span class="podcast-meta-date">Recorded on May 20, 2021</span></p><p>Subscribe: <a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" title="Apple Podcasts" class="podcast-meta-itunes">Apple Podcasts</a> | <a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" title="Stitcher" class="podcast-meta-itunes">Stitcher</a> | <a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" title="Google Podcasts" class="podcast-meta-itunes">Google Podcasts</a> | <a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" title="Spotify" class="podcast-meta-itunes">Spotify</a></p></aside></div></div>
<p class="wp-block-paragraph">During a routine addition of some servers to the Kinesis front end cluster in US-East-1 in November 2020, AWS ran into an OS limit on the max number of threads. That resulted in a multi hour outage that affected a number of other AWS servers, including ECS, EKS, Cognito, and Cloudwatch.</p>
<p class="wp-block-paragraph">We probably won’t do a full episode on it, but this reminded Tom of one of his favorite historical outages: S3’s 2008 outage which combined single-bit corruption with gossip protocols. If you haven’t read it, definitely check out <a href="https://status.aws.amazon.com/s3-20080720.html">the post-mortem</a>.</p>
<p><a href="https://downtimeproject.com/podcast/kinesis-hits-the-thread-limit/#more-93" class="more-link">FULL TRANSCRIPT</a></p>
</div>
<!-- .entry-content -->
<footer class="entry-meta">
<span>7,362 Words</span> <span class="sep"> | </span>
<span class="comments-link"><a href="https://downtimeproject.com/podcast/kinesis-hits-the-thread-limit/#respond">Comment</a></span>
</footer>
<!-- .entry-meta -->
</article><!-- #post-93 -->
<article id="post-83" class="post-83 podcast type-podcast status-publish series-the-downtime-project h-entry hentry">
<header class="entry-header">
<h1 class="entry-title p-name">
<a href="https://downtimeproject.com/podcast/how-coinbase-unleashed-a-thundering-herd/" title="Permalink to How Coinbase Unleashed a Thundering Herd" rel="bookmark">How Coinbase Unleashed a Thundering Herd</a>
</h1>
</header>
<!-- .entry-header -->
<div class="entry-content e-content">
<div class="podcast_player"><div id="1370083180" class="castos-player dark-mode " role="region" aria-label="Podcast player" data-episode="83" data-player_id="1370083180">
<div class="player">
<div class="player__main">
<div class="player__artwork player__artwork-83">
<img decoding="async" src="https://downtimeproject.com/files/2021/03/dp_logo-150x150.png"
alt="The Downtime Project"
title="The Downtime Project">
</div>
<div class="player__body">
<div class="currently-playing">
<div class="show player__podcast-title">
The Downtime Project </div>
<div class="episode-title player__episode-title">How Coinbase Unleashed a Thundering Herd</div>
</div>
<div class="play-progress">
<div class="play-pause-controls">
<button title="Play" aria-label="Play Episode" aria-pressed="false" class="play-btn">
<span class="screen-reader-text">Play Episode</span>
</button>
<button title="Pause" aria-label="Pause Episode" aria-pressed="false" class="pause-btn hide">
<span class="screen-reader-text">Pause Episode</span>
</button>
<img decoding="async" src="https://downtimeproject.com/wp-content/plugins/seriously-simple-podcasting/assets/css/images/player/images/icon-loader.svg" alt="Loading" class="ssp-loader hide"/>
</div>
<div>
<audio preload="none" class="clip clip-83">
<source src="https://downtimeproject.com/files/2021/05/DP-009-Mix1.mp3">
</audio>
<div class="ssp-progress" role="progressbar" title="Seek" aria-valuenow="0" aria-valuemin="0" aria-valuemax="2296">
<span class="progress__filled"></span>
</div>
<div class="ssp-playback playback">
<div class="playback__controls">
<button class="player-btn player-btn__volume" title="Mute/Unmute" aria-label="Mute audio" aria-pressed="false">
</button>
<button data-skip="-10" class="player-btn player-btn__rwd" title="Rewind 10 seconds" aria-label="Rewind 10 seconds">
</button>
<button data-speed="1" class="player-btn player-btn__speed" title="Playback Speed" aria-label="Playback speed, currently 1x">1x</button>
<button data-skip="30" class="player-btn player-btn__fwd" title="Fast Forward 30 seconds" aria-label="Fast forward 30 seconds">
</button>
</div>
<div class="playback__timers">
<time class="ssp-timer" aria-live="off">00:00</time>
<span>/</span>
<!-- We need actual duration here from the server -->
<time class="ssp-duration" datetime="PT0H38M16S">38:16</time>
</div>
</div>
</div>
</div>
<nav class="player-panels-nav">
<button class="subscribe-btn" id="subscribe-btn-83" title="Subscribe" aria-expanded="false">Subscribe</button>
<button class="share-btn" id="share-btn-83" title="Share" aria-expanded="false">Share</button>
</nav>
</div>
</div>
</div>
<div class="player-panels player-panels-83">
<div class="subscribe player-panel subscribe-83">
<button class="close-btn close-btn-83" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="panel__inner">
<div class="subscribe-icons">
<a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" rel="noopener noreferrer"
class="apple_podcasts"
title="Subscribe on Apple Podcasts">
<span></span>
Apple Podcasts </a>
<a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" rel="noopener noreferrer"
class="stitcher"
title="Subscribe on Stitcher">
<span></span>
Stitcher </a>
<a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" rel="noopener noreferrer"
class="google_podcasts"
title="Subscribe on Google Podcasts">
<span></span>
Google Podcasts </a>
<a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" rel="noopener noreferrer"
class="spotify"
title="Subscribe on Spotify">
<span></span>
Spotify </a>
</div>
<div class="player-panel-row" role="group" aria-label="RSS Feed URL">
<div class="title">RSS Feed</div>
<div>
<input value="https://downtimeproject.com/feed/podcast/the-downtime-project" class="input-rss input-rss-83" title="RSS Feed URL" readonly />
</div>
<button class="copy-rss copy-rss-83" title="Copy RSS Feed URL" aria-label="Copy RSS Feed URL"></button>
</div>
</div>
</div>
<div class="share share-83 player-panel">
<button class="close-btn close-btn-83" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="player-panel-row">
<div class="title">
Share </div>
<div class="icons-holder">
<a href="https://www.facebook.com/sharer/sharer.php?u=https://downtimeproject.com/podcast/how-coinbase-unleashed-a-thundering-herd/&t=How Coinbase Unleashed a Thundering Herd"
target="_blank" rel="noopener noreferrer" class="share-icon facebook" title="Share on Facebook" aria-label="Share on Facebook">
<span></span>
</a>
<a href="https://twitter.com/intent/tweet?text=https://downtimeproject.com/podcast/how-coinbase-unleashed-a-thundering-herd/&url=How Coinbase Unleashed a Thundering Herd"
target="_blank" rel="noopener noreferrer" class="share-icon twitter" title="Share on Twitter" aria-label="Share on Twitter">
<span></span>
</a>
<a href="https://downtimeproject.com/files/2021/05/DP-009-Mix1.mp3"
target="_blank" rel="noopener noreferrer" class="share-icon download" title="Download" aria-label="Download" download>
<span></span>
</a>
</div>
</div>
<div class="player-panel-row">
<div class="title">
Link </div>
<div>
<input value="https://downtimeproject.com/podcast/how-coinbase-unleashed-a-thundering-herd/" class="input-link input-link-83" title="Episode URL" readonly />
</div>
<button class="copy-link copy-link-83" title="Copy Episode URL" aria-label="Copy Episode URL"></button>
</div>
<div class="player-panel-row">
<div class="title">
Embed </div>
<div>
<textarea rows="1"
title="Embed Code"
class="input-embed input-embed-83" readonly>&lt;blockquote class=&quot;wp-embedded-content&quot; data-secret=&quot;bD5IcNmO4b&quot;&gt;&lt;a href=&quot;https://downtimeproject.com/podcast/how-coinbase-unleashed-a-thundering-herd/&quot;&gt;How Coinbase Unleashed a Thundering Herd&lt;/a&gt;&lt;/blockquote&gt;&lt;iframe sandbox=&quot;allow-scripts&quot; security=&quot;restricted&quot; src=&quot;https://downtimeproject.com/podcast/how-coinbase-unleashed-a-thundering-herd/embed/#?secret=bD5IcNmO4b&quot; width=&quot;500&quot; height=&quot;350&quot; title=&quot;&amp;#8220;How Coinbase Unleashed a Thundering Herd&amp;#8221; &amp;#8212; The Downtime Project&quot; data-secret=&quot;bD5IcNmO4b&quot; frameborder=&quot;0&quot; marginwidth=&quot;0&quot; marginheight=&quot;0&quot; scrolling=&quot;no&quot; class=&quot;wp-embedded-content&quot;&gt;&lt;/iframe&gt;&lt;script&gt;
/*! This file is auto-generated */
!function(d,l){&quot;use strict&quot;;l.querySelector&amp;&amp;d.addEventListener&amp;&amp;&quot;undefined&quot;!=typeof URL&amp;&amp;(d.wp=d.wp||{},d.wp.receiveEmbedMessage||(d.wp.receiveEmbedMessage=function(e){var t=e.data;if((t||t.secret||t.message||t.value)&amp;&amp;!/[^a-zA-Z0-9]/.test(t.secret)){for(var s,r,n,a=l.querySelectorAll(&#039;iframe[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),o=l.querySelectorAll(&#039;blockquote[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),c=new RegExp(&quot;^https?:$&quot;,&quot;i&quot;),i=0;i&lt;o.length;i++)o[i].style.display=&quot;none&quot;;for(i=0;i&lt;a.length;i++)s=a[i],e.source===s.contentWindow&amp;&amp;(s.removeAttribute(&quot;style&quot;),&quot;height&quot;===t.message?(1e3&lt;(r=parseInt(t.value,10))?r=1e3:~~r&lt;200&amp;&amp;(r=200),s.height=r):&quot;link&quot;===t.message&amp;&amp;(r=new URL(s.getAttribute(&quot;src&quot;)),n=new URL(t.value),c.test(n.protocol))&amp;&amp;n.host===r.host&amp;&amp;l.activeElement===s&amp;&amp;(d.top.location.href=t.value))}},d.addEventListener(&quot;message&quot;,d.wp.receiveEmbedMessage,!1),l.addEventListener(&quot;DOMContentLoaded&quot;,function(){for(var e,t,s=l.querySelectorAll(&quot;iframe.wp-embedded-content&quot;),r=0;r&lt;s.length;r++)(t=(e=s[r]).getAttribute(&quot;data-secret&quot;))||(t=Math.random().toString(36).substring(2,12),e.src+=&quot;#?secret=&quot;+t,e.setAttribute(&quot;data-secret&quot;,t)),e.contentWindow.postMessage({message:&quot;ready&quot;,secret:t},&quot;*&quot;)},!1)))}(window,document);
//# sourceURL=https://downtimeproject.com/wp-includes/js/wp-embed.min.js
&lt;/script&gt;
</textarea>
</div>
<button class="copy-embed copy-embed-83" title="Copy Embed Code" aria-label="Copy Embed Code"></button>
</div>
</div>
</div>
</div>
<div class="podcast_meta"><aside><p><a href="https://downtimeproject.com/podcast-download/83/how-coinbase-unleashed-a-thundering-herd?ref=download" title="How Coinbase Unleashed a Thundering Herd " class="podcast-meta-download">Download file</a> | <a href="https://downtimeproject.com/podcast-download/83/how-coinbase-unleashed-a-thundering-herd?ref=new_window" target="_blank" title="How Coinbase Unleashed a Thundering Herd " class="podcast-meta-new-window">Play in new window</a> | <span class="podcast-meta-duration">Duration: 38:16</span></p><p>Subscribe: <a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" title="Apple Podcasts" class="podcast-meta-itunes">Apple Podcasts</a> | <a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" title="Stitcher" class="podcast-meta-itunes">Stitcher</a> | <a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" title="Google Podcasts" class="podcast-meta-itunes">Google Podcasts</a> | <a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" title="Spotify" class="podcast-meta-itunes">Spotify</a></p></aside></div></div>
<p class="wp-block-paragraph">In November 2020, Coinbase <a href="https://blog.coinbase.com/incident-postmortem-november-16th-2020-234805a82df5">had a problem</a> while rotating their internal TLS certificates and accidentally unleashed a huge amount of traffic on some internal services. This was a refreshingly non-database related incident that led to an interesting discussion about the future of infrastructure as code, the limits of human code review, and how many load balancers might be too many. </p>
<p><a href="https://downtimeproject.com/podcast/how-coinbase-unleashed-a-thundering-herd/#more-83" class="more-link">FULL TRANSCRIPT</a></p>
</div>
<!-- .entry-content -->
<footer class="entry-meta">
<span>6,299 Words</span> <span class="sep"> | </span>
<span class="comments-link"><a href="https://downtimeproject.com/podcast/how-coinbase-unleashed-a-thundering-herd/#respond">Comment</a></span>
</footer>
<!-- .entry-meta -->
</article><!-- #post-83 -->
<article id="post-80" class="post-80 podcast type-podcast status-publish series-the-downtime-project h-entry hentry">
<header class="entry-header">
<h1 class="entry-title p-name">
<a href="https://downtimeproject.com/podcast/auth0s-seriously-congested-database/" title="Permalink to Auth0’s Seriously Congested Database" rel="bookmark">Auth0’s Seriously Congested Database</a>
</h1>
</header>
<!-- .entry-header -->
<div class="entry-content e-content">
<div class="podcast_player"><div id="3963167108" class="castos-player dark-mode " role="region" aria-label="Podcast player" data-episode="80" data-player_id="3963167108">
<div class="player">
<div class="player__main">
<div class="player__artwork player__artwork-80">
<img decoding="async" src="https://downtimeproject.com/files/2021/03/dp_logo-150x150.png"
alt="The Downtime Project"
title="The Downtime Project">
</div>
<div class="player__body">
<div class="currently-playing">
<div class="show player__podcast-title">
The Downtime Project </div>
<div class="episode-title player__episode-title">Auth0’s Seriously Congested Database</div>
</div>
<div class="play-progress">
<div class="play-pause-controls">
<button title="Play" aria-label="Play Episode" aria-pressed="false" class="play-btn">
<span class="screen-reader-text">Play Episode</span>
</button>
<button title="Pause" aria-label="Pause Episode" aria-pressed="false" class="pause-btn hide">
<span class="screen-reader-text">Pause Episode</span>
</button>
<img decoding="async" src="https://downtimeproject.com/wp-content/plugins/seriously-simple-podcasting/assets/css/images/player/images/icon-loader.svg" alt="Loading" class="ssp-loader hide"/>
</div>
<div>
<audio preload="none" class="clip clip-80">
<source src="https://downtimeproject.com/files/2021/05/DP-008-Mix1.mp3">
</audio>
<div class="ssp-progress" role="progressbar" title="Seek" aria-valuenow="0" aria-valuemin="0" aria-valuemax="60">
<span class="progress__filled"></span>
</div>
<div class="ssp-playback playback">
<div class="playback__controls">
<button class="player-btn player-btn__volume" title="Mute/Unmute" aria-label="Mute audio" aria-pressed="false">
</button>
<button data-skip="-10" class="player-btn player-btn__rwd" title="Rewind 10 seconds" aria-label="Rewind 10 seconds">
</button>
<button data-speed="1" class="player-btn player-btn__speed" title="Playback Speed" aria-label="Playback speed, currently 1x">1x</button>
<button data-skip="30" class="player-btn player-btn__fwd" title="Fast Forward 30 seconds" aria-label="Fast forward 30 seconds">
</button>
</div>
<div class="playback__timers">
<time class="ssp-timer" aria-live="off">00:00</time>
<span>/</span>
<!-- We need actual duration here from the server -->
<time class="ssp-duration" datetime="PT0H1M0S">1:00</time>
</div>
</div>
</div>
</div>
<nav class="player-panels-nav">
<button class="subscribe-btn" id="subscribe-btn-80" title="Subscribe" aria-expanded="false">Subscribe</button>
<button class="share-btn" id="share-btn-80" title="Share" aria-expanded="false">Share</button>
</nav>
</div>
</div>
</div>
<div class="player-panels player-panels-80">
<div class="subscribe player-panel subscribe-80">
<button class="close-btn close-btn-80" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="panel__inner">
<div class="subscribe-icons">
<a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" rel="noopener noreferrer"
class="apple_podcasts"
title="Subscribe on Apple Podcasts">
<span></span>
Apple Podcasts </a>
<a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" rel="noopener noreferrer"
class="stitcher"
title="Subscribe on Stitcher">
<span></span>
Stitcher </a>
<a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" rel="noopener noreferrer"
class="google_podcasts"
title="Subscribe on Google Podcasts">
<span></span>
Google Podcasts </a>
<a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" rel="noopener noreferrer"
class="spotify"
title="Subscribe on Spotify">
<span></span>
Spotify </a>
</div>
<div class="player-panel-row" role="group" aria-label="RSS Feed URL">
<div class="title">RSS Feed</div>
<div>
<input value="https://downtimeproject.com/feed/podcast/the-downtime-project" class="input-rss input-rss-80" title="RSS Feed URL" readonly />
</div>
<button class="copy-rss copy-rss-80" title="Copy RSS Feed URL" aria-label="Copy RSS Feed URL"></button>
</div>
</div>
</div>
<div class="share share-80 player-panel">
<button class="close-btn close-btn-80" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="player-panel-row">
<div class="title">
Share </div>
<div class="icons-holder">
<a href="https://www.facebook.com/sharer/sharer.php?u=https://downtimeproject.com/podcast/auth0s-seriously-congested-database/&t=Auth0’s Seriously Congested Database"
target="_blank" rel="noopener noreferrer" class="share-icon facebook" title="Share on Facebook" aria-label="Share on Facebook">
<span></span>
</a>
<a href="https://twitter.com/intent/tweet?text=https://downtimeproject.com/podcast/auth0s-seriously-congested-database/&url=Auth0’s Seriously Congested Database"
target="_blank" rel="noopener noreferrer" class="share-icon twitter" title="Share on Twitter" aria-label="Share on Twitter">
<span></span>
</a>
<a href="https://downtimeproject.com/files/2021/05/DP-008-Mix1.mp3"
target="_blank" rel="noopener noreferrer" class="share-icon download" title="Download" aria-label="Download" download>
<span></span>
</a>
</div>
</div>
<div class="player-panel-row">
<div class="title">
Link </div>
<div>
<input value="https://downtimeproject.com/podcast/auth0s-seriously-congested-database/" class="input-link input-link-80" title="Episode URL" readonly />
</div>
<button class="copy-link copy-link-80" title="Copy Episode URL" aria-label="Copy Episode URL"></button>
</div>
<div class="player-panel-row">
<div class="title">
Embed </div>
<div>
<textarea rows="1"
title="Embed Code"
class="input-embed input-embed-80" readonly>&lt;blockquote class=&quot;wp-embedded-content&quot; data-secret=&quot;757P9AgMBO&quot;&gt;&lt;a href=&quot;https://downtimeproject.com/podcast/auth0s-seriously-congested-database/&quot;&gt;Auth0’s Seriously Congested Database&lt;/a&gt;&lt;/blockquote&gt;&lt;iframe sandbox=&quot;allow-scripts&quot; security=&quot;restricted&quot; src=&quot;https://downtimeproject.com/podcast/auth0s-seriously-congested-database/embed/#?secret=757P9AgMBO&quot; width=&quot;500&quot; height=&quot;350&quot; title=&quot;&amp;#8220;Auth0’s Seriously Congested Database&amp;#8221; &amp;#8212; The Downtime Project&quot; data-secret=&quot;757P9AgMBO&quot; frameborder=&quot;0&quot; marginwidth=&quot;0&quot; marginheight=&quot;0&quot; scrolling=&quot;no&quot; class=&quot;wp-embedded-content&quot;&gt;&lt;/iframe&gt;&lt;script&gt;
/*! This file is auto-generated */
!function(d,l){&quot;use strict&quot;;l.querySelector&amp;&amp;d.addEventListener&amp;&amp;&quot;undefined&quot;!=typeof URL&amp;&amp;(d.wp=d.wp||{},d.wp.receiveEmbedMessage||(d.wp.receiveEmbedMessage=function(e){var t=e.data;if((t||t.secret||t.message||t.value)&amp;&amp;!/[^a-zA-Z0-9]/.test(t.secret)){for(var s,r,n,a=l.querySelectorAll(&#039;iframe[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),o=l.querySelectorAll(&#039;blockquote[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),c=new RegExp(&quot;^https?:$&quot;,&quot;i&quot;),i=0;i&lt;o.length;i++)o[i].style.display=&quot;none&quot;;for(i=0;i&lt;a.length;i++)s=a[i],e.source===s.contentWindow&amp;&amp;(s.removeAttribute(&quot;style&quot;),&quot;height&quot;===t.message?(1e3&lt;(r=parseInt(t.value,10))?r=1e3:~~r&lt;200&amp;&amp;(r=200),s.height=r):&quot;link&quot;===t.message&amp;&amp;(r=new URL(s.getAttribute(&quot;src&quot;)),n=new URL(t.value),c.test(n.protocol))&amp;&amp;n.host===r.host&amp;&amp;l.activeElement===s&amp;&amp;(d.top.location.href=t.value))}},d.addEventListener(&quot;message&quot;,d.wp.receiveEmbedMessage,!1),l.addEventListener(&quot;DOMContentLoaded&quot;,function(){for(var e,t,s=l.querySelectorAll(&quot;iframe.wp-embedded-content&quot;),r=0;r&lt;s.length;r++)(t=(e=s[r]).getAttribute(&quot;data-secret&quot;))||(t=Math.random().toString(36).substring(2,12),e.src+=&quot;#?secret=&quot;+t,e.setAttribute(&quot;data-secret&quot;,t)),e.contentWindow.postMessage({message:&quot;ready&quot;,secret:t},&quot;*&quot;)},!1)))}(window,document);
//# sourceURL=https://downtimeproject.com/wp-includes/js/wp-embed.min.js
&lt;/script&gt;
</textarea>
</div>
<button class="copy-embed copy-embed-80" title="Copy Embed Code" aria-label="Copy Embed Code"></button>
</div>
</div>
</div>
</div>
<div class="podcast_meta"><aside><p><a href="https://downtimeproject.com/podcast-download/80/auth0s-seriously-congested-database?ref=download" title="Auth0’s Seriously Congested Database " class="podcast-meta-download">Download file</a> | <a href="https://downtimeproject.com/podcast-download/80/auth0s-seriously-congested-database?ref=new_window" target="_blank" title="Auth0’s Seriously Congested Database " class="podcast-meta-new-window">Play in new window</a> | <span class="podcast-meta-duration">Duration: 1:00</span> | <span class="podcast-meta-date">Recorded on May 6, 2021</span></p><p>Subscribe: <a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" title="Apple Podcasts" class="podcast-meta-itunes">Apple Podcasts</a> | <a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" title="Stitcher" class="podcast-meta-itunes">Stitcher</a> | <a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" title="Google Podcasts" class="podcast-meta-itunes">Google Podcasts</a> | <a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" title="Spotify" class="podcast-meta-itunes">Spotify</a></p></aside></div></div>
<p class="wp-block-paragraph">Just one day after we released <a href="https://downtimeproject.com/podcast/auth0-silently-loses-some-indexes/">Episode 5 about Auth0’s 2018 outage</a>, Auth0 suffered a <a href="https://cdn.auth0.com/blog/Detailed_Root_Cause_Analysis_(RCA)_4-2021.pdf">4 hour, 20 minute outage</a> that was caused by a combination of several large queries and a series of database cache misses.  This was a very serious outage, as many users were unable to log in to sites across the internet.</p>
<p class="wp-block-paragraph">This episode has a lot of discussion on caching, engineering leadership, and keeping your databases happy.&nbsp;</p>
<p><a href="https://downtimeproject.com/podcast/auth0s-seriously-congested-database/#more-80" class="more-link">FULL TRANSCRIPT</a></p>
</div>
<!-- .entry-content -->
<footer class="entry-meta">
<span>10,838 Words</span> <span class="sep"> | </span>
<span class="comments-link"><a href="https://downtimeproject.com/podcast/auth0s-seriously-congested-database/#respond">Comment</a></span>
</footer>
<!-- .entry-meta -->
</article><!-- #post-80 -->
<article id="post-75" class="post-75 podcast type-podcast status-publish series-the-downtime-project h-entry hentry">
<header class="entry-header">
<h1 class="entry-title p-name">
<a href="https://downtimeproject.com/podcast/talkin-testing-with-sujay-jayakar/" title="Permalink to Talkin&#8217; Testing with Sujay Jayakar" rel="bookmark">Talkin&#8217; Testing with Sujay Jayakar</a>
</h1>
</header>
<!-- .entry-header -->
<div class="entry-content e-content">
<div class="podcast_player"><div id="2991729721" class="castos-player dark-mode " role="region" aria-label="Podcast player" data-episode="75" data-player_id="2991729721">
<div class="player">
<div class="player__main">
<div class="player__artwork player__artwork-75">
<img decoding="async" src="https://downtimeproject.com/files/2021/03/dp_logo-150x150.png"
alt="The Downtime Project"
title="The Downtime Project">
</div>
<div class="player__body">
<div class="currently-playing">
<div class="show player__podcast-title">
The Downtime Project </div>
<div class="episode-title player__episode-title">Talkin' Testing with Sujay Jayakar</div>
</div>
<div class="play-progress">
<div class="play-pause-controls">
<button title="Play" aria-label="Play Episode" aria-pressed="false" class="play-btn">
<span class="screen-reader-text">Play Episode</span>
</button>
<button title="Pause" aria-label="Pause Episode" aria-pressed="false" class="pause-btn hide">
<span class="screen-reader-text">Pause Episode</span>
</button>
<img decoding="async" src="https://downtimeproject.com/wp-content/plugins/seriously-simple-podcasting/assets/css/images/player/images/icon-loader.svg" alt="Loading" class="ssp-loader hide"/>
</div>
<div>
<audio preload="none" class="clip clip-75">
<source src="https://downtimeproject.com/files/2021/05/DP-007-Mix1.mp3">
</audio>
<div class="ssp-progress" role="progressbar" title="Seek" aria-valuenow="0" aria-valuemin="0" aria-valuemax="1777">
<span class="progress__filled"></span>
</div>
<div class="ssp-playback playback">
<div class="playback__controls">
<button class="player-btn player-btn__volume" title="Mute/Unmute" aria-label="Mute audio" aria-pressed="false">
</button>
<button data-skip="-10" class="player-btn player-btn__rwd" title="Rewind 10 seconds" aria-label="Rewind 10 seconds">
</button>
<button data-speed="1" class="player-btn player-btn__speed" title="Playback Speed" aria-label="Playback speed, currently 1x">1x</button>
<button data-skip="30" class="player-btn player-btn__fwd" title="Fast Forward 30 seconds" aria-label="Fast forward 30 seconds">
</button>
</div>
<div class="playback__timers">
<time class="ssp-timer" aria-live="off">00:00</time>
<span>/</span>
<!-- We need actual duration here from the server -->
<time class="ssp-duration" datetime="PT0H29M37S">29:37</time>
</div>
</div>
</div>
</div>
<nav class="player-panels-nav">
<button class="subscribe-btn" id="subscribe-btn-75" title="Subscribe" aria-expanded="false">Subscribe</button>
<button class="share-btn" id="share-btn-75" title="Share" aria-expanded="false">Share</button>
</nav>
</div>
</div>
</div>
<div class="player-panels player-panels-75">
<div class="subscribe player-panel subscribe-75">
<button class="close-btn close-btn-75" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="panel__inner">
<div class="subscribe-icons">
<a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" rel="noopener noreferrer"
class="apple_podcasts"
title="Subscribe on Apple Podcasts">
<span></span>
Apple Podcasts </a>
<a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" rel="noopener noreferrer"
class="stitcher"
title="Subscribe on Stitcher">
<span></span>
Stitcher </a>
<a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" rel="noopener noreferrer"
class="google_podcasts"
title="Subscribe on Google Podcasts">
<span></span>
Google Podcasts </a>
<a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" rel="noopener noreferrer"
class="spotify"
title="Subscribe on Spotify">
<span></span>
Spotify </a>
</div>
<div class="player-panel-row" role="group" aria-label="RSS Feed URL">
<div class="title">RSS Feed</div>
<div>
<input value="https://downtimeproject.com/feed/podcast/the-downtime-project" class="input-rss input-rss-75" title="RSS Feed URL" readonly />
</div>
<button class="copy-rss copy-rss-75" title="Copy RSS Feed URL" aria-label="Copy RSS Feed URL"></button>
</div>
</div>
</div>
<div class="share share-75 player-panel">
<button class="close-btn close-btn-75" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="player-panel-row">
<div class="title">
Share </div>
<div class="icons-holder">
<a href="https://www.facebook.com/sharer/sharer.php?u=https://downtimeproject.com/podcast/talkin-testing-with-sujay-jayakar/&t=Talkin&#039; Testing with Sujay Jayakar"
target="_blank" rel="noopener noreferrer" class="share-icon facebook" title="Share on Facebook" aria-label="Share on Facebook">
<span></span>
</a>
<a href="https://twitter.com/intent/tweet?text=https://downtimeproject.com/podcast/talkin-testing-with-sujay-jayakar/&url=Talkin&#039; Testing with Sujay Jayakar"
target="_blank" rel="noopener noreferrer" class="share-icon twitter" title="Share on Twitter" aria-label="Share on Twitter">
<span></span>
</a>
<a href="https://downtimeproject.com/files/2021/05/DP-007-Mix1.mp3"
target="_blank" rel="noopener noreferrer" class="share-icon download" title="Download" aria-label="Download" download>
<span></span>
</a>
</div>
</div>
<div class="player-panel-row">
<div class="title">
Link </div>
<div>
<input value="https://downtimeproject.com/podcast/talkin-testing-with-sujay-jayakar/" class="input-link input-link-75" title="Episode URL" readonly />
</div>
<button class="copy-link copy-link-75" title="Copy Episode URL" aria-label="Copy Episode URL"></button>
</div>
<div class="player-panel-row">
<div class="title">
Embed </div>
<div>
<textarea rows="1"
title="Embed Code"
class="input-embed input-embed-75" readonly>&lt;blockquote class=&quot;wp-embedded-content&quot; data-secret=&quot;UbbeYZReMQ&quot;&gt;&lt;a href=&quot;https://downtimeproject.com/podcast/talkin-testing-with-sujay-jayakar/&quot;&gt;Talkin&amp;#8217; Testing with Sujay Jayakar&lt;/a&gt;&lt;/blockquote&gt;&lt;iframe sandbox=&quot;allow-scripts&quot; security=&quot;restricted&quot; src=&quot;https://downtimeproject.com/podcast/talkin-testing-with-sujay-jayakar/embed/#?secret=UbbeYZReMQ&quot; width=&quot;500&quot; height=&quot;350&quot; title=&quot;&amp;#8220;Talkin&amp;#8217; Testing with Sujay Jayakar&amp;#8221; &amp;#8212; The Downtime Project&quot; data-secret=&quot;UbbeYZReMQ&quot; frameborder=&quot;0&quot; marginwidth=&quot;0&quot; marginheight=&quot;0&quot; scrolling=&quot;no&quot; class=&quot;wp-embedded-content&quot;&gt;&lt;/iframe&gt;&lt;script&gt;
/*! This file is auto-generated */
!function(d,l){&quot;use strict&quot;;l.querySelector&amp;&amp;d.addEventListener&amp;&amp;&quot;undefined&quot;!=typeof URL&amp;&amp;(d.wp=d.wp||{},d.wp.receiveEmbedMessage||(d.wp.receiveEmbedMessage=function(e){var t=e.data;if((t||t.secret||t.message||t.value)&amp;&amp;!/[^a-zA-Z0-9]/.test(t.secret)){for(var s,r,n,a=l.querySelectorAll(&#039;iframe[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),o=l.querySelectorAll(&#039;blockquote[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),c=new RegExp(&quot;^https?:$&quot;,&quot;i&quot;),i=0;i&lt;o.length;i++)o[i].style.display=&quot;none&quot;;for(i=0;i&lt;a.length;i++)s=a[i],e.source===s.contentWindow&amp;&amp;(s.removeAttribute(&quot;style&quot;),&quot;height&quot;===t.message?(1e3&lt;(r=parseInt(t.value,10))?r=1e3:~~r&lt;200&amp;&amp;(r=200),s.height=r):&quot;link&quot;===t.message&amp;&amp;(r=new URL(s.getAttribute(&quot;src&quot;)),n=new URL(t.value),c.test(n.protocol))&amp;&amp;n.host===r.host&amp;&amp;l.activeElement===s&amp;&amp;(d.top.location.href=t.value))}},d.addEventListener(&quot;message&quot;,d.wp.receiveEmbedMessage,!1),l.addEventListener(&quot;DOMContentLoaded&quot;,function(){for(var e,t,s=l.querySelectorAll(&quot;iframe.wp-embedded-content&quot;),r=0;r&lt;s.length;r++)(t=(e=s[r]).getAttribute(&quot;data-secret&quot;))||(t=Math.random().toString(36).substring(2,12),e.src+=&quot;#?secret=&quot;+t,e.setAttribute(&quot;data-secret&quot;,t)),e.contentWindow.postMessage({message:&quot;ready&quot;,secret:t},&quot;*&quot;)},!1)))}(window,document);
//# sourceURL=https://downtimeproject.com/wp-includes/js/wp-embed.min.js
&lt;/script&gt;
</textarea>
</div>
<button class="copy-embed copy-embed-75" title="Copy Embed Code" aria-label="Copy Embed Code"></button>
</div>
</div>
</div>
</div>
<div class="podcast_meta"><aside><p><a href="https://downtimeproject.com/podcast-download/75/talkin-testing-with-sujay-jayakar?ref=download" title="Talkin&#8217; Testing with Sujay Jayakar " class="podcast-meta-download">Download file</a> | <a href="https://downtimeproject.com/podcast-download/75/talkin-testing-with-sujay-jayakar?ref=new_window" target="_blank" title="Talkin&#8217; Testing with Sujay Jayakar " class="podcast-meta-new-window">Play in new window</a> | <span class="podcast-meta-duration">Duration: 29:37</span> | <span class="podcast-meta-date">Recorded on April 29, 2021</span></p><p>Subscribe: <a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" title="Apple Podcasts" class="podcast-meta-itunes">Apple Podcasts</a> | <a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" title="Stitcher" class="podcast-meta-itunes">Stitcher</a> | <a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" title="Google Podcasts" class="podcast-meta-itunes">Google Podcasts</a> | <a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" title="Spotify" class="podcast-meta-itunes">Spotify</a></p></aside></div></div>
<p class="wp-block-paragraph">Tom was feeling under the weather after joining Team Pfizer last week, so today we have a special guest episode with <a href="https://www.linkedin.com/in/sujayakar/">Sujay Jayakar</a>, Jamie&#8217;s co-founder and engineer extraordinaire.</p>
<p class="wp-block-paragraph">While it&#8217;s great to respond well to an outage, it&#8217;s even better to design and test systems in such a way that outages don&#8217;t happen. As we saw in the<a href="https://downtimeproject.com/podcast/one-subtle-regex-takes-down-cloudflare/"> Cloudflare outage episode</a> from a few weeks ago, there can be very unexpected results from code if all possible variations haven&#8217;t been tested, which is hard for humans to do.</p>
<p class="wp-block-paragraph">In this week&#8217;s episode, Jamie and Sujay talk about some of the ways to use automated tests to drive down the number of untested states in your programs.</p>
<p class="wp-block-paragraph">Some links that came up during the discussion:</p>
<ul class="wp-block-list"><li>Jason Warner (GitHub CTO) <a href="https://twitter.com/jasoncwarner/status/1386895224056602629">tweeted about</a> what it was like behind the scenes during the big outage we discussed last week.</li><li>How Dropbox <a href="https://dropbox.tech/infrastructure/rewriting-the-heart-of-our-sync-engine">decided to rewrite their sync engine</a> and <a href="https://dropbox.tech/infrastructure/-testing-our-new-sync-engine">how they tested it</a></li><li>The <a href="https://en.wikipedia.org/wiki/QuickCheck">QuickCheck</a> testing framework.</li></ul>
<p><a href="https://downtimeproject.com/podcast/talkin-testing-with-sujay-jayakar/#more-75" class="more-link">FULL TRANSCRIPT</a></p>
</div>
<!-- .entry-content -->
<footer class="entry-meta">
<span>4,982 Words</span> <span class="sep"> | </span>
<span class="comments-link"><a href="https://downtimeproject.com/podcast/talkin-testing-with-sujay-jayakar/#respond">Comment</a></span>
</footer>
<!-- .entry-meta -->
</article><!-- #post-75 -->
<article id="post-71" class="post-71 podcast type-podcast status-publish series-the-downtime-project h-entry hentry">
<header class="entry-header">
<h1 class="entry-title p-name">
<a href="https://downtimeproject.com/podcast/githubs-43-second-network-partition/" title="Permalink to GitHub&#8217;s 43 Second Network Partition" rel="bookmark">GitHub&#8217;s 43 Second Network Partition</a>
</h1>
</header>
<!-- .entry-header -->
<div class="entry-content e-content">
<div class="podcast_player"><div id="2124802903" class="castos-player dark-mode " role="region" aria-label="Podcast player" data-episode="71" data-player_id="2124802903">
<div class="player">
<div class="player__main">
<div class="player__artwork player__artwork-71">
<img decoding="async" src="https://downtimeproject.com/files/2021/03/dp_logo-150x150.png"
alt="The Downtime Project"
title="The Downtime Project">
</div>
<div class="player__body">
<div class="currently-playing">
<div class="show player__podcast-title">
The Downtime Project </div>
<div class="episode-title player__episode-title">GitHub's 43 Second Network Partition</div>
</div>
<div class="play-progress">
<div class="play-pause-controls">
<button title="Play" aria-label="Play Episode" aria-pressed="false" class="play-btn">
<span class="screen-reader-text">Play Episode</span>
</button>
<button title="Pause" aria-label="Pause Episode" aria-pressed="false" class="pause-btn hide">
<span class="screen-reader-text">Pause Episode</span>
</button>
<img decoding="async" src="https://downtimeproject.com/wp-content/plugins/seriously-simple-podcasting/assets/css/images/player/images/icon-loader.svg" alt="Loading" class="ssp-loader hide"/>
</div>
<div>
<audio preload="none" class="clip clip-71">
<source src="https://downtimeproject.com/files/2021/04/DP-006-Mix1.mp3">
</audio>
<div class="ssp-progress" role="progressbar" title="Seek" aria-valuenow="0" aria-valuemin="0" aria-valuemax="3194">
<span class="progress__filled"></span>
</div>
<div class="ssp-playback playback">
<div class="playback__controls">
<button class="player-btn player-btn__volume" title="Mute/Unmute" aria-label="Mute audio" aria-pressed="false">
</button>
<button data-skip="-10" class="player-btn player-btn__rwd" title="Rewind 10 seconds" aria-label="Rewind 10 seconds">
</button>
<button data-speed="1" class="player-btn player-btn__speed" title="Playback Speed" aria-label="Playback speed, currently 1x">1x</button>
<button data-skip="30" class="player-btn player-btn__fwd" title="Fast Forward 30 seconds" aria-label="Fast forward 30 seconds">
</button>
</div>
<div class="playback__timers">
<time class="ssp-timer" aria-live="off">00:00</time>
<span>/</span>
<!-- We need actual duration here from the server -->
<time class="ssp-duration" datetime="PT0H53M14S">53:14</time>
</div>
</div>
</div>
</div>
<nav class="player-panels-nav">
<button class="subscribe-btn" id="subscribe-btn-71" title="Subscribe" aria-expanded="false">Subscribe</button>
<button class="share-btn" id="share-btn-71" title="Share" aria-expanded="false">Share</button>
</nav>
</div>
</div>
</div>
<div class="player-panels player-panels-71">
<div class="subscribe player-panel subscribe-71">
<button class="close-btn close-btn-71" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="panel__inner">
<div class="subscribe-icons">
<a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" rel="noopener noreferrer"
class="apple_podcasts"
title="Subscribe on Apple Podcasts">
<span></span>
Apple Podcasts </a>
<a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" rel="noopener noreferrer"
class="stitcher"
title="Subscribe on Stitcher">
<span></span>
Stitcher </a>
<a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" rel="noopener noreferrer"
class="google_podcasts"
title="Subscribe on Google Podcasts">
<span></span>
Google Podcasts </a>
<a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" rel="noopener noreferrer"
class="spotify"
title="Subscribe on Spotify">
<span></span>
Spotify </a>
</div>
<div class="player-panel-row" role="group" aria-label="RSS Feed URL">
<div class="title">RSS Feed</div>
<div>
<input value="https://downtimeproject.com/feed/podcast/the-downtime-project" class="input-rss input-rss-71" title="RSS Feed URL" readonly />
</div>
<button class="copy-rss copy-rss-71" title="Copy RSS Feed URL" aria-label="Copy RSS Feed URL"></button>
</div>
</div>
</div>
<div class="share share-71 player-panel">
<button class="close-btn close-btn-71" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="player-panel-row">
<div class="title">
Share </div>
<div class="icons-holder">
<a href="https://www.facebook.com/sharer/sharer.php?u=https://downtimeproject.com/podcast/githubs-43-second-network-partition/&t=GitHub&#039;s 43 Second Network Partition"
target="_blank" rel="noopener noreferrer" class="share-icon facebook" title="Share on Facebook" aria-label="Share on Facebook">
<span></span>
</a>
<a href="https://twitter.com/intent/tweet?text=https://downtimeproject.com/podcast/githubs-43-second-network-partition/&url=GitHub&#039;s 43 Second Network Partition"
target="_blank" rel="noopener noreferrer" class="share-icon twitter" title="Share on Twitter" aria-label="Share on Twitter">
<span></span>
</a>
<a href="https://downtimeproject.com/files/2021/04/DP-006-Mix1.mp3"
target="_blank" rel="noopener noreferrer" class="share-icon download" title="Download" aria-label="Download" download>
<span></span>
</a>
</div>
</div>
<div class="player-panel-row">
<div class="title">
Link </div>
<div>
<input value="https://downtimeproject.com/podcast/githubs-43-second-network-partition/" class="input-link input-link-71" title="Episode URL" readonly />
</div>
<button class="copy-link copy-link-71" title="Copy Episode URL" aria-label="Copy Episode URL"></button>
</div>
<div class="player-panel-row">
<div class="title">
Embed </div>
<div>
<textarea rows="1"
title="Embed Code"
class="input-embed input-embed-71" readonly>&lt;blockquote class=&quot;wp-embedded-content&quot; data-secret=&quot;VxJ4Oen3CL&quot;&gt;&lt;a href=&quot;https://downtimeproject.com/podcast/githubs-43-second-network-partition/&quot;&gt;GitHub&amp;#8217;s 43 Second Network Partition&lt;/a&gt;&lt;/blockquote&gt;&lt;iframe sandbox=&quot;allow-scripts&quot; security=&quot;restricted&quot; src=&quot;https://downtimeproject.com/podcast/githubs-43-second-network-partition/embed/#?secret=VxJ4Oen3CL&quot; width=&quot;500&quot; height=&quot;350&quot; title=&quot;&amp;#8220;GitHub&amp;#8217;s 43 Second Network Partition&amp;#8221; &amp;#8212; The Downtime Project&quot; data-secret=&quot;VxJ4Oen3CL&quot; frameborder=&quot;0&quot; marginwidth=&quot;0&quot; marginheight=&quot;0&quot; scrolling=&quot;no&quot; class=&quot;wp-embedded-content&quot;&gt;&lt;/iframe&gt;&lt;script&gt;
/*! This file is auto-generated */
!function(d,l){&quot;use strict&quot;;l.querySelector&amp;&amp;d.addEventListener&amp;&amp;&quot;undefined&quot;!=typeof URL&amp;&amp;(d.wp=d.wp||{},d.wp.receiveEmbedMessage||(d.wp.receiveEmbedMessage=function(e){var t=e.data;if((t||t.secret||t.message||t.value)&amp;&amp;!/[^a-zA-Z0-9]/.test(t.secret)){for(var s,r,n,a=l.querySelectorAll(&#039;iframe[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),o=l.querySelectorAll(&#039;blockquote[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),c=new RegExp(&quot;^https?:$&quot;,&quot;i&quot;),i=0;i&lt;o.length;i++)o[i].style.display=&quot;none&quot;;for(i=0;i&lt;a.length;i++)s=a[i],e.source===s.contentWindow&amp;&amp;(s.removeAttribute(&quot;style&quot;),&quot;height&quot;===t.message?(1e3&lt;(r=parseInt(t.value,10))?r=1e3:~~r&lt;200&amp;&amp;(r=200),s.height=r):&quot;link&quot;===t.message&amp;&amp;(r=new URL(s.getAttribute(&quot;src&quot;)),n=new URL(t.value),c.test(n.protocol))&amp;&amp;n.host===r.host&amp;&amp;l.activeElement===s&amp;&amp;(d.top.location.href=t.value))}},d.addEventListener(&quot;message&quot;,d.wp.receiveEmbedMessage,!1),l.addEventListener(&quot;DOMContentLoaded&quot;,function(){for(var e,t,s=l.querySelectorAll(&quot;iframe.wp-embedded-content&quot;),r=0;r&lt;s.length;r++)(t=(e=s[r]).getAttribute(&quot;data-secret&quot;))||(t=Math.random().toString(36).substring(2,12),e.src+=&quot;#?secret=&quot;+t,e.setAttribute(&quot;data-secret&quot;,t)),e.contentWindow.postMessage({message:&quot;ready&quot;,secret:t},&quot;*&quot;)},!1)))}(window,document);
//# sourceURL=https://downtimeproject.com/wp-includes/js/wp-embed.min.js
&lt;/script&gt;
</textarea>
</div>
<button class="copy-embed copy-embed-71" title="Copy Embed Code" aria-label="Copy Embed Code"></button>
</div>
</div>
</div>
</div>
<div class="podcast_meta"><aside><p><a href="https://downtimeproject.com/podcast-download/71/githubs-43-second-network-partition?ref=download" title="GitHub&#8217;s 43 Second Network Partition " class="podcast-meta-download">Download file</a> | <a href="https://downtimeproject.com/podcast-download/71/githubs-43-second-network-partition?ref=new_window" target="_blank" title="GitHub&#8217;s 43 Second Network Partition " class="podcast-meta-new-window">Play in new window</a> | <span class="podcast-meta-duration">Duration: 53:14</span> | <span class="podcast-meta-date">Recorded on April 22, 2021</span></p><p>Subscribe: <a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" title="Apple Podcasts" class="podcast-meta-itunes">Apple Podcasts</a> | <a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" title="Stitcher" class="podcast-meta-itunes">Stitcher</a> | <a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" title="Google Podcasts" class="podcast-meta-itunes">Google Podcasts</a> | <a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" title="Spotify" class="podcast-meta-itunes">Spotify</a></p></aside></div></div>
<p class="wp-block-paragraph">In 2018, after 43 seconds of connectivity issues between their East and West coast datacenters and a rapid promotion of a new primary, GitHub ended up with unique data written to two different databases. As detailed in the <a href="https://github.blog/2018-10-30-oct21-post-incident-analysis/" data-type="URL" data-id="https://github.blog/2018-10-30-oct21-post-incident-analysis/">postmortem</a>, this resulted in 24 hours of degraded service. </p>
<p class="wp-block-paragraph"></p>
<p class="wp-block-paragraph">This episode spends a lot of time on MySQL replication and the different types of problems that it can both prevent and cause. The issues in this outage should be familiar to anyone that has had to deal with the problems when a large amount of data replicated between multiple servers gets slightly out of sync. </p>
<p><a href="https://downtimeproject.com/podcast/githubs-43-second-network-partition/#more-71" class="more-link">FULL TRANSCRIPT</a></p>
</div>
<!-- .entry-content -->
<footer class="entry-meta">
<span>9,722 Words</span> <span class="sep"> | </span>
<span class="comments-link"><a href="https://downtimeproject.com/podcast/githubs-43-second-network-partition/#comments">2 Comments</a></span>
</footer>
<!-- .entry-meta -->
</article><!-- #post-71 -->
<article id="post-67" class="post-67 podcast type-podcast status-publish series-the-downtime-project h-entry hentry">
<header class="entry-header">
<h1 class="entry-title p-name">
<a href="https://downtimeproject.com/podcast/auth0-silently-loses-some-indexes/" title="Permalink to Auth0 Silently Loses Some Indexes" rel="bookmark">Auth0 Silently Loses Some Indexes</a>
</h1>
</header>
<!-- .entry-header -->
<div class="entry-content e-content">
<div class="podcast_player"><div id="2113599592" class="castos-player dark-mode " role="region" aria-label="Podcast player" data-episode="67" data-player_id="2113599592">
<div class="player">
<div class="player__main">
<div class="player__artwork player__artwork-67">
<img decoding="async" src="https://downtimeproject.com/files/2021/03/dp_logo-150x150.png"
alt="The Downtime Project"
title="The Downtime Project">
</div>
<div class="player__body">
<div class="currently-playing">
<div class="show player__podcast-title">
The Downtime Project </div>
<div class="episode-title player__episode-title">Auth0 Silently Loses Some Indexes</div>
</div>
<div class="play-progress">
<div class="play-pause-controls">
<button title="Play" aria-label="Play Episode" aria-pressed="false" class="play-btn">
<span class="screen-reader-text">Play Episode</span>
</button>
<button title="Pause" aria-label="Pause Episode" aria-pressed="false" class="pause-btn hide">
<span class="screen-reader-text">Pause Episode</span>
</button>
<img decoding="async" src="https://downtimeproject.com/wp-content/plugins/seriously-simple-podcasting/assets/css/images/player/images/icon-loader.svg" alt="Loading" class="ssp-loader hide"/>
</div>
<div>
<audio preload="none" class="clip clip-67">
<source src="https://downtimeproject.com/files/2021/04/DP-005-Mix1.mp3">
</audio>
<div class="ssp-progress" role="progressbar" title="Seek" aria-valuenow="0" aria-valuemin="0" aria-valuemax="2820">
<span class="progress__filled"></span>
</div>
<div class="ssp-playback playback">
<div class="playback__controls">
<button class="player-btn player-btn__volume" title="Mute/Unmute" aria-label="Mute audio" aria-pressed="false">
</button>
<button data-skip="-10" class="player-btn player-btn__rwd" title="Rewind 10 seconds" aria-label="Rewind 10 seconds">
</button>
<button data-speed="1" class="player-btn player-btn__speed" title="Playback Speed" aria-label="Playback speed, currently 1x">1x</button>
<button data-skip="30" class="player-btn player-btn__fwd" title="Fast Forward 30 seconds" aria-label="Fast forward 30 seconds">
</button>
</div>
<div class="playback__timers">
<time class="ssp-timer" aria-live="off">00:00</time>
<span>/</span>
<!-- We need actual duration here from the server -->
<time class="ssp-duration" datetime="PT0H47M0S">47:00</time>
</div>
</div>
</div>
</div>
<nav class="player-panels-nav">
<button class="subscribe-btn" id="subscribe-btn-67" title="Subscribe" aria-expanded="false">Subscribe</button>
<button class="share-btn" id="share-btn-67" title="Share" aria-expanded="false">Share</button>
</nav>
</div>
</div>
</div>
<div class="player-panels player-panels-67">
<div class="subscribe player-panel subscribe-67">
<button class="close-btn close-btn-67" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="panel__inner">
<div class="subscribe-icons">
<a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" rel="noopener noreferrer"
class="apple_podcasts"
title="Subscribe on Apple Podcasts">
<span></span>
Apple Podcasts </a>
<a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" rel="noopener noreferrer"
class="stitcher"
title="Subscribe on Stitcher">
<span></span>
Stitcher </a>
<a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" rel="noopener noreferrer"
class="google_podcasts"
title="Subscribe on Google Podcasts">
<span></span>
Google Podcasts </a>
<a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" rel="noopener noreferrer"
class="spotify"
title="Subscribe on Spotify">
<span></span>
Spotify </a>
</div>
<div class="player-panel-row" role="group" aria-label="RSS Feed URL">
<div class="title">RSS Feed</div>
<div>
<input value="https://downtimeproject.com/feed/podcast/the-downtime-project" class="input-rss input-rss-67" title="RSS Feed URL" readonly />
</div>
<button class="copy-rss copy-rss-67" title="Copy RSS Feed URL" aria-label="Copy RSS Feed URL"></button>
</div>
</div>
</div>
<div class="share share-67 player-panel">
<button class="close-btn close-btn-67" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="player-panel-row">
<div class="title">
Share </div>
<div class="icons-holder">
<a href="https://www.facebook.com/sharer/sharer.php?u=https://downtimeproject.com/podcast/auth0-silently-loses-some-indexes/&t=Auth0 Silently Loses Some Indexes"
target="_blank" rel="noopener noreferrer" class="share-icon facebook" title="Share on Facebook" aria-label="Share on Facebook">
<span></span>
</a>
<a href="https://twitter.com/intent/tweet?text=https://downtimeproject.com/podcast/auth0-silently-loses-some-indexes/&url=Auth0 Silently Loses Some Indexes"
target="_blank" rel="noopener noreferrer" class="share-icon twitter" title="Share on Twitter" aria-label="Share on Twitter">
<span></span>
</a>
<a href="https://downtimeproject.com/files/2021/04/DP-005-Mix1.mp3"
target="_blank" rel="noopener noreferrer" class="share-icon download" title="Download" aria-label="Download" download>
<span></span>
</a>
</div>
</div>
<div class="player-panel-row">
<div class="title">
Link </div>
<div>
<input value="https://downtimeproject.com/podcast/auth0-silently-loses-some-indexes/" class="input-link input-link-67" title="Episode URL" readonly />
</div>
<button class="copy-link copy-link-67" title="Copy Episode URL" aria-label="Copy Episode URL"></button>
</div>
<div class="player-panel-row">
<div class="title">
Embed </div>
<div>
<textarea rows="1"
title="Embed Code"
class="input-embed input-embed-67" readonly>&lt;blockquote class=&quot;wp-embedded-content&quot; data-secret=&quot;MrQ2KDh9o8&quot;&gt;&lt;a href=&quot;https://downtimeproject.com/podcast/auth0-silently-loses-some-indexes/&quot;&gt;Auth0 Silently Loses Some Indexes&lt;/a&gt;&lt;/blockquote&gt;&lt;iframe sandbox=&quot;allow-scripts&quot; security=&quot;restricted&quot; src=&quot;https://downtimeproject.com/podcast/auth0-silently-loses-some-indexes/embed/#?secret=MrQ2KDh9o8&quot; width=&quot;500&quot; height=&quot;350&quot; title=&quot;&amp;#8220;Auth0 Silently Loses Some Indexes&amp;#8221; &amp;#8212; The Downtime Project&quot; data-secret=&quot;MrQ2KDh9o8&quot; frameborder=&quot;0&quot; marginwidth=&quot;0&quot; marginheight=&quot;0&quot; scrolling=&quot;no&quot; class=&quot;wp-embedded-content&quot;&gt;&lt;/iframe&gt;&lt;script&gt;
/*! This file is auto-generated */
!function(d,l){&quot;use strict&quot;;l.querySelector&amp;&amp;d.addEventListener&amp;&amp;&quot;undefined&quot;!=typeof URL&amp;&amp;(d.wp=d.wp||{},d.wp.receiveEmbedMessage||(d.wp.receiveEmbedMessage=function(e){var t=e.data;if((t||t.secret||t.message||t.value)&amp;&amp;!/[^a-zA-Z0-9]/.test(t.secret)){for(var s,r,n,a=l.querySelectorAll(&#039;iframe[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),o=l.querySelectorAll(&#039;blockquote[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),c=new RegExp(&quot;^https?:$&quot;,&quot;i&quot;),i=0;i&lt;o.length;i++)o[i].style.display=&quot;none&quot;;for(i=0;i&lt;a.length;i++)s=a[i],e.source===s.contentWindow&amp;&amp;(s.removeAttribute(&quot;style&quot;),&quot;height&quot;===t.message?(1e3&lt;(r=parseInt(t.value,10))?r=1e3:~~r&lt;200&amp;&amp;(r=200),s.height=r):&quot;link&quot;===t.message&amp;&amp;(r=new URL(s.getAttribute(&quot;src&quot;)),n=new URL(t.value),c.test(n.protocol))&amp;&amp;n.host===r.host&amp;&amp;l.activeElement===s&amp;&amp;(d.top.location.href=t.value))}},d.addEventListener(&quot;message&quot;,d.wp.receiveEmbedMessage,!1),l.addEventListener(&quot;DOMContentLoaded&quot;,function(){for(var e,t,s=l.querySelectorAll(&quot;iframe.wp-embedded-content&quot;),r=0;r&lt;s.length;r++)(t=(e=s[r]).getAttribute(&quot;data-secret&quot;))||(t=Math.random().toString(36).substring(2,12),e.src+=&quot;#?secret=&quot;+t,e.setAttribute(&quot;data-secret&quot;,t)),e.contentWindow.postMessage({message:&quot;ready&quot;,secret:t},&quot;*&quot;)},!1)))}(window,document);
//# sourceURL=https://downtimeproject.com/wp-includes/js/wp-embed.min.js
&lt;/script&gt;
</textarea>
</div>
<button class="copy-embed copy-embed-67" title="Copy Embed Code" aria-label="Copy Embed Code"></button>
</div>
</div>
</div>
</div>
<div class="podcast_meta"><aside><p><a href="https://downtimeproject.com/podcast-download/67/auth0-silently-loses-some-indexes?ref=download" title="Auth0 Silently Loses Some Indexes " class="podcast-meta-download">Download file</a> | <a href="https://downtimeproject.com/podcast-download/67/auth0-silently-loses-some-indexes?ref=new_window" target="_blank" title="Auth0 Silently Loses Some Indexes " class="podcast-meta-new-window">Play in new window</a> | <span class="podcast-meta-duration">Duration: 47:00</span></p><p>Subscribe: <a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" title="Apple Podcasts" class="podcast-meta-itunes">Apple Podcasts</a> | <a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" title="Stitcher" class="podcast-meta-itunes">Stitcher</a> | <a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" title="Google Podcasts" class="podcast-meta-itunes">Google Podcasts</a> | <a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" title="Spotify" class="podcast-meta-itunes">Spotify</a></p></aside></div></div>
<p class="wp-block-paragraph">Auth0 experienced multiple hours of degraded performance and increased error rates in November of 2018 after several unexpected events, including a migration that dropped some indexes from their database.</p>
<p class="wp-block-paragraph">The <a href="https://cdn.auth0.com/blog/20181128-Incident-RCA.pdf">published post-mortem</a> has a full timeline and a great list of action items, though it is curiously missing a few details, like exactly what database the company was using.&nbsp;</p>
<p class="wp-block-paragraph">In this episode, you’ll learn a little about what Auth0 does and what happened during the outage itself. You’ll also hear about the things that went well and went poorly, and some speculation about whether something like <a href="http://honeycomb.io/">Honeycomb.io</a> could have reduced the amount of time to resolve the outage.  </p>
<p><a href="https://downtimeproject.com/podcast/auth0-silently-loses-some-indexes/#more-67" class="more-link">FULL TRAnscript</a></p>
</div>
<!-- .entry-content -->
<footer class="entry-meta">
<span>8,567 Words</span> <span class="sep"> | </span>
<span class="comments-link"><a href="https://downtimeproject.com/podcast/auth0-silently-loses-some-indexes/#respond">Comment</a></span>
</footer>
<!-- .entry-meta -->
</article><!-- #post-67 -->
<article id="post-64" class="post-64 podcast type-podcast status-publish series-the-downtime-project h-entry hentry">
<header class="entry-header">
<h1 class="entry-title p-name">
<a href="https://downtimeproject.com/podcast/one-subtle-regex-takes-down-cloudflare/" title="Permalink to One Subtle Regex Takes Down Cloudflare" rel="bookmark">One Subtle Regex Takes Down Cloudflare</a>
</h1>
</header>
<!-- .entry-header -->
<div class="entry-content e-content">
<div class="podcast_player"><div id="1186253451" class="castos-player dark-mode " role="region" aria-label="Podcast player" data-episode="64" data-player_id="1186253451">
<div class="player">
<div class="player__main">
<div class="player__artwork player__artwork-64">
<img decoding="async" src="https://downtimeproject.com/files/2021/03/dp_logo-150x150.png"
alt="The Downtime Project"
title="The Downtime Project">
</div>
<div class="player__body">
<div class="currently-playing">
<div class="show player__podcast-title">
The Downtime Project </div>
<div class="episode-title player__episode-title">One Subtle Regex Takes Down Cloudflare</div>
</div>
<div class="play-progress">
<div class="play-pause-controls">
<button title="Play" aria-label="Play Episode" aria-pressed="false" class="play-btn">
<span class="screen-reader-text">Play Episode</span>
</button>
<button title="Pause" aria-label="Pause Episode" aria-pressed="false" class="pause-btn hide">
<span class="screen-reader-text">Pause Episode</span>
</button>
<img decoding="async" src="https://downtimeproject.com/wp-content/plugins/seriously-simple-podcasting/assets/css/images/player/images/icon-loader.svg" alt="Loading" class="ssp-loader hide"/>
</div>
<div>
<audio preload="none" class="clip clip-64">
<source src="https://downtimeproject.com/files/2021/04/DP-004-Mix1.mp3">
</audio>
<div class="ssp-progress" role="progressbar" title="Seek" aria-valuenow="0" aria-valuemin="0" aria-valuemax="3291">
<span class="progress__filled"></span>
</div>
<div class="ssp-playback playback">
<div class="playback__controls">
<button class="player-btn player-btn__volume" title="Mute/Unmute" aria-label="Mute audio" aria-pressed="false">
</button>
<button data-skip="-10" class="player-btn player-btn__rwd" title="Rewind 10 seconds" aria-label="Rewind 10 seconds">
</button>
<button data-speed="1" class="player-btn player-btn__speed" title="Playback Speed" aria-label="Playback speed, currently 1x">1x</button>
<button data-skip="30" class="player-btn player-btn__fwd" title="Fast Forward 30 seconds" aria-label="Fast forward 30 seconds">
</button>
</div>
<div class="playback__timers">
<time class="ssp-timer" aria-live="off">00:00</time>
<span>/</span>
<!-- We need actual duration here from the server -->
<time class="ssp-duration" datetime="PT0H54M51S">54:51</time>
</div>
</div>
</div>
</div>
<nav class="player-panels-nav">
<button class="subscribe-btn" id="subscribe-btn-64" title="Subscribe" aria-expanded="false">Subscribe</button>
<button class="share-btn" id="share-btn-64" title="Share" aria-expanded="false">Share</button>
</nav>
</div>
</div>
</div>
<div class="player-panels player-panels-64">
<div class="subscribe player-panel subscribe-64">
<button class="close-btn close-btn-64" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="panel__inner">
<div class="subscribe-icons">
<a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" rel="noopener noreferrer"
class="apple_podcasts"
title="Subscribe on Apple Podcasts">
<span></span>
Apple Podcasts </a>
<a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" rel="noopener noreferrer"
class="stitcher"
title="Subscribe on Stitcher">
<span></span>
Stitcher </a>
<a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" rel="noopener noreferrer"
class="google_podcasts"
title="Subscribe on Google Podcasts">
<span></span>
Google Podcasts </a>
<a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" rel="noopener noreferrer"
class="spotify"
title="Subscribe on Spotify">
<span></span>
Spotify </a>
</div>
<div class="player-panel-row" role="group" aria-label="RSS Feed URL">
<div class="title">RSS Feed</div>
<div>
<input value="https://downtimeproject.com/feed/podcast/the-downtime-project" class="input-rss input-rss-64" title="RSS Feed URL" readonly />
</div>
<button class="copy-rss copy-rss-64" title="Copy RSS Feed URL" aria-label="Copy RSS Feed URL"></button>
</div>
</div>
</div>
<div class="share share-64 player-panel">
<button class="close-btn close-btn-64" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="player-panel-row">
<div class="title">
Share </div>
<div class="icons-holder">
<a href="https://www.facebook.com/sharer/sharer.php?u=https://downtimeproject.com/podcast/one-subtle-regex-takes-down-cloudflare/&t=One Subtle Regex Takes Down Cloudflare"
target="_blank" rel="noopener noreferrer" class="share-icon facebook" title="Share on Facebook" aria-label="Share on Facebook">
<span></span>
</a>
<a href="https://twitter.com/intent/tweet?text=https://downtimeproject.com/podcast/one-subtle-regex-takes-down-cloudflare/&url=One Subtle Regex Takes Down Cloudflare"
target="_blank" rel="noopener noreferrer" class="share-icon twitter" title="Share on Twitter" aria-label="Share on Twitter">
<span></span>
</a>
<a href="https://downtimeproject.com/files/2021/04/DP-004-Mix1.mp3"
target="_blank" rel="noopener noreferrer" class="share-icon download" title="Download" aria-label="Download" download>
<span></span>
</a>
</div>
</div>
<div class="player-panel-row">
<div class="title">
Link </div>
<div>
<input value="https://downtimeproject.com/podcast/one-subtle-regex-takes-down-cloudflare/" class="input-link input-link-64" title="Episode URL" readonly />
</div>
<button class="copy-link copy-link-64" title="Copy Episode URL" aria-label="Copy Episode URL"></button>
</div>
<div class="player-panel-row">
<div class="title">
Embed </div>
<div>
<textarea rows="1"
title="Embed Code"
class="input-embed input-embed-64" readonly>&lt;blockquote class=&quot;wp-embedded-content&quot; data-secret=&quot;GGvyAuOZqz&quot;&gt;&lt;a href=&quot;https://downtimeproject.com/podcast/one-subtle-regex-takes-down-cloudflare/&quot;&gt;One Subtle Regex Takes Down Cloudflare&lt;/a&gt;&lt;/blockquote&gt;&lt;iframe sandbox=&quot;allow-scripts&quot; security=&quot;restricted&quot; src=&quot;https://downtimeproject.com/podcast/one-subtle-regex-takes-down-cloudflare/embed/#?secret=GGvyAuOZqz&quot; width=&quot;500&quot; height=&quot;350&quot; title=&quot;&amp;#8220;One Subtle Regex Takes Down Cloudflare&amp;#8221; &amp;#8212; The Downtime Project&quot; data-secret=&quot;GGvyAuOZqz&quot; frameborder=&quot;0&quot; marginwidth=&quot;0&quot; marginheight=&quot;0&quot; scrolling=&quot;no&quot; class=&quot;wp-embedded-content&quot;&gt;&lt;/iframe&gt;&lt;script&gt;
/*! This file is auto-generated */
!function(d,l){&quot;use strict&quot;;l.querySelector&amp;&amp;d.addEventListener&amp;&amp;&quot;undefined&quot;!=typeof URL&amp;&amp;(d.wp=d.wp||{},d.wp.receiveEmbedMessage||(d.wp.receiveEmbedMessage=function(e){var t=e.data;if((t||t.secret||t.message||t.value)&amp;&amp;!/[^a-zA-Z0-9]/.test(t.secret)){for(var s,r,n,a=l.querySelectorAll(&#039;iframe[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),o=l.querySelectorAll(&#039;blockquote[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),c=new RegExp(&quot;^https?:$&quot;,&quot;i&quot;),i=0;i&lt;o.length;i++)o[i].style.display=&quot;none&quot;;for(i=0;i&lt;a.length;i++)s=a[i],e.source===s.contentWindow&amp;&amp;(s.removeAttribute(&quot;style&quot;),&quot;height&quot;===t.message?(1e3&lt;(r=parseInt(t.value,10))?r=1e3:~~r&lt;200&amp;&amp;(r=200),s.height=r):&quot;link&quot;===t.message&amp;&amp;(r=new URL(s.getAttribute(&quot;src&quot;)),n=new URL(t.value),c.test(n.protocol))&amp;&amp;n.host===r.host&amp;&amp;l.activeElement===s&amp;&amp;(d.top.location.href=t.value))}},d.addEventListener(&quot;message&quot;,d.wp.receiveEmbedMessage,!1),l.addEventListener(&quot;DOMContentLoaded&quot;,function(){for(var e,t,s=l.querySelectorAll(&quot;iframe.wp-embedded-content&quot;),r=0;r&lt;s.length;r++)(t=(e=s[r]).getAttribute(&quot;data-secret&quot;))||(t=Math.random().toString(36).substring(2,12),e.src+=&quot;#?secret=&quot;+t,e.setAttribute(&quot;data-secret&quot;,t)),e.contentWindow.postMessage({message:&quot;ready&quot;,secret:t},&quot;*&quot;)},!1)))}(window,document);
//# sourceURL=https://downtimeproject.com/wp-includes/js/wp-embed.min.js
&lt;/script&gt;
</textarea>
</div>
<button class="copy-embed copy-embed-64" title="Copy Embed Code" aria-label="Copy Embed Code"></button>
</div>
</div>
</div>
</div>
<div class="podcast_meta"><aside><p><a href="https://downtimeproject.com/podcast-download/64/one-subtle-regex-takes-down-cloudflare?ref=download" title="One Subtle Regex Takes Down Cloudflare " class="podcast-meta-download">Download file</a> | <a href="https://downtimeproject.com/podcast-download/64/one-subtle-regex-takes-down-cloudflare?ref=new_window" target="_blank" title="One Subtle Regex Takes Down Cloudflare " class="podcast-meta-new-window">Play in new window</a> | <span class="podcast-meta-duration">Duration: 54:51</span> | <span class="podcast-meta-date">Recorded on April 8, 2021</span></p><p>Subscribe: <a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" title="Apple Podcasts" class="podcast-meta-itunes">Apple Podcasts</a> | <a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" title="Stitcher" class="podcast-meta-itunes">Stitcher</a> | <a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" title="Google Podcasts" class="podcast-meta-itunes">Google Podcasts</a> | <a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" title="Spotify" class="podcast-meta-itunes">Spotify</a></p></aside></div></div>
<p class="wp-block-paragraph">On July 2, 2019, a subtle issue in a regular expression took down Cloudflare (and with it, a large portion of the internet) for 30 minutes.</p>
</div>
<!-- .entry-content -->
<footer class="entry-meta">
<span>23 Words</span> <span class="sep"> | </span>
<span class="comments-link"><a href="https://downtimeproject.com/podcast/one-subtle-regex-takes-down-cloudflare/#comments">2 Comments</a></span>
</footer>
<!-- .entry-meta -->
</article><!-- #post-64 -->
<article id="post-61" class="post-61 podcast type-podcast status-publish series-the-downtime-project h-entry hentry">
<header class="entry-header">
<h1 class="entry-title p-name">
<a href="https://downtimeproject.com/podcast/monzos-2019-cassandra-outage/" title="Permalink to Monzo&#8217;s 2019 Cassandra Outage" rel="bookmark">Monzo&#8217;s 2019 Cassandra Outage</a>
</h1>
</header>
<!-- .entry-header -->
<div class="entry-content e-content">
<div class="podcast_player"><div id="1364003873" class="castos-player dark-mode " role="region" aria-label="Podcast player" data-episode="61" data-player_id="1364003873">
<div class="player">
<div class="player__main">
<div class="player__artwork player__artwork-61">
<img decoding="async" src="https://downtimeproject.com/files/2021/03/dp_logo-150x150.png"
alt="The Downtime Project"
title="The Downtime Project">
</div>
<div class="player__body">
<div class="currently-playing">
<div class="show player__podcast-title">
The Downtime Project </div>
<div class="episode-title player__episode-title">Monzo's 2019 Cassandra Outage</div>
</div>
<div class="play-progress">
<div class="play-pause-controls">
<button title="Play" aria-label="Play Episode" aria-pressed="false" class="play-btn">
<span class="screen-reader-text">Play Episode</span>
</button>
<button title="Pause" aria-label="Pause Episode" aria-pressed="false" class="pause-btn hide">
<span class="screen-reader-text">Pause Episode</span>
</button>
<img decoding="async" src="https://downtimeproject.com/wp-content/plugins/seriously-simple-podcasting/assets/css/images/player/images/icon-loader.svg" alt="Loading" class="ssp-loader hide"/>
</div>
<div>
<audio preload="none" class="clip clip-61">
<source src="https://downtimeproject.com/files/2021/04/DP-003-Mix1.mp3">
</audio>
<div class="ssp-progress" role="progressbar" title="Seek" aria-valuenow="0" aria-valuemin="0" aria-valuemax="2606">
<span class="progress__filled"></span>
</div>
<div class="ssp-playback playback">
<div class="playback__controls">
<button class="player-btn player-btn__volume" title="Mute/Unmute" aria-label="Mute audio" aria-pressed="false">
</button>
<button data-skip="-10" class="player-btn player-btn__rwd" title="Rewind 10 seconds" aria-label="Rewind 10 seconds">
</button>
<button data-speed="1" class="player-btn player-btn__speed" title="Playback Speed" aria-label="Playback speed, currently 1x">1x</button>
<button data-skip="30" class="player-btn player-btn__fwd" title="Fast Forward 30 seconds" aria-label="Fast forward 30 seconds">
</button>
</div>
<div class="playback__timers">
<time class="ssp-timer" aria-live="off">00:00</time>
<span>/</span>
<!-- We need actual duration here from the server -->
<time class="ssp-duration" datetime="PT0H43M26S">43:26</time>
</div>
</div>
</div>
</div>
<nav class="player-panels-nav">
<button class="subscribe-btn" id="subscribe-btn-61" title="Subscribe" aria-expanded="false">Subscribe</button>
<button class="share-btn" id="share-btn-61" title="Share" aria-expanded="false">Share</button>
</nav>
</div>
</div>
</div>
<div class="player-panels player-panels-61">
<div class="subscribe player-panel subscribe-61">
<button class="close-btn close-btn-61" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="panel__inner">
<div class="subscribe-icons">
<a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" rel="noopener noreferrer"
class="apple_podcasts"
title="Subscribe on Apple Podcasts">
<span></span>
Apple Podcasts </a>
<a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" rel="noopener noreferrer"
class="stitcher"
title="Subscribe on Stitcher">
<span></span>
Stitcher </a>
<a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" rel="noopener noreferrer"
class="google_podcasts"
title="Subscribe on Google Podcasts">
<span></span>
Google Podcasts </a>
<a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" rel="noopener noreferrer"
class="spotify"
title="Subscribe on Spotify">
<span></span>
Spotify </a>
</div>
<div class="player-panel-row" role="group" aria-label="RSS Feed URL">
<div class="title">RSS Feed</div>
<div>
<input value="https://downtimeproject.com/feed/podcast/the-downtime-project" class="input-rss input-rss-61" title="RSS Feed URL" readonly />
</div>
<button class="copy-rss copy-rss-61" title="Copy RSS Feed URL" aria-label="Copy RSS Feed URL"></button>
</div>
</div>
</div>
<div class="share share-61 player-panel">
<button class="close-btn close-btn-61" aria-label="Close">
<span></span>
<span></span>
</button>
<div class="player-panel-row">
<div class="title">
Share </div>
<div class="icons-holder">
<a href="https://www.facebook.com/sharer/sharer.php?u=https://downtimeproject.com/podcast/monzos-2019-cassandra-outage/&t=Monzo&#039;s 2019 Cassandra Outage"
target="_blank" rel="noopener noreferrer" class="share-icon facebook" title="Share on Facebook" aria-label="Share on Facebook">
<span></span>
</a>
<a href="https://twitter.com/intent/tweet?text=https://downtimeproject.com/podcast/monzos-2019-cassandra-outage/&url=Monzo&#039;s 2019 Cassandra Outage"
target="_blank" rel="noopener noreferrer" class="share-icon twitter" title="Share on Twitter" aria-label="Share on Twitter">
<span></span>
</a>
<a href="https://downtimeproject.com/files/2021/04/DP-003-Mix1.mp3"
target="_blank" rel="noopener noreferrer" class="share-icon download" title="Download" aria-label="Download" download>
<span></span>
</a>
</div>
</div>
<div class="player-panel-row">
<div class="title">
Link </div>
<div>
<input value="https://downtimeproject.com/podcast/monzos-2019-cassandra-outage/" class="input-link input-link-61" title="Episode URL" readonly />
</div>
<button class="copy-link copy-link-61" title="Copy Episode URL" aria-label="Copy Episode URL"></button>
</div>
<div class="player-panel-row">
<div class="title">
Embed </div>
<div>
<textarea rows="1"
title="Embed Code"
class="input-embed input-embed-61" readonly>&lt;blockquote class=&quot;wp-embedded-content&quot; data-secret=&quot;AdU7Vi6vuX&quot;&gt;&lt;a href=&quot;https://downtimeproject.com/podcast/monzos-2019-cassandra-outage/&quot;&gt;Monzo&amp;#8217;s 2019 Cassandra Outage&lt;/a&gt;&lt;/blockquote&gt;&lt;iframe sandbox=&quot;allow-scripts&quot; security=&quot;restricted&quot; src=&quot;https://downtimeproject.com/podcast/monzos-2019-cassandra-outage/embed/#?secret=AdU7Vi6vuX&quot; width=&quot;500&quot; height=&quot;350&quot; title=&quot;&amp;#8220;Monzo&amp;#8217;s 2019 Cassandra Outage&amp;#8221; &amp;#8212; The Downtime Project&quot; data-secret=&quot;AdU7Vi6vuX&quot; frameborder=&quot;0&quot; marginwidth=&quot;0&quot; marginheight=&quot;0&quot; scrolling=&quot;no&quot; class=&quot;wp-embedded-content&quot;&gt;&lt;/iframe&gt;&lt;script&gt;
/*! This file is auto-generated */
!function(d,l){&quot;use strict&quot;;l.querySelector&amp;&amp;d.addEventListener&amp;&amp;&quot;undefined&quot;!=typeof URL&amp;&amp;(d.wp=d.wp||{},d.wp.receiveEmbedMessage||(d.wp.receiveEmbedMessage=function(e){var t=e.data;if((t||t.secret||t.message||t.value)&amp;&amp;!/[^a-zA-Z0-9]/.test(t.secret)){for(var s,r,n,a=l.querySelectorAll(&#039;iframe[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),o=l.querySelectorAll(&#039;blockquote[data-secret=&quot;&#039;+t.secret+&#039;&quot;]&#039;),c=new RegExp(&quot;^https?:$&quot;,&quot;i&quot;),i=0;i&lt;o.length;i++)o[i].style.display=&quot;none&quot;;for(i=0;i&lt;a.length;i++)s=a[i],e.source===s.contentWindow&amp;&amp;(s.removeAttribute(&quot;style&quot;),&quot;height&quot;===t.message?(1e3&lt;(r=parseInt(t.value,10))?r=1e3:~~r&lt;200&amp;&amp;(r=200),s.height=r):&quot;link&quot;===t.message&amp;&amp;(r=new URL(s.getAttribute(&quot;src&quot;)),n=new URL(t.value),c.test(n.protocol))&amp;&amp;n.host===r.host&amp;&amp;l.activeElement===s&amp;&amp;(d.top.location.href=t.value))}},d.addEventListener(&quot;message&quot;,d.wp.receiveEmbedMessage,!1),l.addEventListener(&quot;DOMContentLoaded&quot;,function(){for(var e,t,s=l.querySelectorAll(&quot;iframe.wp-embedded-content&quot;),r=0;r&lt;s.length;r++)(t=(e=s[r]).getAttribute(&quot;data-secret&quot;))||(t=Math.random().toString(36).substring(2,12),e.src+=&quot;#?secret=&quot;+t,e.setAttribute(&quot;data-secret&quot;,t)),e.contentWindow.postMessage({message:&quot;ready&quot;,secret:t},&quot;*&quot;)},!1)))}(window,document);
//# sourceURL=https://downtimeproject.com/wp-includes/js/wp-embed.min.js
&lt;/script&gt;
</textarea>
</div>
<button class="copy-embed copy-embed-61" title="Copy Embed Code" aria-label="Copy Embed Code"></button>
</div>
</div>
</div>
</div>
<div class="podcast_meta"><aside><p><a href="https://downtimeproject.com/podcast-download/61/monzos-2019-cassandra-outage?ref=download" title="Monzo&#8217;s 2019 Cassandra Outage " class="podcast-meta-download">Download file</a> | <a href="https://downtimeproject.com/podcast-download/61/monzos-2019-cassandra-outage?ref=new_window" target="_blank" title="Monzo&#8217;s 2019 Cassandra Outage " class="podcast-meta-new-window">Play in new window</a> | <span class="podcast-meta-duration">Duration: 43:26</span> | <span class="podcast-meta-date">Recorded on April 1, 2021</span></p><p>Subscribe: <a href="https://podcasts.apple.com/us/podcast/the-downtime-project/id1559384383" target="_blank" title="Apple Podcasts" class="podcast-meta-itunes">Apple Podcasts</a> | <a href="https://www.stitcher.com/show/the-downtime-project" target="_blank" title="Stitcher" class="podcast-meta-itunes">Stitcher</a> | <a href="https://podcasts.google.com/feed/aHR0cHM6Ly9kb3dudGltZXByb2plY3QuY29tL2ZlZWQvcG9kY2FzdA" target="_blank" title="Google Podcasts" class="podcast-meta-itunes">Google Podcasts</a> | <a href="https://open.spotify.com/show/0VtH0Kjc6TpbDKxBqTgGAT" target="_blank" title="Spotify" class="podcast-meta-itunes">Spotify</a></p></aside></div></div>
<p class="wp-block-paragraph">Monzo experienced <a href="https://monzo.com/blog/2019/09/08/why-monzo-wasnt-working-on-july-29th">some issues</a> while adding servers to their Cassandra cluster on July 29th, 2019. Thanks to some good practices, the team recovered quickly and no data was permanently lost.</p>
</div>
<!-- .entry-content -->
<footer class="entry-meta">
<span>29 Words</span> <span class="sep"> | </span>
<span class="comments-link"><a href="https://downtimeproject.com/podcast/monzos-2019-cassandra-outage/#respond">Comment</a></span>
</footer>
<!-- .entry-meta -->
</article><!-- #post-61 -->
<nav role="navigation" id="nav-below" class="site-navigation paging-navigation">
<h1 class="screen-reader-text">Post navigation</h1>
<div class="nav-previous"><a href="https://downtimeproject.com/page/2/" ><button><span class="meta-nav">&larr;</span> Older posts</button></a></div>
</nav><!-- #nav-below -->
</main>
<!-- #content .site-content -->
</div><!-- #primary .content-area -->
<div id="secondary" class="widget-area" role="complementary">
<aside id="search" class="widget widget_search">
<form method="get" id="searchform" action="https://downtimeproject.com/" role="search">
<label for="s" class="screen-reader-text">Search</label>
<input type="text" class="field" name="s" value="" id="s" placeholder="Search &hellip;" />
<input type="submit" class="submit" name="submit" id="searchsubmit" value="Search" />
</form>
</aside>
<aside id="archives" class="widget">
<h1 class="widget-title">Archives</h1>
<ul>
</ul>
</aside>
<aside id="meta" class="widget">
<h1 class="widget-title">Meta</h1>
<ul>
<li><a href="https://downtimeproject.com/wp-login.php">Log in</a></li>
</ul>
</aside>
</div><!-- #secondary .widget-area -->
</div><!-- #main .site-main -->
<footer id="colophon" class="site-footer" itemscope="itemscope" itemtype="http://schema.org/WPFooter" role="contentinfo">
<div class="site-info">
<a href="http://independentpublisher.me" rel="designer" title="Independent Publisher: A beautiful reader-focused WordPress theme, for you.">Independent Publisher</a> empowered by <a href="http://wordpress.org/" rel="generator" title="WordPress: A free open-source publishing platform">WordPress</a> </div>
<!-- .site-info -->
</footer><!-- #colophon .site-footer -->
</div><!-- #page .hfeed .site -->
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/files/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/independent-publisher/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<script id="independent-publisher-skip-link-focus-fix-js" src="https://downtimeproject.com/wp-content/themes/independent-publisher/js/skip-link-focus-fix.js?ver=20130115"></script>
<script id="wp-statistics-tracker-js-extra">
var WP_Statistics_Tracker_Object = {"requestUrl":"https://downtimeproject.com/wp-json/wp-statistics/v2","ajaxUrl":"https://downtimeproject.com/wp-admin/admin-ajax.php","hitParams":{"wp_statistics_hit":1,"source_type":"home","source_id":0,"search_query":"","signature":"03468a8b3f03a25b3a2f6a82969d7cfd","endpoint":"hit"},"option":{"dntEnabled":false,"bypassAdBlockers":false,"consentIntegration":{"name":null,"status":[]},"isPreview":false,"userOnline":false,"isWpConsentApiActive":false},"isLegacyEventLoaded":"","customEventAjaxUrl":"https://downtimeproject.com/wp-admin/admin-ajax.php?action=wp_statistics_custom_event&nonce=9616923636","onlineParams":{"wp_statistics_hit":1,"source_type":"home","source_id":0,"search_query":"","signature":"03468a8b3f03a25b3a2f6a82969d7cfd","action":"wp_statistics_online_check"},"jsCheckTime":"60000"};
//# sourceURL=wp-statistics-tracker-js-extra
</script>
<script id="wp-statistics-tracker-js" src="https://downtimeproject.com/wp-content/plugins/wp-statistics/assets/js/tracker.js?ver=14.16.13"></script>
<script id="ssp-castos-player-js-extra">
var ssp_player_i18n = {"playback_speed":"Playback speed, currently %sx","copied":"Copied"};
//# sourceURL=ssp-castos-player-js-extra
</script>
<script id="ssp-castos-player-js" src="https://downtimeproject.com/wp-content/plugins/seriously-simple-podcasting/assets/js/castos-player.min.js?ver=3.17.0"></script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://downtimeproject.com/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=https://downtimeproject.com/wp-includes/js/wp-emoji-loader.min.js
</script>
</body>
</html>