Files
nexus/sreweekly/articles/208/01-anatomy-of-cascading-failure.html
2026-09-12 17:23:01 +08:00

3138 lines
208 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en" xmlns="http://www.w3.org/1999/xhtml">
<head>
<!-- Google Optimize Anti-flicker snippet (recommended) -->
<style>.async-hide { opacity: 0 !important} </style>
<style>.promo{background:#222528;position:fixed;z-index:1001!important;left:0;top:0;right:0;max-height:48px;min-height:48px;padding-top:0!important;padding-bottom:0!important}.promo,.promo p{-webkit-box-align:center;-ms-flex-align:center;align-items:center}.promo p{font-size:.8125rem;line-height:1rem;color:#fff;margin-bottom:0;margin-top:0;margin:0 auto;display:-webkit-box;display:-ms-flexbox;display:flex;font-weight:700}@media only screen and (max-width:650px){.promo p{font-size:.6875rem}}.promo span{overflow:hidden;display:-webkit-box;-webkit-line-clamp:3;-webkit-box-orient:vertical}.promo a{color:#fff!important;text-decoration:underline!important}.promo a.btn{background:#d0021b;padding:7px 20px;text-decoration:none!important;font-weight:700;margin-left:10px;margin-right:10px;white-space:nowrap;border-radius:5px}@media only screen and (max-width:650px){.promo a.btn{font-size:.6875rem;padding:7px 10px}}.promo.container{padding-top:8px;padding-bottom:8px}@media only screen and (min-width:1050px){.promo.container{padding-top:0;padding-bottom:0}}.promo .actions{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.promo .actions__left{-ms-flex-preferred-size:100%;flex-basis:100%;-webkit-box-align:center;-ms-flex-align:center;align-items:center}.promo .actions__right{-ms-flex-preferred-size:40px;flex-basis:40px;margin-top:0;margin-bottom:0;-webkit-box-pack:end;-ms-flex-pack:end;justify-content:flex-end}.promo.hidden{display:none}.promo.show{display:-webkit-box!important;display:-ms-flexbox!important;display:flex!important}.promo.show .container__inner{-webkit-box-flex:1;-ms-flex:1;flex:1}.promo.fixed{position:fixed}.promo.show+header.header{margin-top:48px}.header{background:#fff;-webkit-box-shadow:0 1px 0 #dde2e5;box-shadow:0 1px 0 #dde2e5}.header .actions__left,.header__bottom__events{max-width:100%!important;margin:0}.header .header__events-all{margin:0;display:-webkit-box;display:-ms-flexbox;display:flex;position:relative}.header .header__events-all .header__event-slot{-webkit-box-flex:1;-ms-flex:1 100%;flex:1 100%;border-right:1px solid #dde2e5;margin-top:3px;margin-bottom:3px;padding-left:15px;text-align:left;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center;padding-right:5px;min-width:300px}.header .header__events-all .header__event-slot:hover{text-decoration:none!important}.header .header__events-all .header__event-slot img{min-width:40px;height:40px}.header .header__events-all .header__event-slot div{margin-left:10px}.header .header__events-all .header__event-slot span{font-weight:700!important;font-size:.75rem;margin-bottom:0!important;margin-top:0;display:block;line-height:1.125rem;text-align:left}.header .header__events-all .header__event-slot p{font-weight:400;font-size:.625rem;line-height:130%!important;color:#495057!important;margin:0}@media only screen and (min-width:1050px){.header .header__events-all .header__event-slot p{font-size:.625rem}}.header .header__events-all .header__event-slot:first-child{padding-left:0}.header .header__events-all .header__event-slot:last-child{padding-right:0;border-right:0}@media only screen and (min-width:1050px){.header__bottom,.header__middle,.header__top{position:relative;white-space:nowrap}}.header__top{padding:10px 0}@media only screen and (min-width:800px){.header__top{padding:5px 0}}.header__bottom{padding:5px 0;z-index:29;max-width:100%}.header__bottom a{font-size:.8125rem}.header__bottom .trending{margin-left:0;margin-right:0}.header__bottom .actions{-webkit-box-align:start;-ms-flex-align:start;align-items:flex-start}.header__bottom .actions__left{max-width:calc(100% + 8px)}@media only screen and (min-width:800px){.header__bottom .actions__left{max-width:calc(100% + 24px)}}.header__bottom .actions__right{-webkit-box-flex:1;-ms-flex:1 1 auto;flex:1 1 auto;display:none}@media only screen and (min-width:1050px){.header__bottom .actions__right{display:-webkit-box;display:-ms-flexbox;display:flex}}.header__middle{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-flow:row wrap;flex-flow:row wrap}@media only screen and (min-width:1050px){.header__middle{-ms-flex-wrap:nowrap;flex-wrap:nowrap}}.header__top .actions__left{-webkit-box-align:center;-ms-flex-align:center;align-items:center;-ms-flex-line-pack:center;align-content:center}@media only screen and (min-width:1050px){.header__top .actions__right{max-width:430px}}.no-style.header__nav li:nth-child(3){font-weight:700}.no-style.header__nav li:nth-child(3) a{color:#0e5ef1!important}.header__bottom__events::after{background:-webkit-gradient(linear,left top,right top,color-stop(0,rgba(255,255,255,0)),to(#fff));background:linear-gradient(90deg,rgba(255,255,255,0) 0,#fff 100%);content:'';position:absolute;height:60px;right:10px;width:25px}@media only screen and (min-width:1050px){.header__bottom__events::after{display:none}}.contribute-link{font-weight:400;font-size:.6875rem;color:#000!important;position:relative;padding-left:10px}.contribute-link:hover{color:#00791d!important;text-decoration:none!important}.contribute-link::before{content:'';width:1px;height:12px;position:absolute;top:50%;-webkit-transform:translateY(-50%);transform:translateY(-50%);background:rgba(0,0,0,.1);left:-1px}.my-0{margin-top:0!important;margin-bottom:0!important}.header__desc.my-0{margin-left:0}.header__bottom__events .actions__left{max-width:100%!important;overflow-x:scroll;-ms-overflow-style:none;scrollbar-width:none;display:block;scroll-behavior:smooth;min-width:100%}.header__bottom__events .actions__left::-webkit-scrollbar{display:none}.header__bottom__events .actions__left:-webkit-scrollbar-thumb{background:#fff}.logo{line-height:1rem}.header{position:relative;z-index:41;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.header .input:focus,.header input[type=password]:focus,.header input[type=text]:focus{border:1px solid #00791d}.header a:not(.button):not(.login__action):not(.active){text-decoration:none;color:#222}.header a:not(.button):not(.login__action):not(.active):hover{text-decoration:underline;color:#222}.header__items{display:none;-ms-flex-wrap:wrap;flex-wrap:wrap}@media only screen and (min-width:1050px){.header__items{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row}.header__items nav{position:relative;background:0 0;padding:0;left:0;top:0;line-height:inherit;display:block;-webkit-box-shadow:none;box-shadow:none;max-width:100%;max-height:80px}}.header__items>div{width:100%;margin-bottom:32px;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-flow:column nowrap;flex-flow:column nowrap}@media only screen and (min-width:552px){.header__items>div:not(:nth-last-child(-n+2)){margin-bottom:32px}}@media only screen and (min-width:552px) and (max-width:1050px){.header__items>div{-webkit-box-flex:0;-ms-flex-positive:0;flex-grow:0;-ms-flex-negative:0;flex-shrink:0;-ms-flex-preferred-size:calc(99.7% * 1/2 - (32px - 32px * 1/2));flex-basis:calc(99.7% * 1/2 - (32px - 32px * 1/2));max-width:calc(99.7% * 1/2 - (32px - 32px * 1/2));width:calc(99.7% * 1/2 - (32px - 32px * 1/2))}.header__items>div:nth-child(1n){margin-right:32px;margin-left:0}.header__items>div:last-child{margin-right:0}.header__items>div:nth-child(2n){margin-right:0;margin-left:auto}}@media only screen and (min-width:800px) and (max-width:1050px){.header__items>div{-webkit-box-flex:0;-ms-flex-positive:0;flex-grow:0;-ms-flex-negative:0;flex-shrink:0;-ms-flex-preferred-size:calc(99.7% * 1/4 - (32px - 32px * 1/4));flex-basis:calc(99.7% * 1/4 - (32px - 32px * 1/4));max-width:calc(99.7% * 1/4 - (32px - 32px * 1/4));width:calc(99.7% * 1/4 - (32px - 32px * 1/4))}.header__items>div:nth-child(1n){margin-right:32px;margin-left:0}.header__items>div:last-child{margin-right:0}.header__items>div:nth-child(4n){margin-right:0;margin-left:auto}}@media only screen and (min-width:1050px){.header__items>div{margin-bottom:0!important;margin-right:0!important;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row}}.header__items .language__switcher{display:none}.header__items .language__switcher .li-nav.active>a,.header__items .language__switcher>li.active>a{color:#fff!important}.header__items .language__switcher .li-nav.active>a:hover,.header__items .language__switcher>li.active>a:hover{color:#fff!important;background:#0e5ef1}.header__items .language__switcher .li-nav:hover>a,.header__items .language__switcher>li:hover>a{color:#fff}.header+main{display:block;min-height:210px;-webkit-transition:margin .15s ease;transition:margin .15s ease;margin-top:0!important}.header--hide .header+main{margin-top:50px}.header--hide .header__toggle{opacity:0;top:20px;visibility:hidden}.header--hide .header__logo{max-height:0}.header--hide .header__middle,.header--hide .header__top{max-height:0;overflow:hidden;padding-top:0;padding-bottom:0;border-color:transparent}.header--hide .header__bottom .vue-portal-target{top:3px;right:0;bottom:auto;left:auto;position:absolute}.header__middle,.header__top{-webkit-transition:all .15s ease;transition:all .15s ease}.header__middle{border-bottom:1px solid rgba(0,0,0,.1);z-index:33}.header__middle .vue-portal-target{width:100%}@media only screen and (min-width:1050px){.header__middle .vue-portal-target{display:none}}@media only screen and (min-width:1050px){.header__middle{line-height:5.75rem;text-align:left;padding:0;z-index:30}.header__middle .widget__heading{display:none}}.header__top>.actions__left{display:none}@media only screen and (min-width:1050px){.header__top>.actions__left{display:-webkit-box;display:-ms-flexbox;display:flex}}.header__top>.actions__right{-webkit-box-flex:1;-ms-flex-positive:1;flex-grow:1;margin-right:0;margin-top:0;margin-bottom:0;-webkit-box-pack:end;-ms-flex-pack:end;justify-content:flex-end}.header__top>.actions__right .dropdown__holder{width:calc(100vw - 16px);max-height:80vh}@media only screen and (min-width:600px){.header__top>.actions__right .dropdown__holder{width:auto}}.header__top>.actions__right>*{margin:0}.header__top>.actions__right .search{display:none}@media only screen and (min-width:1050px){.header__top>.actions__right>*{display:-webkit-box;display:-ms-flexbox;display:flex;white-space:nowrap}.header__top>.actions__right .search{display:block}}.header__top .user__login{display:block}.header__top .user__login>.button,.header__top .user__login>button{border-top-right-radius:0;border-bottom-right-radius:0}@media only screen and (min-width:1050px){.header__top{position:relative;right:auto;width:100%}}.header__logo{max-width:165px;position:absolute;top:8px;overflow:hidden;-webkit-transition:all .1s ease;transition:all .1s ease;z-index:32;line-height:2.25rem;height:36px;width:100px;margin-left:50px;-ms-flex-preferred-size:190px;flex-basis:190px}@media only screen and (min-width:800px){.header__logo{top:4px}}@media only screen and (min-width:1050px){.header__logo{position:relative;top:0;overflow:visible;margin-right:20px;margin-left:0;line-height:3.125rem;height:50px}.header__logo>*{width:165px}}@media only screen and (min-width:1280px){.header__logo{margin-right:30px}}.header__desc,.header__more>button{text-transform:capitalize;color:#666;letter-spacing:0;font-size:0;font-weight:400;line-height:1.5rem;vertical-align:top;font-smoothing:antialiased;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI Variable","Segoe UI",system-ui,ui-sans-serif,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji";-webkit-transition:font-size .15s ease-in-out;transition:font-size .15s ease-in-out}@media only screen and (min-width:1050px){.header__desc,.header__more>button{font-size:.6875rem}}.header__desc{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;position:relative;width:auto}.header__topics{white-space:nowrap;float:right;position:relative}.header__topics *{display:inline-block;vertical-align:top}.header__topics a{font-size:.8125rem}@media only screen and (min-width:1050px){.header__topics{float:none}}.header__more{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;vertical-align:top}.header__more:before{left:-12px}.header__more:after,.header__more:before{content:'';width:1px;height:12px;position:absolute;top:50%;-webkit-transform:translateY(-50%);transform:translateY(-50%);background:rgba(0,0,0,.1)}.header__more:after{right:-12px}.header__more:hover{cursor:pointer;color:rgba(0,0,0,.75)}.header__user{display:inline-block;vertical-align:top;white-space:nowrap;margin-left:8px}@media only screen and (min-width:1050px){.header__user{margin-left:0}}.header__user>div{display:inline-block;vertical-align:top}.header__user-nav a:not(.button){text-transform:uppercase;font-size:.75rem;font-weight:600}.header__user-nav a:not(.button):not(.active){opacity:.5}.header__user-nav a:not(.button):not(.active):hover{opacity:.75}.header__user-nav a:not(.button):hover{text-decoration:none}.header__user-nav a:not(.button).active{color:#222;cursor:default;text-decoration:none}.header__user-nav a:not(.button):before{margin-right:0}.header__user-nav a:not(.button):not(:last-child){margin-right:16px}@media only screen and (min-width:1050px){.header__user-nav a:not(.button):not(:last-child){margin-right:32px}}@media only screen and (min-width:1050px){.header__user-nav+.header__topics{margin-left:56px}}.header__search{display:none;vertical-align:top;margin-right:0}@media only screen and (min-width:1050px){.header__search{display:inline-block}}.header__search,.header__user{line-height:inherit}.header__nav{border-bottom:1px solid rgba(0,0,0,.1)}@media only screen and (min-width:800px){.header__nav{border-bottom:0}}.header__nav .button{margin-left:0}.header__nav .button__more{margin-right:20px}.header__nav .li-nav,.header__nav>li{-webkit-transition:all .15s ease;transition:all .15s ease}@media only screen and (min-width:1050px){.header__nav .li-nav,.header__nav>li{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;vertical-align:middle;position:static;border:none;min-height:65px}.header__nav .li-nav:hover.has--subnav .nav .li-nav,.header__nav .li-nav:hover.has--subnav .nav>li,.header__nav>li:hover.has--subnav .nav .li-nav,.header__nav>li:hover.has--subnav .nav>li{white-space:normal}.header__nav .li-nav:hover.has--subnav .nav__subnav,.header__nav>li:hover.has--subnav .nav__subnav{opacity:1;top:100%;visibility:visible;-webkit-transition-delay:.25s;transition-delay:.25s}.header__nav .li-nav:hover.has--subnav .nav__category,.header__nav>li:hover.has--subnav .nav__category{width:100%}.header__nav .li-nav:hover.has--subnav .nav__category:after,.header__nav .li-nav:hover.has--subnav .nav__category:before,.header__nav>li:hover.has--subnav .nav__category:after,.header__nav>li:hover.has--subnav .nav__category:before{-webkit-transition-delay:.25s;transition-delay:.25s;opacity:1}}@media only screen and (min-width:1080px){.header__nav .li-nav:not(:last-child),.header__nav>li:not(:last-child){margin-right:23px;margin-left:23px}}@media only screen and (min-width:1280px){.header__nav .li-nav:not(:last-child),.header__nav>li:not(:last-child){margin-right:23px;margin-left:23px}}@media only screen and (min-width:1338px){.header__nav .li-nav:not(:last-child),.header__nav>li:not(:last-child){margin-right:23px;margin-left:23px}}.header__nav .li-nav>a,.header__nav>li>a{font-size:.875rem;line-height:1.5rem;padding:12px 0;color:#000;display:inline-block;max-width:100%;position:relative;z-index:61;white-space:normal}@media only screen and (min-width:1050px){.header__nav .li-nav>a,.header__nav>li>a{padding:8px 0 0;font-weight:700}}.header__nav .li-nav>a:after,.header__nav .li-nav>a:before,.header__nav>li>a:after,.header__nav>li>a:before{content:'';position:absolute;bottom:-1px;left:50%;width:0;height:0;opacity:0;-webkit-transform:translateX(-50%);transform:translateX(-50%);border-style:solid;border-width:0 5px 5px 5px;-webkit-transition:opacity .15s ease-in-out;transition:opacity .15s ease-in-out;border-color:transparent transparent #fff transparent}.header__nav .li-nav>a:before,.header__nav>li>a:before{left:50%;bottom:0;border-width:0 6px 6px 6px;border-color:transparent transparent #f5f7f8 transparent}@media only screen and (min-width:1050px){.header__nav .li-nav>a,.header__nav>li>a{font-size:1rem}}@media only screen and (min-width:1800px){.header__nav .li-nav>a,.header__nav>li>a{font-size:1.125rem}}.header--open{overflow:hidden}@media only screen and (min-width:600px){.header--open{overflow:visible}}.header--open .content-items{max-height:215px;margin:12px 0 24px}.header--open .search{margin-top:16px;display:block}.header--open .header__toggle:before{z-index:10}.header--open .header__toggle>span:nth-child(1){top:50%;-webkit-transform:rotate(45deg);transform:rotate(45deg)}.header--open .header__toggle>span:nth-child(2){opacity:0}.header--open .header__toggle>span:nth-child(3){top:50%;-webkit-transform:rotate(-45deg);transform:rotate(-45deg)}.header--open .header__items,.header--open .header__items .language__switcher{display:-webkit-box;display:-ms-flexbox;display:flex}.header--open .header__top{z-index:100;position:absolute;left:0;right:8px;background:#fff}@media only screen and (min-width:800px){.header--open .header__top{right:20px}}.header--open .header__top .search{display:none}.header--open .header__logo{z-index:101}.header--open .header__middle{padding-top:60px}.header--open .header__bottom{display:none}.header--open .header__container{max-height:90vh;overflow-x:hidden;overflow-y:auto}@media only screen and (min-width:1050px){.header--open .header__container{overflow:visible}}.header .subnav{position:absolute;-webkit-box-shadow:0 5px 25px 1px rgba(0,0,0,.15);box-shadow:0 5px 25px 1px rgba(0,0,0,.15);background:#fff;visibility:hidden;line-height:1.75rem;max-width:100%;width:100%;left:0;opacity:0;overflow:hidden;border:1px solid #f5f7f8;border-radius:2px;-webkit-transition:all .15s ease-in-out;transition:all .15s ease-in-out;top:105%;z-index:60;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-flow:row nowrap;flex-flow:row nowrap;-webkit-box-align:stretch;-ms-flex-align:stretch;align-items:stretch}.header .subnav .subnav__categories{-webkit-box-flex:0;-ms-flex:0 1 280px;flex:0 1 280px;padding:24px 0;background:#f5f7f8;margin-right:0!important}.header .subnav .subnav__categories>li{display:block;font-size:.9375rem;padding:2px 48px 2px 24px}.header .subnav .subnav__categories>li a{display:block;font-weight:700}.header .subnav .subnav__categories>li:hover{background:#e1e1e1}.header .subnav .subnav__heading{margin-bottom:20px}.header .subnav .subnav__content{position:relative;-ms-flex-item-align:start;align-self:flex-start;padding:24px;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-flow:row wrap;flex-flow:row wrap}@media only screen and (min-width:1050px){.header .subnav .subnav__content{padding-left:32px;padding-right:32px;margin:0 auto;-webkit-box-flex:1;-ms-flex:1 1 600px;flex:1 1 600px}}.header .subnav .subnav__content .heading__container{-ms-flex-preferred-size:100%;flex-basis:100%}.header .subnav .subnav__content:before{content:'';position:absolute;left:0;top:0;bottom:-100%;width:1px;background:rgba(0,0,0,.1)}.languagesEdition .active{font-weight:700}.header__more.dropdown:after{content:'';display:inline-block;vertical-align:middle;-ms-flex-item-align:center;-ms-grid-row-align:center;align-self:center;background-repeat:no-repeat;background-position:center;margin-bottom:2px;width:7px;height:7px;-webkit-transition:-webkit-transform .15s ease;transition:-webkit-transform .15s ease;transition:transform .15s ease;transition:transform .15s ease,-webkit-transform .15s ease;background-color:#fff!important;background-size:contain}.header__more.dropdown button{color:#000!important;margin-right:-20px;padding-right:20px;z-index:1000}.header__more.dropdown .dropdown__holder{width:165px}.header__more.dropdown .dropdown__content{padding:13px;padding-top:5px;padding-bottom:5px}.header__more.dropdown .languagesEdition li{border-bottom:1px solid #e6e6e6}.header__more.dropdown .languagesEdition li:last-child{border:0}.logo__data{display:none;color:#666;font-size:.6875rem}@media only screen and (min-width:1050px){.logo__data{display:block;line-height:.8125rem}}.container{padding-left:12px;padding-right:12px;margin:0 auto;min-width:320px;-webkit-transition:padding .15s ease-in-out;transition:padding .15s ease-in-out}@media only screen and (min-width:600px){.container{padding-left:16px;padding-right:16px}}@media only screen and (min-width:800px){.container{padding-left:20px;padding-right:20px}}@media only screen and (min-width:1250px){.container{padding-left:60px;padding-right:60px}}@media only screen and (min-width:1400px){.container{padding-left:20px;padding-right:20px}}.container.white{background:#fff}.container__inner{max-width:1290px;margin:0 auto;-webkit-transition:max-width .15s ease-out;transition:max-width .15s ease-out}.search{display:block;position:relative;z-index:33;width:100%;max-width:100%;margin:0}.search:before{position:absolute;top:50%;right:24px;-webkit-transform:translateY(-50%);transform:translateY(-50%);margin-right:0;z-index:34}.search__bar{display:block;border-radius:2px;position:relative;z-index:33}.search__bar #search{margin-bottom:0;max-width:100%;background:#fff}.search__go{top:50%;right:0;bottom:0;left:auto;position:absolute;-webkit-transform:translateY(-50%);transform:translateY(-50%);z-index:32;-webkit-appearance:none;-moz-appearance:none;appearance:none;width:36px;height:36px;line-height:2.25rem;-webkit-box-shadow:none;box-shadow:none;display:block;background:0 0;border:0;font-size:0}@media only screen and (min-width:600px){.search__go{z-index:35}}.header #search,.search__go:hover{cursor:pointer}.header #search{height:36px;position:relative;max-width:100%;background-color:#f5f7f8!important}@media only screen and (min-width:600px){.header #search{font-size:.8125rem;min-width:165px;max-width:100%;opacity:1}}.header #search:hover{cursor:auto}@media only screen and (min-width:1050px){.header #search{margin-left:auto;border-top-right-radius:0;border-bottom-right-radius:0;border-right:0}.header #search:focus{min-width:215px}}.header #search:focus{cursor:auto}.header #search .field__desc{display:none}.header #searchForm{width:100%;margin-top:8px}.header #searchForm:before{right:8px}@media only screen and (min-width:600px){.header #searchForm{margin-top:16px}}@media only screen and (min-width:1050px){.header #searchForm{margin-top:0}}</style>
<script>(function(a,s,y,n,c,h,i,d,e){s.className+=' '+y;h.start=1*new Date;
h.end=i=function(){s.className=s.className.replace(RegExp(' ?'+y),'')};
(a[n]=a[n]||[]).hide=h;setTimeout(function(){i();h.end=null},c);h.timeout=c;
})(window,document.documentElement,'async-hide','dataLayer',4000,
{'GTM-W9GJ5DL':true});</script>
<script type="text/javascript">
var loggedIn = false;
if (loggedIn) {
var userCountryId = '';
}
</script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
//CookieControl tool recomendation
// Call the default command before gtag.js or Tag Manager runs to
// adjust how the tags operate when they run. Modify the defaults
// per your business requirements and prior consent granted/denied, e.g.:
gtag('consent', 'default', {
'ad_storage': 'denied',
'ad_user_data': 'denied',
'ad_personalization': 'denied',
'analytics_storage': 'denied',
wait_for_update: 500
});
gtag('set', 'ads_data_redaction', true);
gtag('set', 'url_passthrough', true);
if((typeof loggedIn != "undefined") && loggedIn){
window.dataLayer.push({'logged_in': 'true'});
} else {
window.dataLayer.push({'logged_in': 'false'});
}
window.dataLayer.push({'show_queryz': ''});
gtag('config', 'G-VMVPD4D2JY');
</script>
<!-- Google Tag Manager -->
<script>
var gtmProfile="GTM-W9GJ5DL";
(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer', gtmProfile);</script>
<!-- End Google Tag Manager -->
<title>How to Avoid Cascading Failures in Distributed Systems - InfoQ</title>
<link rel="canonical" href="https://www.infoq.com/articles/anatomy-cascading-failure/"/>
<link rel="alternate" href="https://www.infoq.com/articles/anatomy-cascading-failure/" hreflang="en"/>
<link rel="alternate" href="https://www.infoq.com/br/articles/anatomy-cascading-failure/" hreflang="pt" />
<link rel="alternate" href="https://www.infoq.com/articles/anatomy-cascading-failure/" hreflang="x-default" />
<meta http-equiv="pragma" content="no-cache" />
<meta http-equiv="cache-control" content="private,no-cache,no-store,must-revalidate" />
<meta http-equiv="expires" content="0" />
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
<meta name="copyright" content="&copy; 2006 C4Media" />
<meta name="google-site-verification" content="0qInQx_1WYOeIIbxnh7DnXlw1XOxNgAYakO2k4GhNnY" />
<meta name="msapplication-TileColor" content="#ffffff"/>
<meta name="msapplication-TileImage" content="/styles/static/images/logo/logo.jpg"/>
<meta property="wb:webmaster" content="3eac1729a8bbe046" />
<meta content="width=device-width,initial-scale=1" name="viewport">
<meta http-equiv="X-UA-Compatible" content="IE=10, IE=edge">
<link rel="stylesheet" type="text/css" media="screen" href="https://cdn.infoq.com/statics_s2_20260910111657/styles/style.css"/>
<link rel="stylesheet" type="text/css" media="print" href="https://cdn.infoq.com/statics_s2_20260910111657/styles/print.css"/>
<link rel="preload" type="text/css" href="https://cdn.infoq.com/statics_s2_20260910111657/styles/style_en.css" as="style" onload="this.onload=null;this.rel='stylesheet'">
<link rel="preload" href="https://cdn.infoq.com/statics_s2_20260910111657/styles/icons.css" as="style" onload="this.onload=null;this.rel='stylesheet'">
<link rel="preload" type="text/css" media="screen" href="https://cdn.infoq.com/statics_s2_20260910111657/styles/style_extra.css" as="style" onload="this.onload=null;this.rel='stylesheet'">
<noscript>
<link rel="stylesheet" type="text/css" href="https://cdn.infoq.com/statics_s2_20260910111657/styles/style_en.css"/>
<link rel="stylesheet" href="https://cdn.infoq.com/statics_s2_20260910111657/styles/icons.css">
<link rel="stylesheet" type="text/css" media="screen" href="https://cdn.infoq.com/statics_s2_20260910111657/styles/style_extra.css"/>
</noscript>
<link rel="stylesheet" type="text/css" href="https://cdn.infoq.com/statics_s2_20260910111657/styles/fonts/font.css"/>
<link rel="shortcut icon" href="https://cdn.infoq.com/statics_s2_20260910111657/favicon.ico" type="image/x-icon" />
<link rel="apple-touch-icon" href="https://cdn.infoq.com/statics_s2_20260910111657/apple-touch-icon.png"/>
<script type="text/javascript">
var device='desktop';
var InfoQConstants = {};
InfoQConstants.language = 'en';
InfoQConstants.countryCode = '';
InfoQConstants.pageUrl = (typeof window.location != 'undefined' && window.location && typeof window.location.href != 'undefined' && window.location.href) ? window.location.href : "URL_UNAVAILABLE";
InfoQConstants.cet='pgarAk5gUTjmy6ja';
InfoQConstants.userDetectedCountryCode = 'US';
InfoQConstants.bpadb = 'OkdrOhA8BvInkKBonZb7';
</script>
<script type="text/javascript" src="https://cdn.infoq.com/statics_s2_20260910111657/scripts/combinedJs.min.js"></script>
<script type="text/javascript">
var JSi18n = JSi18n || {}; // define only if not already defined
JSi18n.error='Error';
JSi18n.login_unverifiedAccount='Unverified account';
JSi18n.contentSummary_showPresentations_1='';
JSi18n.contentSummary_showPresentations_2='';
JSi18n.contentSummary_showPresentations_3='';
JSi18n.contentSummary_showInterviews_1='';
JSi18n.contentSummary_showInterviews_2='';
JSi18n.contentSummary_showInterviews_3='';
JSi18n.contentSummary_showMinibooks_1='';
JSi18n.contentSummary_showMinibooks_2='';
JSi18n.login_sendingRequest='Sending request ...';
JSi18n.bookmark_saved='<q>&nbsp;&nbsp;&nbsp;Saved&nbsp;&nbsp;&nbsp;&nbsp;</q>';
JSi18n.bookmark_error='<q style=color:black;>&nbsp;&nbsp;&nbsp;Error&nbsp;&nbsp;&nbsp;&nbsp;</q>';
JSi18n.categoryManagement_showpopup_viewAllLink_viewAllPrefix='View All';
JSi18n.categoryManagement_showpopup_viewAllLink_viewAllSuffix='';
JSi18n.categoryManagement_showpopup_includeExcludeLink_Exclude='Exclude';
JSi18n.categoryManagement_showpopup_includeExcludeLink_Include='Include';
JSi18n.login_invalid_email='Please specify a valid email';
JSi18n.login_email_not_found = 'No user found with that email';
JSi18n.content_datetime_format='MMM dd, yyyy';
// used by frontend
JSi18n.FE = {
labels: {
follow: "Follow",
followTopic: "Follow Topic",
unfollow: "Unfollow",
unfollowTopic: "Unfollow Topic",
following: "Following",
followers: "Followers",
like: "Like",
liked: "Liked",
authors: "Peers",
users : "Users",
topics: "Topics",
hide: "Hide Item",
hidden: "%s was hidden on your profile page.",
undo: "Undo",
showLess: "Show less",
showMore: "Show more",
moreAuthors: "And %s more",
bookmarked: "Content Bookmarked",
unbookmarked: "Content Unbookmarked",
characterLimit: "Characters Remaining"
}
}
var usersInPage = JSON.parse('[{\"id\":\"62671158\",\"ref\":\"author-Monica-Beckwith\",\"url\":\"\/profile\/Monica-Beckwith\",\"followedByCurrentUser\":false,\"minibio\":\"\",\"name\":\"Monica Beckwith\",\"bio\":\"Java Champion, Monica Beckwith is a Java performance engineer. She currently works on improving OpenJDK&#39;s HotSpot VM at Microsoft. Her past experiences include working with Arm, Oracle\/Sun and AMD; optimizing the JVM for server class systems. Monica was voted a JavaOne Rock Star speaker and was the performance lead for Garbage First Garbage Collector (G1 GC). You can follow Monica on twitter &#64;mon_beck\",\"followers\":1833,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s2_20260910111657\/images\/profiles\/QhDv7pXEUK1sdLsYArFmnLxecH4rYhAc.jpg\"},{\"id\":\"45723890\",\"ref\":\"author-Rags-Srinivas\",\"url\":\"\/profile\/Rags-Srinivas\",\"followedByCurrentUser\":false,\"minibio\":\"\",\"name\":\"Rags Srinivas\",\"bio\":\"Raghavan &#34;Rags&#34; Srinivas (&#64;ragss) works as an Architect\/Developer Evangelist goaled with helping developers build highly scalable and available systems. As an OpenStack advocate and solutions architect at Rackspace he was constantly challenged from low level infrastructure to high level application issues. His general focus area is in distributed systems, with a specialization in Cloud Computing and Big Data. He worked on Hadoop, HBase and NoSQL during its early stages. He has spoken on a variety of technical topics at conferences around the world, written for developer portals, conducted and organized Hands-on Labs and taught graduate and online classes in the evening. Rags brings with him over 25 years of hands-on software development and over 15 years of architecture and technology evangelism experience. He has evangelized and influenced the architecture of a number of emerging technology areas. He is also a repeat JavaOne rock star speaker award winner. Rags holds a Masters degree in Computer Science from the Center of Advanced Computer Studies at the University of Louisiana at Lafayette. He likes to hike, run and generally be outdoors but most of all loves to eat.\",\"followers\":356,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s2_20260910111657\/images\/profiles\/t4nyfgw1THkp4wMZ9EZ59RGJx8Hf9Rk8.jpg\"},{\"id\":\"87551248\",\"ref\":\"author-Steef~Jan-Wiggers\",\"url\":\"\/profile\/Steef~Jan-Wiggers\",\"followedByCurrentUser\":false,\"minibio\":\"Cloud Queue Lead Editor | Domain Architect | Cloud Expert\",\"name\":\"Steef-Jan Wiggers\",\"bio\":\"Steef-Jan Wiggers is one of InfoQ&#39;s senior cloud editors and works as a Domain Architect at VGZ in the Netherlands. His current technical expertise focuses on implementing integration platforms, Azure DevOps, AI, and Azure Platform Solution Architectures. Steef-Jan is a regular speaker at conferences and user groups and writes for InfoQ. Furthermore, Microsoft has recognized him as a Microsoft Azure MVP for the past sixteen years.\",\"followers\":895,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s2_20260910111657\/images\/profiles\/BhZx03k3Hj0pZVXmTzGqItwZxtJ06oIb.jpg\"},{\"id\":\"80977916\",\"ref\":\"author-Thomas-Betts\",\"url\":\"\/profile\/Thomas-Betts\",\"followedByCurrentUser\":false,\"minibio\":\"Senior Laureate Application Architect at Blackbaud\",\"name\":\"Thomas Betts\",\"bio\":\"Thomas Betts is the former Lead Editor for Architecture and Design at InfoQ, a co-host of the InfoQ Podcast, and a Senior Laureate Software Architect at Blackbaud, working on Agents for Good.\\r\\n\\r\\nFor over two decades, his focus has always been on providing software solutions that delight his customers. He has worked in a variety of industries, including social good, retail, finance, health care, defense and travel.\\r\\n\\r\\nThomas lives in Denver with his wife and son, and they love hiking and otherwise exploring beautiful Colorado.\",\"followers\":1171,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s2_20260910111657\/images\/profiles\/pSqI6HrU3k9rmmVjwS34OHG0bOMYiE6a.jpg\"},{\"id\":\"110207\",\"ref\":\"author-Charles-Humble\",\"url\":\"\/profile\/Charles-Humble\",\"followedByCurrentUser\":false,\"minibio\":\"Techie, podcaster, editor, author and consultant\",\"name\":\"Charles Humble\",\"bio\":\"Charles Humble is a former software engineer, architect and CTO who has worked as a senior leader and executive of both technology and content groups. He was InfoQ\u2019s editor-in-chief from 2014-2020, and was chief editor for Container Solutions from 2020-2023. He writes regularly for The New Stack and other publications, is a highly experienced content strategist, and has spoken at multiple international conferences including GOTO (keynote), Devoxx, WTF is SRE and QCon. His primary areas of interest are how we build software better, including sustainability and ethics, cloud computing, remote working, diversity and inclusion, and inspiring the next generation of developers. Charles is also a keyboard player, and half of the ambient techno band Twofish.\",\"followers\":2018,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s2_20260910111657\/images\/profiles\/lCB_2ocIPJA40tWqjmkSLeVd5ZhJQlt_.jpg\"},{\"id\":\"72028228\",\"ref\":\"author-Sergio-De-Simone\",\"url\":\"\/profile\/Sergio-De-Simone\",\"followedByCurrentUser\":false,\"minibio\":\"\",\"name\":\"Sergio De Simone\",\"bio\":\"<b>Sergio De Simone<\/b> is a software engineer. Sergio has been working as a software engineer for over twenty five years across a range of different projects and companies, including such different work environments as Siemens, HP, and small startups. For the last 10&#43; years, his focus has been on development for mobile platforms and related technologies. He is currently working for BigML, Inc., where he leads iOS and macOS development.\",\"followers\":777,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s2_20260910111657\/images\/profiles\/NovciOoQOAYWqYqRQBFo97SuMm0xbUiC.jpg\"},{\"id\":\"126467140\",\"ref\":\"author-Renato-Losio\",\"url\":\"\/profile\/Renato-Losio\",\"followedByCurrentUser\":false,\"minibio\":\"Cloud Expert | AWS Data Hero \",\"name\":\"Renato Losio\",\"bio\":\"Renato has extensive experience as a cloud architect, advisor, and cloud services specialist. Currently, he lives in Berlin and works remotely as a principal cloud architect. His primary areas of interest include cloud services and relational databases. He is an editor at InfoQ and a recognized AWS Data Hero. You can connect with him on LinkedIn.\",\"followers\":913,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s2_20260910111657\/images\/profiles\/ptroF8HdI2vWXm0NDaKeS0JdiPxMOAra.jpg\"},{\"id\":\"126464202\",\"ref\":\"author-Johan-Janssen\",\"url\":\"\/profile\/Johan-Janssen\",\"followedByCurrentUser\":false,\"minibio\":\"Architect at ASML\",\"name\":\"Johan Janssen\",\"bio\":\"Architect at ASML, loves to share knowledge mainly around Java. Spoke at conferences such as Devoxx, Oracle Code One, Devnexus, and many more. Assisted conferences by participating in program committees and invented and organized JVMCON. Received the JavaOne Rock Star and Oracle Code One Star awards. Wrote various articles both for digital and printed media. Maintainer of various Java JDK\/JRE packages for Chocolatey with around 100 thousand downloads a month.\",\"followers\":536,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s2_20260910111657\/images\/profiles\/Fb4eZ0mtvMf6MhsmBIhsUVueV4xAs2FD.jpg\"},{\"id\":\"39485652\",\"ref\":\"author-Daniel-Bryant\",\"url\":\"\/profile\/Daniel-Bryant\",\"followedByCurrentUser\":false,\"minibio\":\"InfoQ News Manager | Building Platforms at Syntasso\",\"name\":\"Daniel Bryant\",\"bio\":\"Daniel Bryant is the news manager at InfoQ and the emeritus chair of QCon London. He is also a platform engineer and head of product marketing at Syntasso. Daniel&#39;s technical expertise focuses on \u2018DevOps\u2019 tooling, cloud\/container platforms, and microservice implementations. He is a long-time coder and Java Champion who has contributed to several open source projects. Daniel also writes for InfoQ, O\u2019Reilly, and The New Stack and regularly presents at international conferences such as KubeCon, QCon, and JavaOne. In his copious amounts of free time, he enjoys running, reading, and travelling.\",\"followers\":2751,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s2_20260910111657\/images\/profiles\/tSe5dczMaSGtRUm18VkTR2tcF4W3SogA.jpg\"},{\"id\":\"126398076\",\"ref\":\"author-Laura-Nolan\",\"url\":\"\/profile\/Laura-Nolan\",\"followedByCurrentUser\":false,\"minibio\":\"\",\"name\":\"Laura Nolan\",\"bio\":\"\",\"followers\":19,\"imgSrc\":\"\"},{\"id\":\"63268344\",\"ref\":\"author-Chris-Swan\",\"url\":\"\/profile\/Chris-Swan\",\"followedByCurrentUser\":false,\"minibio\":\"Engineer, Atsign\",\"name\":\"Chris Swan\",\"bio\":\"Chris Swan is an Engineer at <a href=\\\"https:\/\/atsign.com\\\" rel=\\\"nofollow\\\">Atsign<\/a>, building the atPlatform, a technology that is putting people in control of their data and removing the frictions and surveillance associated with today\u2019s Internet. He was previously a Fellow at DXC Technology where he held various CTO roles. Before that he held CTO and Director of R&amp;D roles at Cohesive Networks, UBS, Capital SCF and Credit Suisse, where he worked on app servers, compute grids, security, mobile, cloud, networking and containers. Chris co-hosts the <a href=\\\"https:\/\/techdebtburndown.com\/\\\" rel=\\\"nofollow\\\">Tech Debt Burndown Podcast<\/a> and is a Dart Google Developer Expert (<a href=\\\"https:\/\/developers.google.com\/community\/experts\\\" rel=\\\"nofollow\\\">GDE<\/a>).\",\"followers\":1873,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s2_20260910111657\/images\/profiles\/8PE76nOujWAoCM8yqLn9Hfv2HqW3VlIP.jpg\"},{\"id\":\"4927376\",\"ref\":\"author-Karsten-Silz\",\"url\":\"\/profile\/Karsten-Silz\",\"followedByCurrentUser\":false,\"minibio\":\"Full-Stack Java Developer\",\"name\":\"Karsten Silz\",\"bio\":\"Karsten Silz has worked as a full-stack Java developer for 26 years in Europe and the US. In 2004, he co-founded a software product start-up in the US. Karsten led product development for 13 years and left after the company was sold successfully. Since 2003, he has also worked as a contractor. He co-founded the SaaS start-up &#34;Your Home in Good Hands&#34; as CTO in the UK in 2020.\",\"followers\":399,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s2_20260910111657\/images\/profiles\/p6zmOdOcqXiRj09iiZNeDZap7f0IglQW.jpg\"},{\"id\":\"343314\",\"ref\":\"author-Jonathan-Allen\",\"url\":\"\/profile\/Jonathan-Allen\",\"followedByCurrentUser\":false,\"minibio\":\"Software Architect\",\"name\":\"Jonathan Allen\",\"bio\":\"Jonathan Allen got his start working on MIS projects for a health clinic in the late 90&#39;s, bringing them up from Access and Excel to an enterprise solution by degrees. After spending five years writing automated trading systems for the financial sector, he became a consultant on a variety of projects including the UI for a robotic warehouse, the middle tier for cancer research software, and the big data needs of a major real estate insurance company. In his free time he enjoys studying and writing about martial arts from the 16th century.\\r\\n\",\"followers\":1833,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s2_20260910111657\/images\/profiles\/Wk_C09_mzwK23YkTkKMXResJv3LKUN5D.jpg\"},{\"id\":\"114725059\",\"ref\":\"author-Holly-Cummins\",\"url\":\"\/profile\/Holly-Cummins\",\"followedByCurrentUser\":false,\"minibio\":\"Senior Technical Staff Member, IBM\",\"name\":\"Holly Cummins\",\"bio\":\"Holly Cummins is a Senior Principal Software Engineer on the Red Hat Quarkus team. Before joining Red Hat, Holly was a long time IBMer. In her time at IBM, Holly has been a full-stack javascript developer, a WebSphere Liberty build architect, a client-facing consultant, a JVM performance engineer, and an innovation leader. During her time in the IBM Garage, Holly led projects for enormous banks, tiny startups, and everything in between. Holly has used the power of cloud to understand climate risks, count fish, help a blind athlete run ultra-marathons in the desert solo, and invent stories (although not at all the same time). Holly is also a Java Champion, author, and regular keynote speaker. You can follow her on bluesky at &#64;&#64;hollycummins.com or at hollycummins.com.\\r\\n\\r\\n\\r\\nBefore joining IBM, Holly completed a PhD in Quantum Computation.\\r\\n\",\"followers\":615,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s2_20260910111657\/images\/profiles\/cRsuGlFgKyGmGfEHvafpMO63CxbrEm22.jpg\"}]');
var topicsInPage = JSON.parse('[{\"name\":\"Architecture & Design\",\"id\":\"6816\",\"url\":\"\/architecture-design\",\"followers\":10383,\"followedByCurrentUser\":false},{\"name\":\"Culture & Methods\",\"id\":\"6817\",\"url\":\"\/culture-methods\",\"followers\":3994,\"followedByCurrentUser\":false},{\"name\":\".NET Core\",\"id\":\"15683\",\"url\":\"\/Net-Core\",\"followers\":8030,\"followedByCurrentUser\":false},{\"name\":\"Machine Learning\",\"id\":\"5449\",\"url\":\"\/MachineLearning\",\"followers\":14751,\"followedByCurrentUser\":false},{\"name\":\"Microservices\",\"id\":\"15274\",\"url\":\"\/microservices\",\"followers\":22348,\"followedByCurrentUser\":false},{\"name\":\"Architecture\",\"id\":\"1389\",\"url\":\"\/architecture\",\"followers\":3366,\"followedByCurrentUser\":false},{\"name\":\"Patterns\",\"id\":\"401\",\"url\":\"\/patterns\",\"followers\":121,\"followedByCurrentUser\":false},{\"name\":\"AI, ML & Data Engineering\",\"id\":\"16690\",\"url\":\"\/ai-ml-data-eng\",\"followers\":6077,\"followedByCurrentUser\":false},{\"name\":\"Antipatterns\",\"id\":\"439\",\"url\":\"\/antipatterns\",\"followers\":21,\"followedByCurrentUser\":false},{\"name\":\"Distributed Systems\",\"id\":\"6137\",\"url\":\"\/Distributed_Systems\",\"followers\":338,\"followedByCurrentUser\":false},{\"name\":\"Java9\",\"id\":\"7097\",\"url\":\"\/Java9\",\"followers\":5540,\"followedByCurrentUser\":false},{\"name\":\"DevOps\",\"id\":\"6043\",\"url\":\"\/Devops\",\"followers\":5131,\"followedByCurrentUser\":false},{\"name\":\"Reactive Programming\",\"id\":\"15453\",\"url\":\"\/reactive-programming\",\"followers\":12430,\"followedByCurrentUser\":false},{\"name\":\"Site Reliability Engineering\",\"id\":\"17182\",\"url\":\"\/sre\",\"followers\":82,\"followedByCurrentUser\":false},{\"name\":\"Development\",\"id\":\"6815\",\"url\":\"\/development\",\"followers\":4154,\"followedByCurrentUser\":false}]');
var userContentLikesInPage = [];
var userCommentsLikesInPage = [];
var currentUserId = 2;
</script>
<!-- Latest Version -->
<script src="https://cc.cdn.civiccomputing.com/9/cookieControl-9.x.min.js" type="text/javascript"></script>
<script>
const config = {
apiKey: '8910ea974a96ffb6f927952b4ae9b9b0cc3e5973',
product: 'PRO_MULTISITE',
// general settings
consentCookieExpiry: 90,
encodeCookie: true,
sameSiteCookie: true, // if false, cookie set as SameSite=None;secure;
sameSiteValue: ';secure', // either 'Strict', 'Lax', or 'None'
subDomains: true,
initialState: 'notify', // 'notify','top','box' require pro licence
notifyOnce: false,
setInnerHTML: true,
//layout settings
layout: 'slideout',
position: 'left',
theme: 'light',
acceptButton: true,
rejectButton: true,
closeOnGlobalChange: true,
closeStyle: 'icon',
toggleType: 'slider',
notifyDismissButton: true,
settingsStyle: 'link',
excludedCountries: ['US'],
/*accessibility: {
disableSiteScrolling: true,
},*/
statement: {
description: 'For more detailed information about the cookies we use, see our',
name: 'Cookie Policy',
url: 'https://www.infoq.com/cookie-policy',
updated: '01/01/2024',
},
// cookies starting from 'cookie_expire' are from live.infoq.com but since we use the same tool on the same domain we need to specify those too so
// that infoq.com cookieControl does not delete live.infoq.com cookies(also infoq.com cookies have been specified in live.infoq.com cookieControl configs)
necessaryCookies: ['RegUserCookie', 'UserCookie', 'IdpCookie', 'ConversionTrackingV2_','PSAdialog','*P13NWN*','topbarSurvey','__bkm','JSESSIONID','mp_','_mixpanel','CloudFront-Key-Pair-Id','CloudFront-Policy','CloudFront-Signature','cookie_expire','discount_promo_closed','discount_promo_code','discount_promo_submitted','exit_survey_popup','referrer_popup','voting_popup_*','AWSALB','AWSALBCORS','aws-waf-token','iq_sub_preview'],
optionalCookies: [
{
name: 'analytics',
label: 'Analytics',
description: 'Analytical cookies help us to improve our website by collecting and reporting information on its usage.',
cookies: ['_ga', '_ga*', '_gid', '_gat', '__utma', '__utmt', '__utmb', '__utmc', '__utmz', '__utmv'],
onAccept: function(){
gtag('consent', 'update', {'analytics_storage': 'granted'});
},
onRevoke: function(){
gtag('consent', 'update', {'analytics_storage': 'denied'});
}
},
{
name: 'marketing',
label: 'Advertising',
description: 'We use advertising cookies to display advertisements to you for our products.',
onAccept: function(){
gtag('consent', 'update', {'ad_storage': 'granted', 'ad_personalization': 'granted', 'ad_user_data': 'granted'});
},
onRevoke: function(){
gtag('consent', 'update', {'ad_storage': 'denied', 'ad_personalization': 'denied', 'ad_user_data': 'denied'});
}
}
],
text : {
// main preference panels
title: '<h3>Our use of cookies</h3>',
intro: 'We use necessary cookies to make our site work. Functional cookies help enhance the performance and functionality of the site. '+
'We\'d also like to set analytics cookies to help us improve your experience by measuring how you use the site. '+
'These will be set only if you accept. ',
acceptSettings: 'I Accept',
rejectSettings: 'I Do Not Accept',
necessaryTitle : '<h3>Necessary Cookies</h3>',
necessaryDescription : 'Necessary cookies enable core functionality ' +
'such as page navigation and access to secure areas. '+
'The website cannot function properly without '+
'these cookies, and can only be disabled by changing '+
'your browser preferences.',
closeLabel: 'Close Cookie Control',
cornerButton: 'Set cookie preferences',
// main preference panel controls
on: 'On',
off : 'Off',
thirdPartyTitle : 'Some cookies require your attention',
thirdPartyDescription : 'Consent for the following cookies could not be '+
'automatically revoked. Please follow the link(s) '+
'below to opt out manually.',
// notification panels (only accessible for pro licences)
notifyTitle : 'Your choice regarding cookies on this site',
notifyDescription : 'We use cookies to optimise site functionality and '+
'give you the best possible experience.',
accept : 'I Accept',
reject: 'I Do Not Accept',
settings : 'Settings',
},
branding : {
removeAbout: true,
},
};
// do not load this for local envs only. for testing on local envs remove/modify this condition
if(InfoQConstants.pageUrl.indexOf('local')===-1){
CookieControl.load( config );
}
</script>
<script type="application/ld+json">
{
"@context": "http://schema.org",
"@type": "NewsArticle",
"mainEntityOfPage": {
"@type": "WebPage",
"@id": "https://www.infoq.com/articles/anatomy-cascading-failure/"
},
"headline": "How to Avoid Cascading Failures in Distributed Systems",
"image": ["https://res.infoq.com/articles/anatomy-cascading-failure/en/smallimage/Anatomy-of-cascading-failure-logo-1581673261387.jpg"
],
"datePublished": "2020-02-20T05:30:00+0000",
"dateModified": "2020-02-20T05:30:00+0000",
"author": [
{
"@type": "Person",
"name": "Laura Nolan",
"url": "https://www.infoq.com/profile/Laura-Nolan/"
}
],
"publisher": {
"@type": "Organization",
"name": "InfoQ",
"logo": {
"@type": "ImageObject",
"url": "https://assets.infoq.com/resources/en/infoQ-logo-big.jpg"
}
},
"description": "Cascading failures are failures that involve some kind of feedback mechanism—in other words, vicious cycles in action. Laura Nolan explores them using public accounts of real production incidents."
}
</script>
<meta name="ifq:pageType" content="ARTICLE_PAGE"/>
<script type="text/javascript">
InfoQConstants.pageType = 'ARTICLE_PAGE';
</script>
<meta property="og:type" content="website" />
<meta property="og:image" content="https://res.infoq.com/articles/anatomy-cascading-failure/en/headerimage/Anatomy-of-cascading-failure-h-1581673260484.jpg"/>
<meta property="twitter:image" content="https://res.infoq.com/articles/anatomy-cascading-failure/en/headerimage/Anatomy-of-cascading-failure-h-1581673260484.jpg"/>
<meta property="og:title" content="How to Avoid Cascading Failures in Distributed Systems "/>
<meta property="og:description" content="Cascading failures are failures that involve some kind of feedback mechanism. In distributed software systems they generally involve a feedback loop where some event causes either a reduction in capacity, an increase in latency, or a spike of errors.&#xA0; Laura Nolan explores them using public accounts of real production incidents." />
<meta property="og:site_name" content="InfoQ"/>
<meta property="og:url" content="https://www.infoq.com/articles/anatomy-cascading-failure/"/>
<meta name="twitter:card" content="summary_large_image" />
<meta name="twitter:title" content="How to Avoid Cascading Failures in Distributed Systems "/>
<meta name="twitter:description" content="Cascading failures are failures that involve some kind of feedback mechanism. In distributed software systems they generally involve a feedback loop where some event causes either a reduction in capacity, an increase in latency, or a spike of errors.&#xA0; Laura Nolan explores them using public accounts of real production incidents." />
<link rel="image_src" href="https://res.infoq.com/articles/anatomy-cascading-failure/en/headerimage/Anatomy-of-cascading-failure-h-1581673260484.jpg"/>
<link rel="stylesheet" href="https://cdn.infoq.com/statics_s2_20260910111657/styles/prism.css"/>
<meta name="keywords" content="anatomy cascading failure,Architecture &amp; Design,DevOps,Antipatterns,Distributed Systems,Patterns,Site Reliability Engineering,Architecture,"/>
<meta name="description" content="Cascading failures are failures that involve some kind of feedback mechanism—in other words, vicious cycles in action. Laura Nolan explores them using public accounts of real production incidents."/>
<meta name="tprox" content="1582176600000" />
<script type="text/javascript">
$(function() {
var box = $('.related_sponsors');
if (box.length === 0) {
return;
}
var nextSibling = $(box).next();
if ($('#lowerFullwidthVCR').length === 0) {
moveLowerInThePage(box);
return;
}
if (blockElement(nextSibling)) {
moveLowerInThePage(box);
return;
}
while (!blockElement($(nextSibling).next())) {
nextSibling = $(nextSibling).next();
if (firstLowerThanSecond(nextSibling, box)) {
return;
}
}
moveLowerInThePage(box);
function blockElement(elem) {
if ($(elem).prop('tagName') === 'BLOCKQUOTE'
|| $(elem).prop('tagName') === 'IMG'
|| $(elem).prop('tagName') === 'PRE'
|| ($(elem).prop('tagName') === 'P' && $(elem).children().length === 1 && $(elem).children('img').length === 1)) {
return true;
}
return false;
}
function firstLowerThanSecond(elem1, elem2) {
var elem1BottomYCoordinate = $(elem1).offset().top + $(elem1).height()
var elem2BottomYCoordinate = $(elem2).offset().top + $(elem2).height();
if (elem1BottomYCoordinate > elem2BottomYCoordinate) {
return true;
}
return false;
}
function moveLowerInThePage(box) {
if ($('#lowerFullwidthVCR').length !== 0) {
$('#lowerFullwidthVCR').replaceWith(box);
}
$(box).removeClass('stacked');
}
});
</script>
<script type="text/javascript" src="https://cdn.infoq.com/statics_s2_20260910111657/scripts/relatedVcr.min.js"></script>
<script type="application/javascript">
var communityIds = "2498,1893";
var topicIds = "435,1964,397,5774,876";
VCR.loadAllVcrs(communityIds, topicIds);
</script>
<script type="text/javascript" src='https://cdn.infoq.com/statics_s2_20260910111657/scripts/subscription.js'></script>
<script type="text/javascript" src="https://cdn.infoq.com/statics_s2_20260910111657/scripts/infoq.js"></script>
<script type="text/javascript">
document.addEventListener('DOMContentLoaded', function() {
if (!window || !window.infoq) return
infoq.init()
})
</script>
<script defer data-domain="infoq.com" src="https://plausible.io/js/script.file-downloads.hash.outbound-links.pageview-props.tagged-events.js"></script>
<script>window.plausible = window.plausible || function() { (window.plausible.q = window.plausible.q || []).push(arguments) }</script>
</head>
<body >
<!-- Google Tag Manager (noscript) -->
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-W9GJ5DL"
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager (noscript) -->
<div class="intbt">
<a href="/int/bt/" title="bt">BT</a>
</div>
<script type="text/javascript">
var allCountries = [{"id":3,"name":"Afghanistan"},{"id":244,"name":"Åland"},{"id":6,"name":"Albania"},{"id":61,"name":"Algeria"},{"id":13,"name":"American Samoa"},{"id":1,"name":"Andorra"},{"id":9,"name":"Angola"},{"id":5,"name":"Anguilla"},{"id":11,"name":"Antarctica"},{"id":4,"name":"Antigua and Barbuda"},{"id":12,"name":"Argentina"},{"id":7,"name":"Armenia"},{"id":16,"name":"Aruba"},{"id":15,"name":"Australia"},{"id":14,"name":"Austria"},{"id":17,"name":"Azerbaijan"},{"id":31,"name":"Bahamas"},{"id":24,"name":"Bahrain"},{"id":20,"name":"Bangladesh"},{"id":19,"name":"Barbados"},{"id":35,"name":"Belarus"},{"id":21,"name":"Belgium"},{"id":36,"name":"Belize"},{"id":26,"name":"Benin"},{"id":27,"name":"Bermuda"},{"id":32,"name":"Bhutan"},{"id":29,"name":"Bolivia"},{"id":254,"name":"Bonaire, Sint Eustatius, and Saba"},{"id":18,"name":"Bosnia and Herzegovina"},{"id":34,"name":"Botswana"},{"id":33,"name":"Bouvet Island"},{"id":30,"name":"Brazil"},{"id":104,"name":"British Indian Ocean Territory"},{"id":28,"name":"Brunei Darussalam"},{"id":23,"name":"Bulgaria"},{"id":22,"name":"Burkina Faso"},{"id":25,"name":"Burundi"},{"id":114,"name":"Cambodia"},{"id":46,"name":"Cameroon"},{"id":37,"name":"Canada"},{"id":52,"name":"Cape Verde"},{"id":121,"name":"Cayman Islands"},{"id":40,"name":"Central African Republic"},{"id":207,"name":"Chad"},{"id":45,"name":"Chile"},{"id":47,"name":"China"},{"id":53,"name":"Christmas Island"},{"id":38,"name":"Cocos (Keeling) Islands"},{"id":48,"name":"Colombia"},{"id":116,"name":"Comoros"},{"id":39,"name":"Congo (Democratic Republic)"},{"id":41,"name":"Congo (People\u0027s Republic)"},{"id":44,"name":"Cook Islands"},{"id":49,"name":"Costa Rica"},{"id":43,"name":"Cote D\u0027Ivoire"},{"id":97,"name":"Croatia"},{"id":51,"name":"Cuba"},{"id":253,"name":"Curaçao"},{"id":54,"name":"Cyprus"},{"id":55,"name":"Czech Republic"},{"id":58,"name":"Denmark"},{"id":57,"name":"Djibouti"},{"id":59,"name":"Dominica"},{"id":60,"name":"Dominican Republic"},{"id":213,"name":"East Timor"},{"id":62,"name":"Ecuador"},{"id":64,"name":"Egypt"},{"id":203,"name":"El Salvador"},{"id":87,"name":"Equatorial Guinea"},{"id":66,"name":"Eritrea"},{"id":63,"name":"Estonia"},{"id":68,"name":"Ethiopia"},{"id":72,"name":"Falkland Islands (Malvinas)"},{"id":74,"name":"Faroe Islands"},{"id":71,"name":"Fiji"},{"id":70,"name":"Finland"},{"id":75,"name":"France"},{"id":80,"name":"French Guiana"},{"id":170,"name":"French Polynesia"},{"id":208,"name":"French Southern Territories"},{"id":76,"name":"Gabon"},{"id":84,"name":"Gambia"},{"id":79,"name":"Georgia"},{"id":56,"name":"Germany"},{"id":81,"name":"Ghana"},{"id":82,"name":"Gibraltar"},{"id":88,"name":"Greece"},{"id":83,"name":"Greenland"},{"id":78,"name":"Grenada"},{"id":86,"name":"Guadeloupe"},{"id":91,"name":"Guam"},{"id":90,"name":"Guatemala"},{"id":249,"name":"Guernsey"},{"id":85,"name":"Guinea"},{"id":92,"name":"Guinea-Bissau"},{"id":93,"name":"Guyana"},{"id":98,"name":"Haiti"},{"id":95,"name":"Heard Island and McDonald Islands"},{"id":96,"name":"Honduras"},{"id":94,"name":"Hong Kong"},{"id":99,"name":"Hungary"},{"id":107,"name":"Iceland"},{"id":103,"name":"India"},{"id":100,"name":"Indonesia"},{"id":106,"name":"Iran"},{"id":105,"name":"Iraq"},{"id":101,"name":"Ireland"},{"id":245,"name":"Isle of Man"},{"id":102,"name":"Israel"},{"id":108,"name":"Italy"},{"id":109,"name":"Jamaica"},{"id":111,"name":"Japan"},{"id":250,"name":"Jersey"},{"id":110,"name":"Jordan"},{"id":122,"name":"Kazakhstan"},{"id":112,"name":"Kenya"},{"id":115,"name":"Kiribati"},{"id":243,"name":"Kosovo"},{"id":120,"name":"Kuwait"},{"id":113,"name":"Kyrgyzstan"},{"id":123,"name":"Laos"},{"id":132,"name":"Latvia"},{"id":124,"name":"Lebanon"},{"id":129,"name":"Lesotho"},{"id":128,"name":"Liberia"},{"id":133,"name":"Libya"},{"id":126,"name":"Liechtenstein"},{"id":130,"name":"Lithuania"},{"id":131,"name":"Luxembourg"},{"id":143,"name":"Macau"},{"id":139,"name":"Macedonia"},{"id":137,"name":"Madagascar"},{"id":151,"name":"Malawi"},{"id":153,"name":"Malaysia"},{"id":150,"name":"Maldives"},{"id":140,"name":"Mali"},{"id":148,"name":"Malta"},{"id":138,"name":"Marshall Islands"},{"id":145,"name":"Martinique"},{"id":146,"name":"Mauritania"},{"id":149,"name":"Mauritius"},{"id":238,"name":"Mayotte"},{"id":152,"name":"Mexico"},{"id":73,"name":"Micronesia"},{"id":136,"name":"Moldova"},{"id":135,"name":"Monaco"},{"id":142,"name":"Mongolia"},{"id":246,"name":"Montenegro"},{"id":147,"name":"Montserrat"},{"id":134,"name":"Morocco"},{"id":154,"name":"Mozambique"},{"id":141,"name":"Myanmar"},{"id":155,"name":"Namibia"},{"id":164,"name":"Nauru"},{"id":163,"name":"Nepal"},{"id":161,"name":"Netherlands"},{"id":8,"name":"Netherlands Antilles"},{"id":156,"name":"New Caledonia"},{"id":166,"name":"New Zealand"},{"id":160,"name":"Nicaragua"},{"id":157,"name":"Niger"},{"id":159,"name":"Nigeria"},{"id":165,"name":"Niue"},{"id":158,"name":"Norfolk Island"},{"id":118,"name":"North Korea"},{"id":144,"name":"Northern Mariana Islands"},{"id":162,"name":"Norway"},{"id":167,"name":"Oman"},{"id":173,"name":"Pakistan"},{"id":180,"name":"Palau"},{"id":178,"name":"Palestinian Territory"},{"id":168,"name":"Panama"},{"id":171,"name":"Papua New Guinea"},{"id":181,"name":"Paraguay"},{"id":169,"name":"Peru"},{"id":172,"name":"Philippines"},{"id":176,"name":"Pitcairn"},{"id":174,"name":"Poland"},{"id":179,"name":"Portugal"},{"id":177,"name":"Puerto Rico"},{"id":182,"name":"Qatar"},{"id":183,"name":"Reunion"},{"id":184,"name":"Romania"},{"id":185,"name":"Russian Federation"},{"id":186,"name":"Rwanda"},{"id":193,"name":"Saint Helena"},{"id":117,"name":"Saint Kitts and Nevis"},{"id":125,"name":"Saint Lucia"},{"id":251,"name":"Saint Martin"},{"id":175,"name":"Saint Pierre and Miquelon"},{"id":229,"name":"Saint Vincent and the Grenadines"},{"id":247,"name":"Saint-Barthélemy"},{"id":236,"name":"Samoa"},{"id":198,"name":"San Marino"},{"id":202,"name":"Sao Tome and Principe"},{"id":187,"name":"Saudi Arabia"},{"id":199,"name":"Senegal"},{"id":248,"name":"Serbia"},{"id":189,"name":"Seychelles"},{"id":197,"name":"Sierra Leone"},{"id":192,"name":"Singapore"},{"id":252,"name":"Sint Maarten"},{"id":196,"name":"Slovakia"},{"id":194,"name":"Slovenia"},{"id":188,"name":"Solomon Islands"},{"id":200,"name":"Somalia"},{"id":239,"name":"South Africa"},{"id":89,"name":"South Georgia and the South Sandwich Islands"},{"id":119,"name":"South Korea"},{"id":255,"name":"South Sudan"},{"id":67,"name":"Spain"},{"id":127,"name":"Sri Lanka"},{"id":190,"name":"Sudan"},{"id":201,"name":"Suriname"},{"id":195,"name":"Svalbard and Jan Mayen"},{"id":205,"name":"Swaziland"},{"id":191,"name":"Sweden"},{"id":42,"name":"Switzerland"},{"id":204,"name":"Syria"},{"id":220,"name":"Taiwan"},{"id":211,"name":"Tajikistan"},{"id":221,"name":"Tanzania"},{"id":210,"name":"Thailand"},{"id":209,"name":"Togo"},{"id":212,"name":"Tokelau"},{"id":216,"name":"Tonga"},{"id":218,"name":"Trinidad and Tobago"},{"id":215,"name":"Tunisia"},{"id":217,"name":"Turkey"},{"id":214,"name":"Turkmenistan"},{"id":206,"name":"Turks and Caicos Islands"},{"id":219,"name":"Tuvalu"},{"id":223,"name":"Uganda"},{"id":222,"name":"Ukraine"},{"id":2,"name":"United Arab Emirates"},{"id":77,"name":"United Kingdom"},{"id":224,"name":"United States Minor Outlying Islands"},{"id":226,"name":"Uruguay"},{"id":225,"name":"USA"},{"id":227,"name":"Uzbekistan"},{"id":234,"name":"Vanuatu"},{"id":228,"name":"Vatican City (Holy See)"},{"id":230,"name":"Venezuela"},{"id":233,"name":"Vietnam"},{"id":231,"name":"Virgin Islands (British)"},{"id":232,"name":"Virgin Islands (U.S.)"},{"id":235,"name":"Wallis and Futuna"},{"id":65,"name":"Western Sahara"},{"id":237,"name":"Yemen"},{"id":241,"name":"Zaire"},{"id":240,"name":"Zambia"},{"id":242,"name":"Zimbabwe"}];
var gdprCountriesIds = [196,194,191,184,179,174,161,148,132,131,130,108,101,99,97,88,77,75,70,67,63,58,56,55,54,37,23,21,14];
</script>
<section data-nosnippet class="section container subscribe-box hidden">
<div class="container__inner">
<div class="actions">
<div class="actions__left">
<h2>InfoQ Software Architects' Newsletter</h2>
<span><p>A monthly overview of things you need to know as an architect or aspiring architect.</p>
<p><a href="https://www.infoq.com/software-architects-newsletter#placeholderPastIssues">View an example</a></p>
</span>
<div class="newsletter__subscribe">
<form class="form gdpr" name="dataCollectCampaignNewsletterForm" id="dataCollectCampaignNewsletterForm" action="#" onsubmit="dataCollectNewsletter.saveSubscription(); return false;">
<div class="field newsletter__mail input__text input__no-label input__medium email">
<label for="email-dataCollectnewsletter-infoq" class="label field__label">Enter your e-mail address</label>
<input id="email-dataCollectnewsletter-infoq" name="footerNewsletterEmail" placeholder="Enter your e-mail address" class="input field__input" type="email"/>
<input type="text" name="emailH" id="input_email_h_d" aria-required="false" style="display:none !important" tabindex="-1" autocomplete="off"/>
<input type="hidden" id="fnt_d" name="fnt_d" value="pgarAk5gUTjmy6ja"/>
<input type="hidden" id="dataCollectNewsletterType" name="dataCollectNewsletterType" value="regular"/>
<input type="hidden" id="cmpi_d" name="cmpi_d" value="4"/>
</div>
<div class="hidden">
<span class="input__select field country">
<label for="input-dataCollect-newsletter-country" class="label field__label">Select your country</label>
<select id="input-dataCollect-newsletter-country" class="select field__input">
<option value="" class="select__option">Select a country</option>
</select>
<p class="input__message field__desc"></p>
</span>
<span class="input__checkbox field hidden">
<input type="checkbox" id="gdpr-consent-campaign">
<label for="gdpr-consent-campaign" class="label"><span>I consent to InfoQ.com handling my data as explained in this <a href="https://www.infoq.com/privacy-notice">Privacy Notice</a>.</span></label>
</span>
</div>
<input type="submit" role="button" value="Subscribe" class="button button__medium button__red" onclick="return dataCollectNewsletter.validateEmail('Invalid email address');"/>
</form>
<p class="meta">
<a href="/privacy-notice/" target="_blank">We protect your privacy.</a>
</p>
<span class="success" style="display:none;" id="dataCollectNewsletterMessage"></span>
</div>
</div>
<div class="actions__right">
<button aria-label="Close" class="close closeBox button button__unstyled button__icon icon icon__close-black icon--only">Close</button>
</div>
</div>
</div>
</section>
<script type="text/javascript">
var dataCollectNewsletter = new Newsletter('Enter your e-mail address',
'email-dataCollectnewsletter-infoq', 'dataCollectNewsletterType','dataCollectNewsletterMessage', 'fnt_d', 'input_email_h_d', 'input-dataCollect-newsletter-country', 'cmpi_d','popup_all_pages');
</script>
<div class="infoq" id="infoq">
<!-- ####### SITE START ######### -->
<section class="section container promo hidden">
<div class="container__inner">
<div class="actions">
<div class="actions__left">
<p>
<span>Online InfoQ Architect Certification (Sept 14): This is where senior engineers pressure-test real architecture decisions.</span>
<a class="btn" href="/url/pb/bd633791-e46d-4845-82fa-330bf7e3c0c6/" target="_blank" rel="nofollow">
Register Now
</a>
</p>
</div>
<div class="actions__right">
<button aria-label="Close"
class="close button button__unstyled button__icon icon icon__close-white icon--only close-top-promo">Close
</button>
</div>
</div>
</div>
</section>
<header class="header">
<button aria-label="Toggle Navigation" tabindex="0" class="burger header__toggle button">Toggle Navigation <span></span><span></span><span></span></button>
<div class="header__container container">
<div class="container__inner">
<div data-nosnippet class="actions header__top">
<div class="actions__left">
<p class="header__desc my-0">
Facilitating the Spread of Knowledge and Innovation in Professional Software Development
</p>
<div class="button__dropdown dropdown header__more my-0 dropdown__center">
<button aria-label="English edition" class="button button__unstyled button__small">English edition </button>
<div class="dropdown__holder">
<!---->
<div class="dropdown__content">
<ul class="no-style dropdown__nav languagesEdition">
<li class="active"><a href="#" onclick="return false;">English edition</a></li>
<li><a href="https://www.infoq.cn">Chinese edition</a></li>
<li><a href="/jp/">Japanese edition</a></li>
<li><a href="/fr/">French edition</a></li>
</ul>
</div>
<!---->
</div>
</div>
<a class="my-0 contribute-link" role="button" href="/write-for-infoq/" title="Write for InfoQ">
Write for InfoQ
</a>
</div>
<div class="actions__right">
<div>
<form id="searchForm" name="search-form" action="/search.action" enctype="multipart/form-data" class="search icon__search icon icon__green">
<div class="field search__bar input__text input__no-label input__small">
<label for="search" class="label field__label">Search</label>
<input name="queryString" type="text" id="search" value="" placeholder="Search" class="input field__input">
<input type="hidden" name="page" value="1"/>
<input type="hidden" size="15" name="searchOrder">
</div>
<input value="Search" type="submit" class="search__go">
</form>
</div>
<div class="action_button__group">
<a href="/reginit.action?" onclick="return UserActions_Login.addLoginInfo(this);" class="button button__green button__small">Register</a>
<a href="/social/keycloakLogin.action?fl=login" onclick="return UserActions_Login.addLoginInfo(this);" class="button button__small button__outline">Sign in</a>
</div>
<div class="modal_auth_required">
<div class="modal_auth_required__actions">
<h2 class="h2">Unlock the full InfoQ experience</h2>
<p>Unlock the full InfoQ experience by logging in! Stay updated with your favorite authors and topics, engage with content, and download exclusive resources.</p>
<a href="/social/keycloakLogin.action?fl=login" onclick="return UserActions_Login.addLoginInfo(this);" class="button button__has-transition">Log In</a>
<div class="separator"><span>or</span></div>
<h3 class="h3">Don't have an InfoQ account?</h3>
<a href="/reginit.action?" onclick="return UserActions_Login.addLoginInfo(this);" class="button__outline button__has-transition">Register</a>
</div>
<div class="modal_auth_required__content">
<ul class="newsletter__features">
<li><strong>Stay updated on topics and peers that matter to you</strong>Receive instant alerts on the latest insights and trends.</li>
<li><strong>Quickly access free resources for continuous learning</strong>Minibooks, videos with transcripts, and training materials.</li>
<li><strong>Save articles and read at anytime</strong>Bookmark articles to read whenever youre ready.</li>
</ul>
</div>
<form id="login-form" data-vv-scope="login" class="login__form form" action="#" data-gtm-form-interact-id="0">
<input type="hidden" id="loginWidgetOrigin" name="loginWidgetOrigin" value="mainLogin">
<input type="hidden" name="fromP13N" id="isP13n" value="">
<input type="hidden" name="fromP13NId" id="p13n-id" value="">
<input type="hidden" name="fromP13NType" id="p13n-type" value="">
</form>
</div>
</div>
</div>
<div class="header__middle">
<div class="logo header__logo">
<a href="/" class="logo__symbol active">
Logo - Back to homepage
</a>
</div>
<div class="content-items">
<a href="/news/" class="icon icon__news">News</a>
<a href="/articles/" class="icon icon__articles">Articles</a>
<a href="/presentations/" class="icon icon__presentations">Presentations</a>
<a href="/podcasts/" class="icon icon__podcasts">Podcasts</a>
<a href="/minibooks/" class="icon icon__guides">Guides</a>
</div>
<div class="header__items columns">
<div>
<h3 class="widget__heading">Topics</h3>
<nav class="nav header__nav topics" data-trk-ref="header_personas">
<div class="has--subnav li-nav">
<a href="/development/" title="Development" class="nav__category">Development</a>
<div class="nav__subnav subnav">
<ul class="subnav__categories no-style">
<li><a href="/java/" title="Java">Java</a></li>
<li><a href="/kotlin/" title="Kotlin">Kotlin</a></li>
<li><a href="/dotnet/" title=".Net">.Net</a></li>
<li><a href="/c_sharp/" title="C#">C#</a></li>
<li><a href="/swift/" title="Swift">Swift</a></li>
<li><a href="/golang/" title="Go">Go</a></li>
<li><a href="/rust/" title="Rust">Rust</a></li>
<li><a href="/javascript/" title="JavaScript">JavaScript</a></li>
</ul>
<div class="subnav__content" data-id="6815">
<div class="heading__container actions">
<div class="actions__left">
<h3 class="heading section__heading">Featured in Development</h3>
</div>
</div>
<ul data-size="large" data-horizontal="true" data-tax="" taxonomy="articles" class="cards no-style">
<li>
<div class="card__content">
<div class="card__data">
<h4 class="card__title">
<a href="/presentations/solopreneur-journey">A Solopreneur's Journey: From Engineer to Puzzle Master and Storyteller</a>
</h4>
<p class="card__excerpt">Joe Cassavaugh shares his journey from software engineer to successful solopreneur with a $2M+ indie franchise. He explains how he scaled production to 10 games in 5 years, adopted Unity to boost velocity 4-6x, optimized content pipelines, and leveraged refactoring patterns. He discusses key trade-offs between corporate engineering and solopreneurship for senior devs and leaders.</p>
<div class="card__footer"></div>
</div>
<a href="/presentations/solopreneur-journey" class="card__header">
<img loading="lazy" alt="A Solopreneur's Journey: From Engineer to Puzzle Master and Storyteller" src="https://imgopt.infoq.com/fit-in/100x100/filters:quality(80)/presentations/solopreneur-journey/en/smallimage/joe-cassavaugh-thumbnail-1788338336404.jpg" class="card__image"/>
</a>
</div>
</li>
</ul>
<a href="/development/" class="button__more button button__large button__arrow arrow__right">All in development</a>
</div>
</div>
</div>
<div class="has--subnav li-nav">
<a href="/architecture-design/" title="Architecture &amp; Design" class="nav__category">Architecture &amp; Design</a>
<div class="nav__subnav subnav">
<ul class="subnav__categories no-style">
<li><a href="/architecture/" title="Architecture">Architecture</a></li>
<li><a href="/enterprise-architecture/" title="Enterprise Architecture">Enterprise Architecture</a></li>
<li><a href="/performance-scalability/" title="Scalability/Performance">Scalability/Performance</a></li>
<li><a href="/design/" title="Design">Design</a></li>
<li><a href="/Case_Study/" title="Case Studies">Case Studies</a></li>
<li><a href="/microservices/" title="Microservices">Microservices</a></li>
<li><a href="/servicemesh/" title="Service Mesh">Service Mesh</a></li>
<li><a href="/DesignPattern/" title="Patterns">Patterns</a></li>
<li><a href="/Security/" title="Security">Security</a></li>
</ul>
<div class="subnav__content" data-id="6816">
<div class="heading__container actions">
<div class="actions__left">
<h3 class="heading section__heading">Featured in Architecture &amp; Design</h3>
</div>
</div>
<ul data-size="large" data-horizontal="true" data-tax="" taxonomy="articles" class="cards no-style">
<li>
<div class="card__content">
<div class="card__data">
<h4 class="card__title">
<a href="/minibooks/next-gen-architecture-ai-era">Next-Gen Architecture Playbook: Insights and Patterns for the AI Era</a>
</h4>
<p class="card__excerpt">This eMag examines how architects can lead with clarity in a rapidly evolving engineering world, distilling industry insights into field-tested practices for teams. Together, these stories reveal a core theme: the technology leader’s role is expanding from building systems to guiding how tech behaves and learns, while enabling engineers and organizations to bring out their best.</p>
<div class="card__footer"></div>
</div>
<a href="/minibooks/next-gen-architecture-ai-era" class="card__header">
<img loading="lazy" alt="Next-Gen Architecture Playbook: Insights and Patterns for the AI Era" src="https://imgopt.infoq.com/fit-in/100x100/filters:quality(80)/minibooks/next-gen-architecture-ai-era/en/smallimage/thumb-image-emag-next-gen-architecture-playbook-1787922032442.jpg" class="card__image"/>
</a>
</div>
</li>
</ul>
<a href="/architecture-design/" class="button__more button button__large button__arrow arrow__right">All in architecture-design</a>
</div>
</div>
</div>
<div class="has--subnav li-nav">
<a href="/ai-ml-data-eng/" title="AI Infrastructure" class="nav__category">AI Infrastructure</a>
<div class="nav__subnav subnav">
<ul class="subnav__categories no-style">
<li><a href="/bigdata/" title="Big Data">Big Data</a></li>
<li><a href="/machinelearning/" title="Machine Learning">Machine Learning</a></li>
<li><a href="/nosql/" title="NoSQL">NoSQL</a></li>
<li><a href="/database/" title="Database">Database</a></li>
<li><a href="/data-analytics/" title="Data Analytics">Data Analytics</a></li>
<li><a href="/streaming/" title="Streaming">Streaming</a></li>
</ul>
<div class="subnav__content" data-id="16690">
<div class="heading__container actions">
<div class="actions__left">
<h3 class="heading section__heading">Featured in AI, ML &amp; Data Engineering</h3>
</div>
</div>
<ul data-size="large" data-horizontal="true" data-tax="" taxonomy="articles" class="cards no-style">
<li>
<div class="card__content">
<div class="card__data">
<h4 class="card__title">
<a href="/articles/when-spec-driven-development-pays-off">When Spec-Driven Development Pays Off</a>
</h4>
<p class="card__excerpt">AI coding assistants have become a core part of software development. AI-generated code has shown productivity gains, but it's also contributing to security weaknesses and familiar bug patterns. In this article, author Nitin Garg highlights the bottleneck has moved from code generation to code verification, and how to detect &amp; mitigate it when the AI-generated behavior diverges from the intent.</p>
<div class="card__footer"></div>
</div>
<a href="/articles/when-spec-driven-development-pays-off" class="card__header">
<img loading="lazy" alt="When Spec-Driven Development Pays Off" src="https://imgopt.infoq.com/fit-in/100x100/filters:quality(80)/articles/when-spec-driven-development-pays-off/en/smallimage/When-Spec-Driven-Development-Pays-Off-thumb-1788790605567.jpg" class="card__image"/>
</a>
</div>
</li>
</ul>
<a href="/ai-ml-data-eng/" class="button__more button button__large button__arrow arrow__right">All in ai-ml-data-eng</a>
</div>
</div>
</div>
<div class="has--subnav li-nav">
<a href="/culture-methods/" title="Culture &amp; Methods" class="nav__category">Culture &amp; Methods</a>
<div class="nav__subnav subnav">
<ul class="subnav__categories no-style">
<li><a href="/agile/" title="Agile">Agile</a></li>
<li><a href="/diversity/" title="Diversity">Diversity</a></li>
<li><a href="/leadership/" title="Leadership">Leadership</a></li>
<li><a href="/lean/" title="Lean/Kanban">Lean/Kanban</a></li>
<li><a href="/personal-growth/" title="Personal Growth">Personal Growth</a></li>
<li><a href="/scrum/" title="Scrum">Scrum</a></li>
<li><a href="/sociocracy/" title="Sociocracy">Sociocracy</a></li>
<li><a href="/software_craftsmanship/" title="Software Craftmanship">Software Craftmanship</a></li>
<li><a href="/team-collaboration/" title="Team Collaboration">Team Collaboration</a></li>
<li><a href="/testing/" title="Testing">Testing</a></li>
<li><a href="/ux/" title="UX">UX</a></li>
</ul>
<div class="subnav__content" data-id="6817">
<div class="heading__container actions">
<div class="actions__left">
<h3 class="heading section__heading">Featured in Culture &amp; Methods</h3>
</div>
</div>
<ul data-size="large" data-horizontal="true" data-tax="" taxonomy="articles" class="cards no-style">
<li>
<div class="card__content">
<div class="card__data">
<h4 class="card__title">
<a href="/presentations/rust-refactoring">Accelerating Performance by Incrementally Integrating Rust Into Existing Codebase</a>
</h4>
<p class="card__excerpt">Lily Mara explains how to avoid high-risk software rewrites through incremental FFI refactoring. She shares how engineering teams can replace Python bottlenecks with Rust via PyO3, demonstrating how to achieve dramatic function-level speedups, seamless integration testing, and meaningful infrastructure cost savings without microservice overhead.</p>
<div class="card__footer"></div>
</div>
<a href="/presentations/rust-refactoring" class="card__header">
<img loading="lazy" alt="Accelerating Performance by Incrementally Integrating Rust Into Existing Codebase" src="https://imgopt.infoq.com/fit-in/100x100/filters:quality(80)/presentations/rust-refactoring/en/smallimage/lily-mara-thumbnail-1788338447565.jpeg" class="card__image"/>
</a>
</div>
</li>
</ul>
<a href="/culture-methods/" class="button__more button button__large button__arrow arrow__right">All in culture-methods</a>
</div>
</div>
</div>
<div class="has--subnav li-nav">
<a href="/devops/" class="nav__category">DevOps</a>
<div class="nav__subnav subnav">
<ul class="subnav__categories no-style">
<li><a href="/infrastructure/" title="Infrastructure">Infrastructure</a></li>
<li><a href="/continuous_delivery/" title="Continuous Delivery">Continuous Delivery</a></li>
<li><a href="/automation/" title="Automation">Automation</a></li>
<li><a href="/containers/" title="Containers">Containers</a></li>
<li><a href="/cloud-computing/" title="Cloud">Cloud</a></li>
<li><a href="/observability/" title="Observability">Observability</a></li>
</ul>
<div class="subnav__content" data-id="6043">
<div class="heading__container actions">
<div class="actions__left">
<h3 class="heading section__heading">Featured in DevOps</h3>
</div>
</div>
<ul data-size="large" data-horizontal="true" data-tax="" taxonomy="articles" class="cards no-style">
<li>
<div class="card__content">
<div class="card__data">
<h4 class="card__title">
<a href="/presentations/ai-platform-engineering-roundtable">Platform Engineering in the Age of AI</a>
</h4>
<p class="card__excerpt">The panelists explain how platform teams adapt to support AI-assisted engineering, highlighting which capabilities belong in the platform. They discuss trade-offs between standardization and developer autonomy, while sharing strategies to manage AI tooling, security guardrails, and shifting workflows.</p>
<div class="card__footer"></div>
</div>
<a href="/presentations/ai-platform-engineering-roundtable" class="card__header">
<img loading="lazy" alt="Platform Engineering in the Age of AI" src="https://imgopt.infoq.com/fit-in/100x100/filters:quality(80)/presentations/ai-platform-engineering-roundtable/en/smallimage/thumb-1788861349861.jpg" class="card__image"/>
</a>
</div>
</li>
</ul>
<a href="/devops/" class="button__more button button__large button__arrow arrow__right">All in devops</a>
</div>
</div>
</div>
<div class="li-nav">
<a rel="noreferrer noopener" href="https://events.infoq.com/" class="nav__category" title="Events" target="_blank">Events</a>
</div>
</nav>
</div>
<div>
<h3 class="widget__heading">Helpful links</h3>
<ul class="no-style header__nav">
<li>
<a href="/about-infoq" title="About InfoQ">
About InfoQ
</a>
</li>
<li>
<a href="/infoq-editors" title="InfoQ Editors">
InfoQ Editors
</a>
</li>
<li>
<a href="/write-for-infoq" title="Write for InfoQ">
Write for InfoQ
</a>
</li>
<li>
<a href="https://c4media.com/" target="_blank" title="About C4Media">
About C4Media
</a>
</li>
<li>
<a rel="noreferrer noopener" href="https://c4media.com/diversity" title="Diversity" target="_blank">Diversity</a>
</li>
</ul>
</div>
<div>
<h3 class="widget__heading">Choose your language</h3>
<ul class="language__switcher no-style">
<li class="active"><a href="#" onclick="return false;" title="InfoQ English">En</a></li>
<li><a href="https://www.infoq.cn">中文</a></li>
<li><a href="/jp/">日本</a></li>
<li><a href="/fr/">Fr</a></li>
</ul>
</div>
</div>
</div>
<div data-nosnippet class="actions header__bottom header__bottom__events">
<div class="actions__left">
<div class="events-list" role="list" aria-label="Upcoming events">
<div class="event-item" role="listitem" data-event-type="certification" data-sort-date="2026-09-14">
<a class="event-card" href="https://certification.qconferences.com/architecture?utm_source=infoq&amp;utm_medium=referral&amp;utm_campaign=homepageheader_onlinecohortarchitecture26" target="_blank" rel="noopener noreferrer" aria-label="Online InfoQ Architect Certification, Sep 14, 2026">
<div class="event-date">
<div class="event-date-box" aria-hidden="true"><span>Sep</span><span>14,</span><span>2026</span></div>
</div>
<div class="event-content">
<p class="event-title">Architect Certification</p>
<p class="event-description">Distributed systems, decentralized decisions, platform engineering, and AI architecture. <br/><strong>Online. Register Now.</strong></p>
</div>
</a>
</div>
<div class="event-item" role="listitem" data-event-type="certification" data-sort-date="2026-09-18">
<a class="event-card" href="https://certification.qconferences.com/ai-assisted-engineering?utm_source=infoq&utm_medium=referral&utm_campaign=homepageheader_onlinecohortaiassistedengineering26" target="_blank" rel="noopener noreferrer" aria-label="AI-Assisted Engineering Certification, Sep 18, 2026">
<div class="event-date">
<div class="event-date-box" aria-hidden="true"><span>Sep</span><span>18,</span><span>2026</span></div>
</div>
<div class="event-content">
<p class="event-title">AI-Assisted Engineering Certification</p>
<p class="event-description">Your coding agent moves fast. Who checks whether it's making the codebase better? <br/><strong>Online. Register Now.</strong></p>
</div>
</a>
</div>
<div class="event-item" role="listitem" data-event-type="certification" data-sort-date="2026-10-14">
<a class="event-card" href="https://certification.qconferences.com/ai-security-privacy?utm_source=infoq&amp;utm_medium=referral&amp;utm_campaign=homepageheader_onlinecohortaisecurity26" target="_blank" rel="noopener noreferrer" aria-label="Online InfoQ AI Security &amp; Privacy Engineering Certification, August 26, 2026">
<div class="event-date">
<div class="event-date-box" aria-hidden="true"><span>Oct</span><span>14,</span><span>2026</span></div>
</div>
<div class="event-content">
<p class="event-title">AI Security &amp; Privacy Engineering Certification</p>
<p class="event-description">Secure and govern production AI systems, from sensitive data to guardrails, evals, and audits.<br><strong>Online. Register now.</strong></p>
</div>
</a>
</div>
<div class="event-item" role="listitem" data-event-type="qcon" data-sort-date="2026-11-16">
<a class="event-card" href="https://qconsf.com/?utm_source=infoq&amp;utm_medium=referral&amp;utm_campaign=homepageheader_qsf26" target="_blank" rel="noopener noreferrer" aria-label="QCon San Francisco, Nov 16-20, 2026">
<div class="event-date">
<div class="event-date-box" aria-hidden="true"><span>Nov</span><span>16-20,</span><span>2026</span></div>
</div>
<div class="event-content">
<p class="event-title">QCon San Francisco</p>
<p class="event-description">What's working across AI, architecture, and leadership, from the teams doing it. <br/><strong>Register. Early bird ends August 11.</strong></p>
</div>
</a>
</div>
<div class="event-item" role="listitem" data-event-type="qcon" data-sort-date="2026-12-15">
<a class="event-card" href="https://newyork.qcon.ai/?utm_source=infoq&utm_medium=referral&utm_campaign=homepageheader_qainy26" target="_blank" rel="noopener noreferrer" aria-label="QCon AI New York, December 15-16, 2026">
<div class="event-date">
<div class="event-date-box" aria-hidden="true"><span>Dec</span><span>15-16,</span><span>2026</span></div>
</div>
<div class="event-content">
<p class="event-title">QCon AI New York</p>
<p class="event-description">Production AI across agents, context, evals, security, and infrastructure, from the senior engineers building it.<br/><strong>Registration open.</strong></p>
</div>
</a>
</div>
<div class="event-item" role="listitem" data-event-type="qcon" data-sort-date="2027-04-13">
<a class="event-card" href="https://qconlondon.com/?utm_source=infoq&amp;utm_medium=referral&amp;utm_campaign=homepageheader_qconlondon27" target="_blank" rel="noopener noreferrer" aria-label="QCon London, April 13-16, 2027">
<div class="event-date">
<div class="event-date-box" aria-hidden="true"><span>Apr</span><span>13-16,</span><span>2027</span></div>
</div>
<div class="event-content">
<p class="event-title">QCon London</p>
<p class="event-description">What early-adopter teams have proven in production, across 15 engineering tracks. <br/><strong> Register. Early bird ends August 11.</strong></p>
</div>
</a>
</div>
</div>
</div>
<!---->
</div>
</div>
</div>
</header>
<!-- ####### CONTENT START ######### -->
<main>
<article data-type="article" class="article">
<section class="section container white">
<div class="container__inner">
<p class="crumbs">
<span data-nosnippet><a href="/" title="InfoQ Homepage">InfoQ Homepage</a></span>
<span data-nosnippet><a href="/articles" title="Articles">Articles</a></span>
<span data-nosnippet class="active">How to Avoid Cascading Failures in Distributed Systems</span>
</p>
<div class="actions">
<div class="actions__left">
<div class="article__category architecture-design">
<a
href="/architecture-design/"
class="article__tag box--info" title="Architecture &amp; Design">
Architecture &amp; Design
</a>
</div>
</div>
<div class="actions__right">
<div data-nosnippet class="notice">
<div class="box--warning">
<a target="_blank" rel="nofollow" href="https://certification.qconferences.com/ai-assisted-engineering?utm_source=infoq&amp;utm_medium=referral&amp;utm_campaign=infoqyellowbox_onlinecohortaiassistedengineering26 ">InfoQ Certified AI-Assisted Engineering Program (online, Oct 19): Build the harness that holds. </a>
</div>
</div>
</div>
</div>
<div class="actions heading__container article__heading">
<div class="actions__left">
<h1 class="heading">
How to Avoid Cascading Failures in Distributed Systems
</h1>
</div>
</div>
<p id="translated_br" class="article__translated">
Leia em
<a
href="/br/articles/anatomy-cascading-failure"
>
Português
</a>
<span class="icon flag br"></span>
</p>
<script type="text/javascript">
$("#translated_"+InfoQConstants.userDetectedCountryCode.toLowerCase()).show();
</script>
<div class="columns article__explore">
<div class="article__main" data-col="4/6">
<div class="article__metadata metadata">
<p class="article__readTime date">Feb 20, 2020<span class="dot"></span>
15
min read
</p>
<div class="widget article__authors">
<div>
<p class="meta">by</p>
<ul class="no-style authors">
<li data-id="author-Laura-Nolan">
<p class="meta author__bio">
<a href="/profile/Laura-Nolan/" class="avatar author__avatar" aria-label="Laura Nolan"></a>
<span class="author__name">
<a href="/profile/Laura-Nolan/" class="author__link">Laura Nolan</a>
</span>
</p>
</li>
</ul>
</div>
<div>
<p class="meta">reviewed by</p>
<ul class="no-style authors">
<li data-id="author-Charles-Humble">
<p class="meta author__bio">
<a href="/profile/Charles-Humble/" class="avatar author__avatar" aria-label="Charles Humble"></a>
<span class="author__name">
<a href="/profile/Charles-Humble/" class="author__link">Charles Humble</a>
</span>
</p>
</li>
</ul>
</div>
</div>
<div data-nosnippet class="cta_write_infoq">
<h4 class="heading text-left footer__heading">Follow us on </h4>
<div class="social__links columns social__links__row">
<a style="color:#222 !important;" rel="noreferrer noopener" href="https://bit.ly/4bg6QM8" target="_blank"><span class="icon icon__large icon__social icon__youtube"></span><div><span class="social__count" style="font-weight:700 !important">Youtube</span><span class="social__followers">232K Followers</span></div></a>
<a style="color:#222 !important;" rel="noreferrer noopener" href="https://bit.ly/44IzAtf" target="_blank"><span class="icon icon__large icon__social icon__linkedin"></span><div><span class="social__count" style="font-weight:700 !important">Linkedin</span><span class="social__followers">26K Followers</span></div></a>
<a style="color:#222 !important;" rel="noreferrer noopener" href="https://bit.ly/4eYXrtM" target="_blank"><span class="icon icon__large icon__social icon__instagram"></span><div><span class="social__count" style="font-weight:700 !important">Instagram</span><span class="social__followers">New</span></div></a>
<a style="color:#222 !important;" rel="noreferrer noopener" href="https://bit.ly/3RaJalC" target="_blank"><span class="icon icon__large icon__social icon__rss"></span><div><span class="social__count" style="font-weight:700 !important">RSS</span><span class="social__followers">19K Readers</span></div></a>
<a style="color:#222 !important;" rel="noreferrer noopener" href="https://bit.ly/4pfxivv" target="_blank"><span class="icon icon__large icon__social icon__twitter"></span><div><span class="social__count" style="font-weight:700 !important">X</span><span class="social__followers">57.1k Followers</span></div></a>
<a style="color:#222 !important;" rel="noreferrer noopener" href="https://bit.ly/3QrGMH2" target="_blank"><span class="icon icon__large icon__social icon__fb"></span><div><span class="social__count" style="font-weight:700 !important">Facebook</span><span class="social__followers">21K Likes</span></div></a>
<a style="color:#222 !important;" rel="noreferrer noopener" href="https://bit.ly/4eS8FjG" target="_blank"><span class="icon icon__large icon__social icon__bluesky"></span><div><span class="social__count" style="font-weight:700 !important">Bluesky</span><span class="social__followers">New</span></div></a>
</div>
</div>
</div>
<div class="article__content">
<div class="actions__left actions__sidebar article__actions">
<button id="toggleLikeContent" class="icon button button__icon like button__gray icon__like" aria-label="Like">Like</button>
<div class="bookmarkReading">
<button id="bookmarkBtn" data-ref="bookmarkPage" aria-label="Read later" class="login button__gray button button__icon icon icon__bookmark icon--only"> </button>
<!-- new dropdown rading list -->
<div class="my-0 button__dropdown dropdown reading_list dropdown__center" id="showBookmarks">
<button aria-label="Reading list" class="button button__unstyled button__small"></button>
<div class="dropdown__holder">
<div class="dropdown__content">
<ul class="no-style dropdown__nav">
<li class="active"><a href="/showbookmarks.action">Reading list</a></li>
</ul>
</div>
</div>
</div>
<div id="toastContainer" class="toast-none toast-hide">
<div class="toast"><span></span></div>
</div>
</div>
<script type="text/javascript">
if(loggedIn){
$('#showBookmarks').css('display', 'flex');
}
function performBookmark() {
Bookmarks.toggleBookmark('articles', 'anatomy-cascading-failure');
}
infoq.event.on('bookmarkRequested', function(e) {
Bookmarks.toggleBookmark('articles', 'anatomy-cascading-failure');
});
infoq.event.on("loaded", function(){
if(loggedIn){
var href = window.location.href;
if(href.indexOf("#bookmarkPage") != -1){
$('#bookmarkBtn').click();
}
}
});
$(document).ready(function() {
if(Bookmarks.isContentBookmarked == 'true'){
$('#bookmarkBtn').addClass('button__green');
$('#bookmarkBtn').removeClass('button__gray');
}else{
$('#bookmarkBtn').removeClass('button__green');
$('#bookmarkBtn').addClass('button__gray');
}
});
</script>
</div>
<div class="article__data">
<div class="takeaways">
<h3 class="icon icon__takeaways heading">Key Takeaways</h3>
<ul>
<li>Cascading failures are failures that involve some kind of feedback mechanism. In distributed software systems they generally involve a feedback loop where some event causes either a reduction in capacity, an increase in latency, or a spike of errors; then the response of the other components of the system makes the original problem worse.&nbsp;</li>
<li>It&rsquo;s often very difficult to scale out of a cascading failure by adding more capacity to your service: new healthy instances get hit with &nbsp;excess load instantly and become saturated, so you can&rsquo;t get to a point where you have enough serving capacity to handle the load.&nbsp;</li>
<li>Sometimes, the only fix is to take your entire service down in order to recover, and then reintroduce load.</li>
<li>The potential for cascading failures is inherent in many, if not most, distributed systems. If you haven&rsquo;t seen one in your system yet, it doesn&rsquo;t mean you&rsquo;re immune; you may just be operating comfortably within your system&rsquo;s limits. There&rsquo;s no guarantee that will be true tomorrow, or next week.</li>
</ul>
<div class="separator separator__three-dots center"></div>
</div>
<h2>What is a Cascading Failure?</h2>
<style type="text/css">.shame--article-innersidebar {
background: #f5f7f9;
padding: 12px;
}
@media screen and (min-width: 768px) {
.shame--article-innersidebar {
float: right;
width: 45%;
margin: 0 0 12px 12px;
}
.shame--article-innerwidth {
width: 50%;
}
}
</style>
<div class="shame--article-innersidebar">
<p style="font-size: 13px; line-height: 1.65; margin-top: 12px;">In this article I&rsquo;ll discuss public accounts of real production incidents. I have the greatest respect for all engineering organisations concerned. Everyone who runs distributed software systems experiences outages, but not all organisations are open and honest enough to write clear public accounts of their production incidents. Those who do deserve only appreciation and esteem (and sometimes, our sympathies!).</p>
</div>
<p>Cascading failures are failures that involve some kind of feedback mechanism&mdash;in&nbsp; other words, vicious cycles in action.<br />
&nbsp;<br />
The outage that <a href="https://aws.amazon.com/message/5467D2/">took down Amazon DynamoDB in US-East-1 on September 20, 2015</a> for over four hours is a classic example of a cascading failure. There were two subsystems involved: storage servers and a metadata service. Storage servers request their data partition assignments from the metadata service, which is replicated across data centers. At the time the incident occurred, the average time to retrieve partition assignments had risen significantly, because of the introduction of a new index type (Global Secondary Indexes or GSIs), but the capacity of the metadata service hadn&rsquo;t been increased,&nbsp; nor had the configured deadlines for the data-partition assignment request operation. Any request that didn&rsquo;t succeed within that deadline was considered to have failed, and the client would retry.<br />
&nbsp;<br />
<img _href="img://2Anatomy-of-cascading-failure-1-1581675001504.jpg" alt="" data-src="articles/anatomy-cascading-failure/en/resources/2Anatomy-of-cascading-failure-1-1581675001504.jpg" style="width: 294px; height: 303px;" src="https://imgopt.infoq.com/fit-in/3000x4000/filters:quality(85)/filters:no_upscale()/articles/anatomy-cascading-failure/en/resources/2Anatomy-of-cascading-failure-1-1581675001504.jpg" rel="share"><br />
&nbsp;<small><strong>Figure 1: Services involved in September 2015 DynamoDB outage</strong></small></p>
<div class="clear"></div>
<script type="text/javascript">
var uriMapping = "articles";
var showVcr = "false";
var fillWithVcr = "true";
var sponsorshipsJson = "{&quot;links&quot;:[{&quot;styleName&quot;:&quot;pdf&quot;,&quot;style&quot;:&quot;PDF&quot;,&quot;text&quot;:&quot;The Practical Agentic Coding Playbook&quot;,&quot;id&quot;:&quot;94d846d6-a8fa-4030-80d6-183fe4cefb7f&quot;,&quot;target&quot;:&quot;https://www.infoq.com/vendorcontent/show.action?vcr=430ffe46-9047-4739-a817-b7d64d070721&amp;utm_source=infoq&amp;utm_medium=VCR&amp;utm_campaign=vcr_fixed_link&quot;,&quot;active&quot;:true}]}";
var sponsoredLinks = $.parseJSON($("<div/>").html(sponsorshipsJson).text()).links;
var numberOfSponsoredVcrIds = sponsoredLinks != null ? sponsoredLinks.length : 0;
var maxItems = 5 - numberOfSponsoredVcrIds;
var intervalVcrSponsorship = setInterval(function() {
if (window.vcrsLoaded) {
clearInterval(intervalVcrSponsorship);
if(showVcr || fillWithVcr) {
if(fillWithVcr) {
for(var index in window.vcrList) {
if(VCR.isVcrSponsored(sponsoredLinks, window.vcrList[index])) {
VCR.addToExcludedList(window.vcrList[index]);
}
}
}
var vcrs = VCR.getByTopicsAndCommunities(window.vcrList, topicIds, communityIds, maxItems, false, null);
if (vcrs != null && vcrs.length > 0 || (sponsoredLinks != null && sponsoredLinks.length > 0)) {
VCR.addToExcludedList(vcrs);
var $widget = jQuery('.related__vc');
getCommonElements(vcrs, uriMapping, "EMBEDDED");
$widget.css("display", "flex");
} else {
var $widget = jQuery('.related__vc');
$widget.remove();
}
}
window.contentVcrFinished = true;
// search for infoq.event.on("contentVcrFinished",... to see how/where it is used
infoq.event.trigger("contentVcrFinished");
}
}, 200);
</script>
<input type="hidden" name="" value="2498" id="cont_item_primary_topic"/>
<div class="related__group related__vc nocontent f_rvcbox" data-trk-view="true" data-trk-impr="true" data-place="EMBEDDED"
style="display: flex"
>
<ul class="no-style rvc__list">
<h4 class="heading">Related Sponsors</h4>
<div class="f_rvcList"></div>
<li>
<span class="icon rvc__icon icon__small icon__pdf"></span>
<h5 class="rvc__title">
<a href="/url/f/94d846d6-a8fa-4030-80d6-183fe4cefb7f/" class="rvc__link" rel="nofollow">
The Practical Agentic Coding Playbook
</a>
</h5>
</li>
</ul>
<div class="related__prsp f_sponsorship" data-trk-view="true" data-place="EMBEDDED" data-trk-impr="true" jsh="{&quot;topic&quot;:&quot;Architecture &amp; Design&quot;,&quot;id&quot;:&quot;12605306-ee86-4a5c-bc22-ab8b965bfc4a&quot;,&quot;title&quot;:&quot;Moderne - Architect Influence Program 08/11/2026 - 11/11/2026&quot;}">
<h4 class="heading">Related Sponsor</h4>
<a href="/url/f/e7ff8a76-884f-40bc-b88b-f0845d30d60d/" target="_blank" rel="nofollow">
<img loading="lazy" src="https://imgopt.infoq.com//fit-in/290x500/filters:quality(100)/filters:no_upscale()/sponsorship/topic/12605306-ee86-4a5c-bc22-ab8b965bfc4a/ModerneLogoMicrosite-1786453474094.png" class="related__img" alt="Related sponsor icon"/>
</a>
<div class="related__desc">
<p style="clear: both; padding: 10px 0 0 0;"><b>Automated code transformation at scale. Fix vulnerabilities, migrate frameworks, and modernize legacy code across every repo. <a href="/url/f/5e9fca59-ba83-4a5a-8d7e-b8b1bdcc8ebe/" target="_blank" rel="nofollow">Try Moderne Free</a>.</b></p>
</div>
</div>
</div>
<p>The incident was triggered by a transient network problem, which caused some of the storage servers to not receive their partition assignments.</p>
<p>Those storage servers removed themselves from service, and they also continued to retry their requests for partition assignments. The metadata servers became overwhelmed with load from these requests and were therefore&nbsp; slower to respond, which caused more requests sent to them to timeout and be retried. These retries increased the load on the service further. The metadata service was so badly overloaded that operators had to firewall the it off from the storage servers in order to add extra capacity.&nbsp; This meant effectively taking the entire DynamoDB service offline in US-East-1.</p>
<h2>Why are Cascading Failures So Bad?</h2>
<p>The biggest issue with cascading failures is that they can take down your entire system, toppling instances of your service one by one, until your entire load-balanced service is unhealthy.</p>
<div id="lowerFullwidthVCR"></div>
<p>The second problem is they&rsquo;re an exceptionally hard type of failure from which to recover. They normally start with some small perturbation &mdash; like a transient network issue, a small spike in load, or the failure of a few instances. Instead of recovering to a normal state over time, the system gets into a worse state. A system in cascading failure won&rsquo;t self-heal; it&rsquo;ll only be restored through human intervention.</p>
<p>The third problem is that if the right conditions exist in your system, cascading failures can strike with no warning. Unfortunately, the basic preconditions for cascading failures are difficult to avoid: it&rsquo;s simply failover. If failure of a component can cause retries, or cause load to shift to other parts of your system, then the basic conditions for cascading failure are there. But all is not lost: there are patterns we can apply that help us defend our systems against cascading failures.</p>
<h2>Feedback Cycles: How Cascading Failures Take Down Our Systems</h2>
<p>Cascading failures in distributed software systems generally involve a feedback loop where some event causes either a reduction in capacity, an increase in latency, or a spike of errors; then the response of the other components of the system makes the original problem worse.</p>
<p>The Causal Loop Diagram (CLD) is a good tool to understand these incidents. Below is a CLD for the DynamoDB incident from earlier.</p>
<p><img _href="img://1Anatomy-of-cascading-failure-2-1581673057928.jpg" alt="" class="zoom-image" data-src="articles/anatomy-cascading-failure/en/resources/1Anatomy-of-cascading-failure-2-1581673057928.jpg" style="width: 640px; height: 480px;" src="https://imgopt.infoq.com/fit-in/3000x4000/filters:quality(85)/filters:no_upscale()/articles/anatomy-cascading-failure/en/resources/1Anatomy-of-cascading-failure-2-1581673057928.jpg" rel="share"><br />
<small><strong>Figure 2: Causal Loop Diagram for September 2015 DynamoDB outage</strong></small></p>
<p><em>CLDs are a tool from System Dynamics, an approach to modelling complex systems invented by Jay Forrester at MIT. Each arrow shows how two quantities in the system interact. A &lsquo;+&rsquo; beside the arrow means that an increase in the first quantity will tend to increase the second quantity, and a &lsquo;-&rsquo; means there is an inverse relationship. So, it follows that increasing the capacity of the service, i.e. the number of instances serving it, will reduce the load per instance. Adding a new type of index, or retries from failing requests will tend to increase it.</em></p>
<p>Where we have a cycle in the diagram, as we do here, we can look at the signs and see if the cycle is balanced, a mixture of &lsquo;+&rsquo; and &lsquo;-&rsquo;. Here, we have all &lsquo;+&rsquo; signs in the cycle, meaning that it&rsquo;s not balanced. In System Dynamics, this is called a &quot;reinforcing cycle&quot; (hence the &lsquo;R&rsquo; in the centre with the arrow around it).</p>
<p>Having a reinforcing cycle in your system doesn&rsquo;t mean&nbsp; it&rsquo;ll constantly be in overload. If capacity is sufficient to meet demand, it will work fine. However, it does mean that in the right circumstances &mdash; a reduction in capacity, a spike in load, or anything else that that pushes latency or timeouts above a critical threshold &mdash; a cascading failure might occur, such as happened to DynamoDB.</p>
<p>A key realisation &mdash; a very similar cycle exists for most replicated services with clients that retry on failure. This is a very, very common pattern. Later in this article we will examine some patterns that help prevent this cycle turning into a cascading failure scenario.</p>
<p>Let&rsquo;s look at another example of a cascading failure: <a href="https://blog.parse.ly/post/1738/kafkapocalypse/">Parsely&rsquo;s Kafkapocalyspe</a>. The systems involved here are different, but the pattern is similar. Due to a launch, Parsely had increased the load on their systems, including their Kafka cluster. Unbeknownst to them, they were close to the network limits on the EC2 nodes on which they were running their Kafka brokers. At some point, one broker hit its network limit, and became unavailable. Load increased on other brokers, as clients failed over, and very quickly all the brokers were down.</p>
<p>As with the earlier AWS scenario, we see from the Parsely outage how quickly a system can go from being stable and predictable to a very nonlinear and dysfunctional state once a limit is breached, and how recovery doesn&rsquo;t happen until operators intervene.</p>
<h2>Recovering From Cascading Failure</h2>
<p>It&rsquo;s often very difficult to scale out of a cascading failure by adding more capacity to your service: new healthy instances get hit with&nbsp; excess load instantly and become saturated, so you can&rsquo;t get to a point where you have enough serving capacity to handle the load.</p>
<p>Many load-balancing systems use a health check to send requests only to healthy instances,&nbsp; though you might need to turn that behavior off during an incident to avoid focusing all the load on brand-new instances as they are brought up. The same is true of any kind of orchestration or management service that kills instances of your servers that fail health checks (such as <a href="https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/">kubernetes liveness probes</a>); they will remove overloaded instances, contributing to the capacity problem.</p>
<p>Sometimes, the only fix is to take your entire service down in order to recover, and then reintroduce load. We saw this in the DynamoDB outage. <a href="https://labs.spotify.com/2013/06/04/incident-management-at-spotify/">Spotify had an outage in 2013</a> where they also had to take the impacted service offline to recover. This is especially likely where the overloaded service doesn&rsquo;t impose any limit on the number of queued or current requests.</p>
<h2>Six Cascading Failure Antipatterns</h2>
<h3>Antipattern 1: Accepting unbounded numbers of incoming requests</h3>
<p>Anyone who&rsquo;s done much benchmarking has probably noticed that individual instances of a service generally hit a peak in throughput; then, if load increases further, you see a drop in throughput and an increase in latency. This change happens because some of the work in any service is not parallelizable (there&rsquo;s a good explanation of the maths in <a href="https://www.usenix.org/conference/srecon18americas/presentation/schwartz">Baron Schwartz&#39;s talk &#39;Approaching the Unacceptable Workload Boundary&#39;</a>). In a state of cascading failure, individual service instances can end up with so many queued requests, or so many concurrent threads trying to execute, that the service can become totally unresponsive and may not recover without intervention (generally, a restart). Duo experienced conditions like this during an <a href="https://status.duo.com/incidents/4w07bmvnt359">outage in 2018</a>: <em>&quot;We determined that limiting was ineffective because of the way our application queues requests while waiting for a database connection. In this case, these queued requests had built up in such a way that the database could not recover as it tried to process this large backlog of requests, even after traffic subsided and the limits were in place.&quot;</em></p>
<p>This is why setting a limit on the load on each instance of your service is so important. <a href="https://www.usenix.org/conference/srecon17europe/program/presentation/cruz">Loadshedding</a> at a load-balancer works, but you should set limits in your service as well, for defense in depth. The mechanisms to implement a limit on concurrent requests vary, depending on the programming language and server framework you&rsquo;re using, but might be as simple as a semaphore. Netflix&rsquo;s <a href="https://github.com/Netflix/concurrency-limits">concurrency-limits</a> tool is a Java-based example.</p>
<p>Failing requests early when a server is heavily loaded is actually also good for clients. It&rsquo;s better to get a fast failure and retry to a different instance of the service, or serve an error or a degraded experience, than wait until the request deadline is up (or indefinitely, if there&rsquo;s no request deadline set). Allowing this to happen can lead to slowness that spreads&nbsp; through an entire microservice architecture, and it can sometimes be tricky to find the service that is the underlying cause, when every service has ground to a halt.</p>
<h3>Antipattern 2: Dangerous client retry behaviour</h3>
<p>We don&rsquo;t always have control over client behaviour, but if you do control your clients, moderating client request patterns can be a very useful tool. At the most basic level, clients should limit the number of times they retry a failed request within a short period of time. In a system where clients retry too many times in a tight loop, any minor spike of errors can cause a flood of retried requests, effectively DOSing the service. Square experienced this in <a href="https://medium.com/square-corner-blog/incident-summary-2017-03-16-2f65be39297">March 2017</a> when their Redis instance became unavailable because of a code path that would retry a transaction up to 500 times. Here is sample Golang code for that simple retry loop:</p>
<pre>
<code class="language-go">const MAX_RETRIES = 500
for i := 0; i &lt; MAX_RETRIES; i++ {
_, err := doServerRequest()
if err == nil {
break
}
}</code></pre>
<p>When Square&rsquo;s engineers rolled out a fix to reduce the number of retries, the feedback loop immediately ended and their service began serving normally.</p>
<p>Clients should use an exponentially increasing backoff between retry attempts. It&rsquo;s also good practice to add a little random noise, or jitter, to the backoff time. This &lsquo;smears&rsquo; a wave of retries out over time, so a service that&rsquo;s temporarily glitching for a few milliseconds doesn&rsquo;t get hit with twice its normal load when all clients simultaneously retry. The number of retries and how long to wait is application specific. User-facing requests should fail fast or return a degraded result of some kind, whereas batch or asynchronous processing can wait much longer.</p>
<p>Here is&nbsp; sample Golang code for a retry loop with exponential backoff and jitter:</p>
<pre>
<code class="language-go">const MAX_RETRIES = 5
const JITTER_RANGE_MSEC = 200
steps_msec := []int{100, 500, 1000, 5000, 15000}
rand.Seed(time.Now().UTC().UnixNano())
for i := 0; i &lt; MAX_RETRIES; i++ {
_, err := doServerRequest()
if err == nil {
break
}
time.Sleep(time.Duration(steps_msec[i] + rand.Intn(JITTER_RANGE_MSEC)) *
time.Millisecond)
}</code></pre>
<p>Modern best practice goes a step beyond exponential backoff and jitter. The <a href="https://www.martinfowler.com/bliki/CircuitBreaker.html">Circuit Breaker</a> application design pattern wraps calls to an external service and tracks success and failure of those calls over time. A sequence of failed calls will &lsquo;trip&rsquo; the circuit breaker, meaning that no more calls will be made to the failing external service, and clients attempting to make such calls will immediately get an error. Periodically, the circuit breaker will probe the external service by allowing one call through. If the probe request succeeds, the circuit breaker will reset and again start making calls to the external service.</p>
<p>Circuit breakers are powerful because they can share state across all requests from a client to the same backend, whereas exponential backoff is specific to a single request. Circuit breakers reduce the load on a struggling backend service more than any other approach. Here&rsquo;s a <a href="https://github.com/matttproud/golang_circuitbreaker">circuit breaker implementation</a> for Golang. Netflix&rsquo;s Hystrix includes a <a href="https://github.com/Netflix/Hystrix/wiki/How-it-Works#CircuitBreaker">Java circuit breaker</a>.</p>
<h3>Antipattern 3: Crashing on bad input &mdash; the &lsquo;Query of Death&rsquo;</h3>
<p>&nbsp;This &lsquo;query of death&rsquo; is any request to your system that can cause it to crash. A client may send a query of death, crash one instance of your service, and keep retrying, bringing further instances down. The reduction in capacity can then potentially bring your entire service down as the remaining instances get overloaded from the normal workload.</p>
<p>This kind of scenario can be the result of an attack on your service, but it may not be malicious, just bad luck. This is why it&rsquo;s a best practice to never exit or crash on unexpected inputs; a program should exit unexpectedly only if internal state seems to be incorrect and it would be unsafe to continue serving.</p>
<p>Fuzz testing is an automated testing practice that can help detect programs that crash on malformed inputs. Fuzz testing is especially important for any service that is exposed to untrusted inputs, which means anything outside your organisation.</p>
<h3>Antipattern 4: Proximity-based failover and the domino effect</h3>
<p>What do your systems do if an entire data center or availability zone goes down? If the answer is &lsquo;fail over to the next nearest one&rsquo; then your systems have the potential for a cascading failure.</p>
<p><img _href="img://1Anatomy-of-cascading-failure-3-1581673058193.jpg" alt="" class="zoom-image" data-src="articles/anatomy-cascading-failure/en/resources/1Anatomy-of-cascading-failure-3-1581673058193.jpg" style="width: 640px; height: 480px;" src="https://imgopt.infoq.com/fit-in/3000x4000/filters:quality(85)/filters:no_upscale()/articles/anatomy-cascading-failure/en/resources/1Anatomy-of-cascading-failure-3-1581673058193.jpg" rel="share"></p>
<p><small><strong>Figure 3: Map of data center locations</strong></small></p>
<p>If you lost one of your US East Coast data centers in a topology, like the one shown above, then the other data center in that region would get roughly twice the load as soon as users failed over. If the remaining US East Coast data center couldn&rsquo;t manage the load and also failed, then the load would likely go primarily to US West Coast data centers (it&rsquo;s cheaper than sending traffic to Europe, usually). If those failed, then your remaining locations would likely go down next: like dominos. Your failover plan, which is intended to improve your system&rsquo;s reliability, has brought your entire service down.</p>
<p>Geographically balanced systems like this need to do one of two things: either make sure that load fails over in a way that doesn&rsquo;t overload the remaining data centers, or else maintain a lot of capacity everywhere.</p>
<p>Systems that are based on IP Anycast (like most DNS services and many CDNs) generally overprovision, specifically because anycast, which serves a single IP from many points on the Internet, gives you no way to control inbound traffic.</p>
<p>This level of overprovisioning for failure can be very expensive. For many systems, using a way to direct load to data centers that have capacity available makes more sense. This is often done using DNS load balancing (for example <a href="https://ns1.com/blog/using-load-shedding-for-intelligent-traffic-distribution-1">NS1&rsquo;s intelligent traffic distribution</a>).</p>
<h3>Antipattern 5: Work prompted by failure</h3>
<p>Sometimes, our services do work when a failure occurs. Consider a hypothetical distributed data store system that splits our data into blocks. We want a minimum number of replicas of each block, and we regularly check that we have the right number of copies. If we don&rsquo;t, then we start making new copies. Here&rsquo;s a pseudo-code snippet:</p>
<pre>
<code class="language-go">replicaChecker()
while true {
for each block in filesystem.GetAllBlocks() {
if block.replicasHeartbeatedOK() &lt; minReplicas {
block.StartCopyNewReplica()
}
}
}
}</code></pre>
<p><br />
<img _href="img://1Anatomy-of-cascading-failure-4-1581673058460.jpg" alt="" class="zoom-image" data-src="articles/anatomy-cascading-failure/en/resources/1Anatomy-of-cascading-failure-4-1581673058460.jpg" style="width: 640px; height: 480px;" src="https://imgopt.infoq.com/fit-in/3000x4000/filters:quality(85)/filters:no_upscale()/articles/anatomy-cascading-failure/en/resources/1Anatomy-of-cascading-failure-4-1581673058460.jpg" rel="share"></p>
<p><small><strong>Figure 4: Replication of data blocks after a failure.</strong></small></p>
<p>This approach will probably work fine if we lose one block, or one server of many. But what if we lose a substantial proportion of the servers? An entire rack? The serving capacity of the system will be reduced, and the remaining servers are going to be busy re-replicating data. We haven&rsquo;t put any limits on how much replication we&rsquo;re going to do at a time. Here&rsquo;s a Causal Loop Diagram showing the feedback loop.</p>
<p><img _href="img://1Anatomy-of-cascading-failure-5-1581673059011.jpg" alt="" class="zoom-image" data-src="articles/anatomy-cascading-failure/en/resources/1Anatomy-of-cascading-failure-5-1581673059011.jpg" style="width: 640px; height: 480px;" src="https://imgopt.infoq.com/fit-in/3000x4000/filters:quality(85)/filters:no_upscale()/articles/anatomy-cascading-failure/en/resources/1Anatomy-of-cascading-failure-5-1581673059011.jpg" rel="share"></p>
<p><small><strong>Figure 5: Causal loop diagram showing the feedback loop in the system</strong></small></p>
<p>The usual way around this is to delay replication (because failure is often transient), and limit the number of in-flight replication processes with something like the <a href="https://en.wikipedia.org/wiki/Token_bucket">token bucket</a> algorithm. The Causal Loop Diagram below shows how this changes the system: we still have a feedback loop, but there&rsquo;s now an inner balanced loop that prevents the feedback cycle from running away.</p>
<p><img _href="img://1Anatomy-of-cascading-failure-6-1581673058741.jpg" alt="" class="zoom-image" data-src="articles/anatomy-cascading-failure/en/resources/1Anatomy-of-cascading-failure-6-1581673058741.jpg" style="width: 640px; height: 480px;" src="https://imgopt.infoq.com/fit-in/3000x4000/filters:quality(85)/filters:no_upscale()/articles/anatomy-cascading-failure/en/resources/1Anatomy-of-cascading-failure-6-1581673058741.jpg" rel="share"></p>
<p><small><strong>Figure 6: Causal loop diagram showing rate limit on replication</strong></small></p>
<h3>Antipattern 6: Long startup times</h3>
<p>Sometimes, services are designed to do a lot of work on startup&mdash;perhaps by reading and caching a lot of data. This pattern is best avoided, for two reasons. First, it makes any form of autoscaling hard:by the time you&rsquo;ve detected an increase in load and started up your slow-to-start-up service, you may be in trouble. Second, if instances of your service fail for some reason (out of memory, or a query of death causes them to crash)&nbsp; it will take you a long time to get back to your usual serving capacity. Both of these conditions can easily lead to overload on your service.</p>
<h2>Reducing Cascading Failure Risks</h2>
<p>The potential for cascading failures is inherent in many, if not most, distributed systems. If you haven&rsquo;t seen one in your system yet, it doesn&rsquo;t mean you&rsquo;re immune; you may just be operating comfortably within your system&rsquo;s limits. There&rsquo;s no guarantee that will be true tomorrow, or next week.</p>
<p>We&rsquo;ve listed&nbsp; a number of antipatterns to avoid if you want to reduce the risk of experiencing a cascading failure. No service can withstand an arbitrary spike of load. Nobody wants their service to serve errors, but sometimes it&rsquo;s the lesser evil, when the alternative is to see your entire service grind to a standstill trying to deal with every incoming request.</p>
<h2>Further Reading</h2>
<ul>
<li>&#39;<a href="https://landing.google.com/sre/sre-book/chapters/addressing-cascading-failures/">Addressing Cascading Failures</a>,&#39; by Mike Ulrich, in <em>Site Reliability Engineering: How Google runs Production Systems</em>.&nbsp;</li>
<li>&#39;Stability Patterns&#39; chapter in <em>Release It!</em> by Michael T Nygard.</li>
<li>&#39;<a href="https://landing.google.com/sre/sre-book/chapters/handling-overload/">Handling Overload</a>&#39; chapter by Alejandro Forero Cuervo in <em>Site Reliability Engineering: How Google runs Production Systems</em>.&nbsp;</li>
</ul>
<h2>About the Author</h2>
<p><strong><img _href="img://2Laura-Nolan-1581673166424.jpg" alt="" data-src="articles/anatomy-cascading-failure/en/resources/2Laura-Nolan-1581673166424.jpg" style="margin-left: 10px; margin-right: 10px; float: left; width: 85px; height: 100px;" src="https://imgopt.infoq.com/fit-in/3000x4000/filters:quality(85)/filters:no_upscale()/articles/anatomy-cascading-failure/en/resources/2Laura-Nolan-1581673166424.jpg" rel="share">Laura Nolan</strong> is a Senior Staff Engineer at Slack Technologies in Dublin. Her background is in Site Reliability Engineering, software engineering, distributed systems, and computer science. She wrote the &#39;Managing Critical State&#39; chapter in the O&#39;Reilly &#39;Site Reliability Engineering&#39; book, as well as contributing to the more recent &#39;Seeking SRE&#39;. She is a member of the USENIX SREcon steering committee.</p>
<div id="with-toc">&nbsp;</div>
</div>
<div id="zoom-container"></div>
<input type="hidden" name="" value="Thank you for your review!" id="cr_messages_submitSuccess"/>
<input type="hidden" name="" value="Rating is required" id="cr_messages_ratingRequired"/>
<input type="hidden" name="" value="How to Avoid Cascading Failures in Distributed Systems" id="cr_item_title"/>
<input type="hidden" name="" value="Laura Nolan" id="cr_item_author"/>
<input type="hidden" name="" value="http://www.infoq.com/articles/anatomy-cascading-failure/" id="cr_item_url"/>
<input type="hidden" name="" value="articles" id="cr_item_ctype"/>
<input type="hidden" name="" value="en" id="cr_item_lang"/>
<input type="hidden" name="" value="1582176600000" id="cr_item_published_time"/>
<input type="hidden" name="" value="2498" id="cr_item_primary_topic"/>
<script type="text/javascript">
ContentRating.readMessages();
ContentRating.readContentItem();
</script>
<form class="box__border box form rate contentRatingWidget">
<h3 class="heading">Rate this Article</h3>
<div class="criterias">
<div class="crit" id="relevance_fieldset">
<div class="crit__name">Adoption</div>
<span class="stars">
<input type="radio" id="relevance-star5" name="rating-relevance" value="5" /><label class="star" for="relevance-star5" title="Innovator"></label>
<input type="radio" id="relevance-star4" name="rating-relevance" value="4" /><label class="star" for="relevance-star4" title="Early adopter"></label>
<input type="radio" id="relevance-star3" name="rating-relevance" value="3" /><label class="star" for="relevance-star3" title="Early majority"></label>
<input type="radio" id="relevance-star2" name="rating-relevance" value="2" /><label class="star" for="relevance-star2" title="Late majority"></label>
<input type="radio" id="relevance-star1" name="rating-relevance" value="1" /><label class="star" for="relevance-star1" title="Laggards"></label>
</span>
<span class="stars__total"></span>
</div>
<div class="crit" id="style_fieldset">
<div class="crit__name">Style</div>
<span class="stars">
<input type="radio" id="style-star5" name="rating-style" value="5" /><label class="star" for="style-star5" title="Exceptional"></label>
<input type="radio" id="style-star4" name="rating-style" value="4" /><label class="star" for="style-star4" title="Good"></label>
<input type="radio" id="style-star3" name="rating-style" value="3" /><label class="star" for="style-star3" title="Average"></label>
<input type="radio" id="style-star2" name="rating-style" value="2" /><label class="star" for="style-star2" title="Acceptable"></label>
<input type="radio" id="style-star1" name="rating-style" value="1" /><label class="star" for="style-star1" title="Poor"></label>
</span>
<span class="stars__total"></span>
</div>
</div>
</form>
<div class="reviews tabs contentRatingWidget">
<div id="editorReview" class="tab" data-title="Editor Review">
<form class="box__border form contentRatingWidget" onsubmit="return false;">
<span class="field input__textarea" aria-required="false">
<textarea id="editor_input" placeholder="Click to leave your review..." name="textarea" value="" class="field__input"></textarea>
<p class="input__message field__desc serverCallFeedback"></p>
</span>
<input class="button button__large submit_rating" type="submit" value="Submit"/>
</form>
</div>
<div id="chiefEditorReview" class="tab" data-title="Chief Editor Action">
<form class="box__border form contentRatingWidget" onsubmit="return false;">
<span class="field input__textarea">
<textarea id="chiefEditor_input" placeholder="Chief Editor action..." value="" class="field__input"></textarea>
<p class="input__message field__desc serverCallFeedback"></p>
</span>
<div class="actions">
<div class="actions__left">
<input class="button button__large submit_rating" type="submit" value="Submit"/>
</div>
<div class="actions__right">
<span class="field input__checkbox input--small">
<input type="checkbox" id="auth-checkbox" />
<label class="label" for="auth-checkbox"><span></span> Author Contacted</label>
</span>
</div>
</div>
</form>
</div>
</div>
<script type="text/javascript">
if (!InfoQConstants.editorUser || InfoQConstants.editorUser == 'false') {
$('.contentRatingWidget').remove();
} else {
if (InfoQConstants.chiefEditor !== 'undefined' && InfoQConstants.chiefEditor == 'false') {
$('#chiefEditorReview').remove();
}
}
</script>
<div class="widget article__fromTopic topics">
<div class="widget__head related__for-topic" data-id="439" data-trk-ref="content_primary_topic">
<h4 class="heading related__heading">
This content is in the <a href='/antipatterns/'>Antipatterns</a> topic
</h4>
</div>
<h5 class="heading related__inline">Related Topics:</h5>
<ul class="no-style topics related__topics topics__small" data-trk-ref="content_related_topic">
<li data-id="6816">
<a href="/architecture-design/" class="button related__topic button__small button__black">Architecture &amp; Design</a>
</li>
<li data-id="6043">
<a href="/Devops/" class="button related__topic button__small button__black">DevOps</a>
</li>
<li data-id="439">
<a href="/antipatterns/" class="button related__topic button__small button__black">Antipatterns</a>
</li>
<li data-id="6137">
<a href="/Distributed_Systems/" class="button related__topic button__small button__black">Distributed Systems</a>
</li>
<li data-id="401">
<a href="/patterns/" class="button related__topic button__small button__black">Patterns</a>
</li>
<li data-id="17182">
<a href="/sre/" class="button related__topic button__small button__black">Site Reliability Engineering</a>
</li>
<li data-id="1389">
<a href="/architecture/" class="button related__topic button__small button__black">Architecture</a>
</li>
</ul>
</div>
</div>
<script type="text/javascript">
$(document).ready(function() {
$.ajax({
url: "/api/recommendationlinks.action",
contentType: "application/x-www-form-urlencoded; charset=utf-8",
type: 'POST',
data: {
"primaryTopicAlias": "architecture-design",
"topicIds": "1964,435,5774",
"title": "How to Avoid Cascading Failures in Distributed Systems",
"contentPath": "/articles/anatomy-cascading-failure",
"language": "en"
},
success: displayRelatedEditorial,
async: false
});
});
function displayRelatedEditorial(data) {
$('.related__editorial h4').text("Related Editorial");
if (data && data.length > 0) {
if(data[0].fromEs) {
//change title and tracking params
var box_title="Architecture &amp; Design";
//replace html entity since it conflicts with style
box_title=box_title.replace("&amp;","&");
$('.related__editorial h4').text("Popular in " + box_title);
}
for (var i = 0; i < data.length; i++) {
if (i === 5) {
break;
}
if (data[i].url.indexOf("/articles/anatomy-cascading-failure") !== -1) {
console.log("Removing the current item from list...");
continue;
}
var theLinkURL = data[i].url;
if(!theLinkURL.endsWith("/")) {
theLinkURL = theLinkURL + "/";
}
var link = $('<li><h5 class="rvc__title"><a title="" href="' + theLinkURL + '/">' + data[i].title + '</a></h5></li>');
$('.related__editorial ul').append(link);
}
$('.related__editorial').show();
displayWidget = true;
}else{
$('.related__editorial').parent("li").remove();
}
if(displayWidget==true){
$('.related__group').attr("data-cols", $('.related__group').find(">li").length);
$('.related__group').css("display", "flex");
}
}
</script>
<ul class="no-style related__group nocontent cards">
<li>
<div class="related__editorial">
<h4 class="heading">Related Editorial</h4>
<ul></ul>
</div>
</li>
<li class="bg-gray is-movable">
<div class="related__prsp">
<h4 class="heading">Popular across InfoQ</h4>
<ul>
<li>
<h5 class="rvc__title">
<a href="/news/2026/09/netflix-flink-autoscaler/" class="rvc__link">
Netflix Moves toward Open Source Flink Autoscaler for 30,000+ Streaming Jobs
</a>
</h5>
</li>
<li>
<h5 class="rvc__title">
<a href="/news/2026/09/java-news-roundup-aug31-2026/" class="rvc__link">
Java News Roundup: TornadoVM 6, JReleaser, LangChain4j, Java Operator SDK, JHipster, Yupiik Fusion
</a>
</h5>
</li>
<li>
<h5 class="rvc__title">
<a href="/news/2026/09/google-mantis-vulnerability-scan/" class="rvc__link">
Google Mantis: an Agentic Vulnerability Scanning Harness for Reducing False Positives
</a>
</h5>
</li>
<li>
<h5 class="rvc__title">
<a href="/news/2026/09/blume-docs-ai/" class="rvc__link">
Blume: Zero-Config Docs Framework That Turns a Markdown Folder into an AI-Ready Website
</a>
</h5>
</li>
<li>
<h5 class="rvc__title">
<a href="/news/2026/09/cern-debian-infra/" class="rvc__link">
CERN Renounces RHEL in Favor of Debian for its Accelerator Controls Infrastructure
</a>
</h5>
</li>
<li>
<h5 class="rvc__title">
<a href="/podcasts/personality-over-skillset/" class="rvc__link">
Personality Over Skillset: How Adam Wachtel Builds Engineering Teams
</a>
</h5>
</li>
</ul>
</div>
</li>
</ul>
</div>
<div class="article__more f_article_rightbar" data-col="2/6">
<script type="text/javascript">
window.finishedRightbarVcr = false;
var _gaq = _gaq || [];
var recomJson ="[{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1789136220000,&quot;url&quot;:&quot;/news/2026/09/netflix-conductor-4-workflow&quot;,&quot;title&quot;:&quot;Netflix Reworks Conductor for 420 Million Monthly Workflow Executions and 10X Larger Workflows&quot;,&quot;authorsList&quot;:[&quot;Leela Kumili&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/netflix-conductor-4-workflow&quot;,&quot;score&quot;:20,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1788789960000,&quot;url&quot;:&quot;/news/2026/09/netflix-flink-autoscaler&quot;,&quot;title&quot;:&quot;Netflix Moves toward Open Source Flink Autoscaler for 30,000+ Streaming Jobs&quot;,&quot;authorsList&quot;:[&quot;Leela Kumili&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/netflix-flink-autoscaler&quot;,&quot;score&quot;:19,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1788532200000,&quot;url&quot;:&quot;/news/2026/09/airbnb-server-driven-login&quot;,&quot;title&quot;:&quot;Airbnb Cuts Authentication Code by 60% with Server Driven Architecture&quot;,&quot;authorsList&quot;:[&quot;Leela Kumili&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/airbnb-server-driven-login&quot;,&quot;score&quot;:18,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1787927400000,&quot;url&quot;:&quot;/news/2026/08/uber-gitfarm-git-as-a-service&quot;,&quot;title&quot;:&quot;Uber Builds GitFarm to Run Git Operations as a Service for Large-Scale Monorepos&quot;,&quot;authorsList&quot;:[&quot;Leela Kumili&quot;],&quot;itemPath&quot;:&quot;/news/2026/08/uber-gitfarm-git-as-a-service&quot;,&quot;score&quot;:17,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1787206800000,&quot;url&quot;:&quot;/news/2026/08/harper-vercel-benchmark&quot;,&quot;title&quot;:&quot;Harper Argues against the Multi-System Stack and Releases 5.2&quot;,&quot;authorsList&quot;:[&quot;Renato Losio&quot;],&quot;itemPath&quot;:&quot;/news/2026/08/harper-vercel-benchmark&quot;,&quot;score&quot;:16,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/presentations/netflix-commerce-architecture-evolution/en/smallimage/kasia-trapszo-thumbnail-1787217920531.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1787914800000,&quot;url&quot;:&quot;/presentations/netflix-commerce-architecture-evolution&quot;,&quot;title&quot;:&quot;From DVDs to Global Streaming: How Netflix’s Commerce Architecture Actually Evolved&quot;,&quot;authorsList&quot;:[&quot;Kasia Trapszo&quot;],&quot;itemPath&quot;:&quot;/presentations/netflix-commerce-architecture-evolution&quot;,&quot;score&quot;:15,&quot;contentType&quot;:&quot;presentations&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1786718520000,&quot;url&quot;:&quot;/news/2026/08/cloudflare-cdnjs-migration&quot;,&quot;title&quot;:&quot;Cloudflare Migrates JavaScript CDN Serving 9B Requests a Day to Its Developer Platform&quot;,&quot;authorsList&quot;:[&quot;Leela Kumili&quot;],&quot;itemPath&quot;:&quot;/news/2026/08/cloudflare-cdnjs-migration&quot;,&quot;score&quot;:15,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/presentations/progressive-collapse-system-resilience/en/smallimage/sam-newman-thumbnail-1785845419904.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1787137200000,&quot;url&quot;:&quot;/presentations/progressive-collapse-system-resilience&quot;,&quot;title&quot;:&quot;Understanding Progressive Collapse: How To Avoid A Cascading Failure&quot;,&quot;authorsList&quot;:[&quot;Sam Newman&quot;],&quot;itemPath&quot;:&quot;/presentations/progressive-collapse-system-resilience&quot;,&quot;score&quot;:14,&quot;contentType&quot;:&quot;presentations&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/presentations/disaggregation-industrial-systems/en/smallimage/murat-demirbas-thumbnail-1784809202010.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1785419400000,&quot;url&quot;:&quot;/presentations/disaggregation-industrial-systems&quot;,&quot;title&quot;:&quot;Parting the Clouds: the Rise of Disaggregated Systems&quot;,&quot;authorsList&quot;:[&quot;Murat Demirbas&quot;],&quot;itemPath&quot;:&quot;/presentations/disaggregation-industrial-systems&quot;,&quot;score&quot;:13,&quot;contentType&quot;:&quot;presentations&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/articles/platform-reliability-cycle/en/smallimage/platform-reliability-cycle-thumbnail-1777467114542.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1777971600000,&quot;url&quot;:&quot;/articles/platform-reliability-cycle&quot;,&quot;title&quot;:&quot;Three Pillars of Platform Engineering: a Virtuous Cycle&quot;,&quot;authorsList&quot;:[&quot;Pratik Agarwal&quot;],&quot;itemPath&quot;:&quot;/articles/platform-reliability-cycle&quot;,&quot;score&quot;:11,&quot;contentType&quot;:&quot;articles&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/articles/configuration-control-plane/en/smallimage/configuration-as-a-control-plane-designing-for-safety-and-reliability-at-scale-thumb-1773657574566.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1773997200000,&quot;url&quot;:&quot;/articles/configuration-control-plane&quot;,&quot;title&quot;:&quot;Configuration as a Control Plane: Designing for Safety and Reliability at Scale&quot;,&quot;authorsList&quot;:[&quot;Karthiek Maralla&quot;],&quot;itemPath&quot;:&quot;/articles/configuration-control-plane&quot;,&quot;score&quot;:11,&quot;contentType&quot;:&quot;articles&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/articles/adaptive-hedged-requests-p99-latency/en/smallimage/thumbnail-1779785816730.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1779958800000,&quot;url&quot;:&quot;/articles/adaptive-hedged-requests-p99-latency&quot;,&quot;title&quot;:&quot;Stragglers, Not Failures: How Adaptive Hedged Requests Reduce p99 Latency by 74 Percent&quot;,&quot;authorsList&quot;:[&quot;Prathamesh Bhope&quot;],&quot;itemPath&quot;:&quot;/articles/adaptive-hedged-requests-p99-latency&quot;,&quot;score&quot;:10,&quot;contentType&quot;:&quot;articles&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/presentations/latency-techniques/en/smallimage/amir-langer-thumbnail-1774444318217.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1775831280000,&quot;url&quot;:&quot;/presentations/latency-techniques&quot;,&quot;title&quot;:&quot;Latency: the Race to Zero...Are We There Yet?&quot;,&quot;authorsList&quot;:[&quot;Amir Langer&quot;],&quot;itemPath&quot;:&quot;/presentations/latency-techniques&quot;,&quot;score&quot;:10,&quot;contentType&quot;:&quot;presentations&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/articles/replacing-database-sequences/en/smallimage/replacing-database-sequences-thumbnail-1774603762739.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1775206800000,&quot;url&quot;:&quot;/articles/replacing-database-sequences&quot;,&quot;title&quot;:&quot;Replacing Database Sequences at Scale without Breaking 100+ Services&quot;,&quot;authorsList&quot;:[&quot;Saumya Tyagi&quot;],&quot;itemPath&quot;:&quot;/articles/replacing-database-sequences&quot;,&quot;score&quot;:10,&quot;contentType&quot;:&quot;articles&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/articles/event-driven-banking-architecture/en/smallimage/event-driven-banking-architecture-thumbnail-1774430827143.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1774947600000,&quot;url&quot;:&quot;/articles/event-driven-banking-architecture&quot;,&quot;title&quot;:&quot;Event-Driven Patterns for Cloud-Native Banking: Lessons from What Works and What Hurts&quot;,&quot;authorsList&quot;:[&quot;Chris Tacey-Green&quot;],&quot;itemPath&quot;:&quot;/articles/event-driven-banking-architecture&quot;,&quot;score&quot;:10,&quot;contentType&quot;:&quot;articles&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/articles/chaos-engineering-ecs-payments/en/smallimage/chaos-engineering-ecs-payments-thumbnail-1788254676411.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1788858000000,&quot;url&quot;:&quot;/articles/chaos-engineering-ecs-payments&quot;,&quot;title&quot;:&quot;Implementing Chaos Engineering in Financial Payment Systems: Lessons from Enterprise ECS Deployments&quot;,&quot;authorsList&quot;:[&quot;Salim Adedeji&quot;],&quot;itemPath&quot;:&quot;/articles/chaos-engineering-ecs-payments&quot;,&quot;score&quot;:1,&quot;contentType&quot;:&quot;articles&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/presentations/claude-sre-incidents/en/smallimage/alex-palcuie-thumbnail-1786539336984.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1787742000000,&quot;url&quot;:&quot;/presentations/claude-sre-incidents&quot;,&quot;title&quot;:&quot;Can Claude Fix Itself? Using LLMs for Incident Response&quot;,&quot;authorsList&quot;:[&quot;Alex Palcuie&quot;],&quot;itemPath&quot;:&quot;/presentations/claude-sre-incidents&quot;,&quot;score&quot;:1,&quot;contentType&quot;:&quot;presentations&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1787313600000,&quot;url&quot;:&quot;/news/2026/08/cloudflare-ai-enforcement&quot;,&quot;title&quot;:&quot;Cloudflare Turns Engineering Standards into an AI-Enforced Control System&quot;,&quot;authorsList&quot;:[&quot;Craig Risi&quot;],&quot;itemPath&quot;:&quot;/news/2026/08/cloudflare-ai-enforcement&quot;,&quot;score&quot;:1,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/presentations/ai-sre-incident-response/en/smallimage/infoq-live-thumbnail-1777368469045.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1777375260000,&quot;url&quot;:&quot;/presentations/ai-sre-incident-response&quot;,&quot;title&quot;:&quot;AI-Powered SRE for Autonomous Incident Response&quot;,&quot;authorsList&quot;:[&quot;Rohit Dhawan&quot;,&quot;Pavan Madduri&quot;,&quot;Alina Astapovich&quot;,&quot;Goutham Rao&quot;,&quot;Renato Losio&quot;],&quot;itemPath&quot;:&quot;/presentations/ai-sre-incident-response&quot;,&quot;score&quot;:1,&quot;contentType&quot;:&quot;presentations&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/presentations/outage-lessons/en/smallimage/molly-struve-thumbnail-1776864399990.jpeg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1777371180000,&quot;url&quot;:&quot;/presentations/outage-lessons&quot;,&quot;title&quot;:&quot;Week-Long Outage: Lifelong Lessons&quot;,&quot;authorsList&quot;:[&quot;Molly Struve&quot;],&quot;itemPath&quot;:&quot;/presentations/outage-lessons&quot;,&quot;score&quot;:1,&quot;contentType&quot;:&quot;presentations&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/articles/sovereign-fault-domains-cloud-resilience/en/smallimage/sovereign-fault-domains-cloud-resilience-thumbnail-1776430533702.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1776848400000,&quot;url&quot;:&quot;/articles/sovereign-fault-domains-cloud-resilience&quot;,&quot;title&quot;:&quot;When a Cloud Region Fails: Rethinking High Availability in a Geopolitically Unstable World&quot;,&quot;authorsList&quot;:[&quot;Rohan Vardhan&quot;],&quot;itemPath&quot;:&quot;/articles/sovereign-fault-domains-cloud-resilience&quot;,&quot;score&quot;:1,&quot;contentType&quot;:&quot;articles&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1789138800000,&quot;url&quot;:&quot;/news/2026/09/nvidia-pair-ai-task-router&quot;,&quot;title&quot;:&quot;NVIDIA Personal AI Router Distributes AI Tasks across Local Compute&quot;,&quot;authorsList&quot;:[&quot;Sergio De Simone&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/nvidia-pair-ai-task-router&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1789128120000,&quot;url&quot;:&quot;/news/2026/09/tsgolint-oxlint-typescript&quot;,&quot;title&quot;:&quot;tsgolint Reaches Stable v7, Bringing Go-Powered Type-Aware Linting to Oxlint&quot;,&quot;authorsList&quot;:[&quot;Daniel Curtis&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/tsgolint-oxlint-typescript&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1789128000000,&quot;url&quot;:&quot;/news/2026/09/terraform-aws-provider-6-62&quot;,&quot;title&quot;:&quot;Terraform AWS Provider Continues Rapid Expansion as AWS Infrastructure Becomes More Complex&quot;,&quot;authorsList&quot;:[&quot;Craig Risi&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/terraform-aws-provider-6-62&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/presentations/form3-multicloud-architecture/en/smallimage/ross-mcfarlane-kevin-holditch-thumbnail-1788338652847.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1789124400000,&quot;url&quot;:&quot;/presentations/form3-multicloud-architecture&quot;,&quot;title&quot;:&quot;How To Run on Three Clouds at Once, and When Not To&quot;,&quot;authorsList&quot;:[&quot;Ross McFarlane&quot;,&quot;Kevin Holditch&quot;],&quot;itemPath&quot;:&quot;/presentations/form3-multicloud-architecture&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;presentations&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1789120800000,&quot;url&quot;:&quot;/news/2026/09/linkedin-ai-multi-teacher&quot;,&quot;title&quot;:&quot;How LinkedIn Trains AI Job Search 8x Faster with Multi-Teacher Distillation&quot;,&quot;authorsList&quot;:[&quot;Claudio Masolo&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/linkedin-ai-multi-teacher&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1789114440000,&quot;url&quot;:&quot;/news/2026/09/observability-ai-agents&quot;,&quot;title&quot;:&quot;Session Traces and Cost Controls Help Diagnose AI Agent Failures&quot;,&quot;authorsList&quot;:[&quot;Mark Silvester&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/observability-ai-agents&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1789103100000,&quot;url&quot;:&quot;/news/2026/09/tinygo-devkit&quot;,&quot;title&quot;:&quot;Advancing Embedded Go: Recoverable Panics, UEFI, Radio and Hardware Dev Kit&quot;,&quot;authorsList&quot;:[&quot;Olimpiu Pop&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/tinygo-devkit&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1789062540000,&quot;url&quot;:&quot;/news/2026/09/openai-gpt6-astra&quot;,&quot;title&quot;:&quot;OpenAI Releases GPT-6 Astra for Coding and Computer Use&quot;,&quot;authorsList&quot;:[&quot;Daniel Dominguez&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/openai-gpt6-astra&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1789036140000,&quot;url&quot;:&quot;/news/2026/09/autonomous-software-teams&quot;,&quot;title&quot;:&quot;Beyond Autonomous Teams in Software Product Development&quot;,&quot;authorsList&quot;:[&quot;Ben Linders&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/autonomous-software-teams&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/presentations/rust-refactoring/en/smallimage/lily-mara-thumbnail-1788338447565.jpeg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1789032840000,&quot;url&quot;:&quot;/presentations/rust-refactoring&quot;,&quot;title&quot;:&quot;Accelerating Performance by Incrementally Integrating Rust Into Existing Codebase&quot;,&quot;authorsList&quot;:[&quot;Lily Mara&quot;],&quot;itemPath&quot;:&quot;/presentations/rust-refactoring&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;presentations&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/articles/when-spec-driven-development-pays-off/en/smallimage/When-Spec-Driven-Development-Pays-Off-thumb-1788790605567.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1789030800000,&quot;url&quot;:&quot;/articles/when-spec-driven-development-pays-off&quot;,&quot;title&quot;:&quot;When Spec-Driven Development Pays Off&quot;,&quot;authorsList&quot;:[&quot;Nitin Garg&quot;],&quot;itemPath&quot;:&quot;/articles/when-spec-driven-development-pays-off&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;articles&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1789021620000,&quot;url&quot;:&quot;/news/2026/09/http-query-method&quot;,&quot;title&quot;:&quot;IETF Publishes RFC 10008, Adding the QUERY Method for Safe Requests with a Body&quot;,&quot;authorsList&quot;:[&quot;Daniel Curtis&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/http-query-method&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1789016700000,&quot;url&quot;:&quot;/news/2026/09/riscv-cpython&quot;,&quot;title&quot;:&quot;CPython Officially Adds RISC-V Support as a Tier 3 Platform&quot;,&quot;authorsList&quot;:[&quot;Olimpiu Pop&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/riscv-cpython&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1788976800000,&quot;url&quot;:&quot;/news/2026/09/meta-organizational-agents&quot;,&quot;title&quot;:&quot;Meta's Recipe for Building Agents as \\\&quot;Organizational Second Brains\\\&quot;&quot;,&quot;authorsList&quot;:[&quot;Sergio De Simone&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/meta-organizational-agents&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/presentations/unikraft-microvm-sandboxes-cloud-scaling/en/smallimage/felipe-huici-thumbnail-1787813506607.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1788951600000,&quot;url&quot;:&quot;/presentations/unikraft-microvm-sandboxes-cloud-scaling&quot;,&quot;title&quot;:&quot;Fixing the AI Infra Scale Problem by Stuffing 1M Sandboxes in a Single Server&quot;,&quot;authorsList&quot;:[&quot;Felipe Huici&quot;],&quot;itemPath&quot;:&quot;/presentations/unikraft-microvm-sandboxes-cloud-scaling&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;presentations&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1788944040000,&quot;url&quot;:&quot;/news/2026/09/avd-hybrid-ga&quot;,&quot;title&quot;:&quot;Azure Virtual Desktop Hybrid Reaches GA with Licensing Details Unpublished&quot;,&quot;authorsList&quot;:[&quot;Steef-Jan Wiggers&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/avd-hybrid-ga&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1788933960000,&quot;url&quot;:&quot;/news/2026/09/async-lua-neovim&quot;,&quot;title&quot;:&quot;vim.async's Addition Modernizes Neovim’s Async Architecture for Better Stability&quot;,&quot;authorsList&quot;:[&quot;Olimpiu Pop&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/async-lua-neovim&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;news&quot;},{&quot;imageStoragePath&quot;:&quot;https://imgopt.infoq.com/fit-in/50x50/filters:quality(80)/presentations/ai-platform-engineering-roundtable/en/smallimage/thumb-1788861349861.jpg&quot;,&quot;topicsIds&quot;:null,&quot;date&quot;:1788876000000,&quot;url&quot;:&quot;/presentations/ai-platform-engineering-roundtable&quot;,&quot;title&quot;:&quot;Platform Engineering in the Age of AI&quot;,&quot;authorsList&quot;:[&quot;St&#xE9;phane Di Cesare&quot;,&quot;Davide de Paolis&quot;,&quot;Stephen Cihak&quot;,&quot;Camila Macedo&quot;,&quot;Renato Losio&quot;],&quot;itemPath&quot;:&quot;/presentations/ai-platform-engineering-roundtable&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;presentations&quot;},{&quot;imageStoragePath&quot;:null,&quot;topicsIds&quot;:null,&quot;date&quot;:1788876000000,&quot;url&quot;:&quot;/news/2026/09/hashicorp-packer-verification&quot;,&quot;title&quot;:&quot;HashiCorp Packer 1.16 Adds Native SLSA Provenance Generation and Verification for Machine Images&quot;,&quot;authorsList&quot;:[&quot;Claudio Masolo&quot;],&quot;itemPath&quot;:&quot;/news/2026/09/hashicorp-packer-verification&quot;,&quot;score&quot;:0,&quot;contentType&quot;:&quot;news&quot;}]";
var whitepaperVcrsJson = null;
var topicSponsorshipJson = "{&quot;iconLink&quot;:&quot;/url/f/e7ff8a76-884f-40bc-b88b-f0845d30d60d/&quot;,&quot;iconHref&quot;:&quot;https://imgopt.infoq.com//fit-in/275x500/filters:quality(100)/filters:no_upscale()/sponsorship/topic/12605306-ee86-4a5c-bc22-ab8b965bfc4a/ModerneLogoMicrosite-1786453474094.png&quot;,&quot;id&quot;:&quot;4e2ffe12-08ae-48af-a982-1979fc55b0ab&quot;}";
var vcrOptionalListJson = null;
/* do not delete these two, as they are used further in the code */
var contentDatetimeFormat='MMM dd, yyyy';
var contentUriMapping="articles";
JSi18n.relatedRightbar_relatedContent='Related Content';
JSi18n.relatedRightbar_sponsoredContent='Related Sponsors';
JSi18n.relatedRightbar_sponsoredBy='Sponsored by';
var topicIds = "435,1964,397,5774,876";
var communityIds = "2498,1893";
var company = "Moderne";
// this event is fired by frontend once all the necessary things have been done(mobile display, moving vcr boxes around when needed...)
var canStartTrackingCustomRightbar = false;
infoq.event.on('loaded', function(e) {
canStartTrackingCustomRightbar = true;
});
var intervalRightbar = setInterval(function() {
if (window.vcrsLoaded) {
clearInterval(intervalRightbar);
if(company != null && company != "") {
whitepaperVcrsJson = VCR.filterByCompany(company, window.vcrList);
} else {
whitepaperVcrsJson = VCR.getByTopicsAndCommunities(window.vcrList, topicIds, communityIds, 5, false, null);
}
vcrOptionalListJson = VCR.getByTopicsAndCommunities(window.vcrList, topicIds, communityIds, 10, true, null);
VCR.displayCustomRightbar(recomJson, whitepaperVcrsJson, topicSponsorshipJson);
VCR.displayCustomRightbarOptionalVcrWidget(vcrOptionalListJson);
window.finishedRightbarVcr = true;
}
}, 200);
// these two events can happen one before another async(no precedence any can be first or second). Make sure tracking starts when both happened
var intervalTrackingRightbar = setInterval(function() {
if(canStartTrackingCustomRightbar && window.finishedRightbarVcr){
clearInterval(intervalTrackingRightbar);
VCR.doTrackingCustomRightbar();
}
}, 200);
</script>
<noscript>
<div class="widget related__content article__widget">
<h3 class="widget__heading">Related Content</h3>
<ul class="no-style cards" data-horizontal="true" data-size="xs" data-tax="">
</ul>
</div>
</noscript>
<div class="newsletter widget" data-bg="infoq">
<h3 class="heading"><strong>The InfoQ</strong> Newsletter</h3>
<p class="intro">
A round-up of last week&#x2019;s content on InfoQ sent out every Tuesday. Join a community of over 250,000 senior developers.
<a target="_blank" href="https://assets.infoq.com/newsletter/regular/en/newsletter_sample/newsletter_sample.html">View an example</a>
</p>
<div class="newsletter__subscribe">
<form class="form gdpr" id="floatingNewsletterForm" action="#" onsubmit="floatingNewsletterForm.saveSubscription(); return false;">
<div class="field input__text input__no-label input__medium newsletter__mail email">
<label class="field__label label" for="email-newsletter-infoq-guide">Enter your e-mail address</label>
<input class="field__input input" id="email-newsletter-infoq-guide" placeholder="Enter your e-mail address" type="email">
<input type="text" name="emailH" id="input_floating_email_h" aria-required="false" style="display:none !important" tabindex="-1" autocomplete="off">
<input type="hidden" id="floating_fnt" name="fnt" value="pgarAk5gUTjmy6ja"/>
<input type="hidden" id="floatingNewsletterType" name="footerNewsletterType" value="regular"/>
<input type="hidden" id="cmpi_f" name="cmpi" value="1"/>
</div>
<div class="hidden">
<span class="input__select field country">
<label for="input-floating-newsletter-country" class="label field__label">Select your country</label>
<select id="input-floating-newsletter-country" class="select field__input">
<option value="" class="select__option">Select a country</option>
</select>
<p class="input__message field__desc"></p>
</span>
<span class="input__checkbox field hidden">
<input type="checkbox" id="gdpr-consent-simple-floating-nl">
<label for="gdpr-consent-simple-floating-nl" class="label"><span>I consent to InfoQ.com handling my data as explained in this <a href="https://www.infoq.com/privacy-notice">Privacy Notice</a>.</span></label>
</span>
</div>
<input class="button button__medium button__yellow" type="submit" role="button" value="Subscribe" onclick="return floatingNewsletterForm.validateEmail('Invalid email address');">
</form>
<p class="meta">
<a href="/privacy-notice/" target="_blank">We protect your privacy.</a>
</p>
<span class="success" style="display:none;" id="floatingNewsletterEmailMessage"></span>
</div>
</div>
<script type="text/javascript">
var floatingNewsletterForm = new Newsletter('Enter your e-mail address',
'email-newsletter-infoq-guide', 'floatingNewsletterType','floatingNewsletterEmailMessage', 'floating_fnt', 'input_floating_email_h', 'input-floating-newsletter-country','cmpi_f','floating_box');
</script>
</div>
</div>
</div>
</section>
</article>
<script type="text/javascript">
// global vars that can be used for this page, us ethis section to add more.
var contentTitle = "How to Avoid Cascading Failures in Distributed Systems",
contentPath = "/articles/anatomy-cascading-failure",
contentUUID = "ebd03567-06f7-47f9-8bd0-794801fbec6f",
authorUserCSVIds = "126398076";
</script>
<script src="https://cdn.infoq.com/statics_s2_20260910111657/scripts/prism-build.js"></script>
<script src="https://cdn.infoq.com/statics_s2_20260910111657/scripts/lib/MathJax/MathJax.js?config=TeX-AMS_HTML"></script>
</main>
<footer class="footer ">
<section data-nosnippet class="section container">
<div class="container__inner">
<ul data-cols="5" class="no-style columns boxes topic__boxes">
<li class="development">
<div class="box__header">
<a class="t_all_footer_more-boxes-header" href="/development/">Development</a>
</div>
<div class="box__content">
<ul class="no-style box__list small">
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/tinygo-devkit/" title="Advancing Embedded Go: Recoverable Panics, UEFI, Radio and Hardware Dev Kit">Advancing Embedded Go: Recoverable Panics, UEFI, Radio and Hardware Dev Kit</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/riscv-cpython/" title="CPython Officially Adds RISC-V Support as a Tier 3 Platform">CPython Officially Adds RISC-V Support as a Tier 3 Platform</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/async-lua-neovim/" title="vim.async&#039;s Addition Modernizes Neovim&amp;rsquo;s Async Architecture for Better Stability">vim.async's Addition Modernizes Neovim&rsquo;s Async Architecture for Better Stability</a></h5>
</li>
</ul>
</div>
</li>
<li class="architecture-design">
<div class="box__header">
<a class="t_all_footer_more-boxes-header" href="/architecture-design/">Architecture &amp; Design</a>
</div>
<div class="box__content">
<ul class="no-style box__list small">
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/netflix-conductor-4-workflow/" title="Netflix Reworks Conductor for 420 Million Monthly Workflow Executions and 10X Larger Workflows">Netflix Reworks Conductor for 420 Million Monthly Workflow Executions and 10X Larger Workflows</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/meta-organizational-agents/" title="Meta&#039;s Recipe for Building Agents as &amp;quot;Organizational Second Brains&amp;quot;">Meta's Recipe for Building Agents as &quot;Organizational Second Brains&quot;</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/netflix-flink-autoscaler/" title="Netflix Moves toward Open Source Flink Autoscaler for 30,000+ Streaming Jobs">Netflix Moves toward Open Source Flink Autoscaler for 30,000+ Streaming Jobs</a></h5>
</li>
</ul>
</div>
</li>
<li class="culture-methods">
<div class="box__header">
<a class="t_all_footer_more-boxes-header" href="/culture-methods/">Culture &amp; Methods</a>
</div>
<div class="box__content">
<ul class="no-style box__list small">
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/autonomous-software-teams/" title="Beyond Autonomous Teams in Software Product Development">Beyond Autonomous Teams in Software Product Development</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/presentations/rust-refactoring/" title="Accelerating Performance by Incrementally Integrating Rust Into Existing Codebase">Accelerating Performance by Incrementally Integrating Rust Into Existing Codebase</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/podcasts/personality-over-skillset/" title="Personality Over Skillset: How Adam Wachtel Builds Engineering Teams">Personality Over Skillset: How Adam Wachtel Builds Engineering Teams</a></h5>
</li>
</ul>
</div>
</li>
<li class="ai-ml-data-eng">
<div class="box__header">
<a class="t_all_footer_more-boxes-header" href="/ai-ml-data-eng/">AI, ML &amp; Data Engineering</a>
</div>
<div class="box__content">
<ul class="no-style box__list small">
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/nvidia-pair-ai-task-router/" title="NVIDIA Personal AI Router Distributes AI Tasks across Local Compute">NVIDIA Personal AI Router Distributes AI Tasks across Local Compute</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/linkedin-ai-multi-teacher/" title="How LinkedIn Trains AI Job Search 8x Faster with Multi-Teacher Distillation">How LinkedIn Trains AI Job Search 8x Faster with Multi-Teacher Distillation</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/openai-gpt6-astra/" title="OpenAI Releases GPT-6 Astra for Coding and Computer Use">OpenAI Releases GPT-6 Astra for Coding and Computer Use</a></h5>
</li>
</ul>
</div>
</li>
<li class="devops">
<div class="box__header">
<a class="t_all_footer_more-boxes-header" href="/devops/">DevOps</a>
</div>
<div class="box__content">
<ul class="no-style box__list small">
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/terraform-aws-provider-6-62/" title="Terraform AWS Provider Continues Rapid Expansion as AWS Infrastructure Becomes More Complex">Terraform AWS Provider Continues Rapid Expansion as AWS Infrastructure Becomes More Complex</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/observability-ai-agents/" title="Session Traces and Cost Controls Help Diagnose AI Agent Failures">Session Traces and Cost Controls Help Diagnose AI Agent Failures</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/hashicorp-packer-verification/" title="HashiCorp Packer 1.16 Adds Native SLSA Provenance Generation and Verification for Machine Images">HashiCorp Packer 1.16 Adds Native SLSA Provenance Generation and Verification for Machine Images</a></h5>
</li>
</ul>
</div>
</li>
</ul>
</div>
</section>
<section data-nosnippet class="container section section__newsletter">
<div class="columns container__inner">
<div class="newsletter" data-col="1/2"
id="infoq-nl"
>
<h2 class="heading">
<strong>The InfoQ</strong> Newsletter
</h2>
<p class="intro">A round-up of last week&#x2019;s content on InfoQ sent out every Tuesday. Join a community of over 250,000 senior developers.
<a target="_blank" href="https://assets.infoq.com/newsletter/regular/en/newsletter_sample/newsletter_sample.html">View an example</a>
</p>
<ul class="newsletter__features">
<li>Get a quick overview of content published on a variety of innovator and early adopter technologies</li>
<li>Learn what you don&#x2019;t know that you don&#x2019;t know</li>
<li>Stay up to date with the latest information from the topics you are interested in</li>
</ul>
<div class="newsletter__subscribe">
<form class="form gdpr" name="footerNewsletterForm" id="footerNewsletterForm" action="#" onsubmit="footerNewsletter.saveSubscription(); return false;">
<div class="field newsletter__mail input__text input__no-label input__medium email">
<label for="email-newsletter-infoq" class="label field__label">Enter your e-mail address</label>
<input id="email-newsletter-infoq" name="footerNewsletterEmail" placeholder="Enter your e-mail address" class="input field__input" type="email"/>
<input type="text" name="emailH" id="input_email_h" aria-required="false" style="display:none !important" tabindex="-1" autocomplete="off"/>
<input type="hidden" id="fnt" name="fnt" value="pgarAk5gUTjmy6ja"/>
<input type="hidden" id="footerNewsletterType" name="footerNewsletterType" value="regular"/>
<input type="hidden" id="cmpi" name="cmpi" value="1"/>
</div>
<div class="hidden">
<span class="input__select field country">
<label for="input-simple-newsletter-country" class="label field__label">Select your country</label>
<select id="input-simple-newsletter-country" class="select field__input">
<option value="" class="select__option">Select a country</option>
</select>
<p class="input__message field__desc"></p>
</span>
<span class="input__checkbox field hidden">
<input type="checkbox" id="gdpr-consent-simple-nl">
<label for="gdpr-consent-simple-nl" class="label"><span>I consent to InfoQ.com handling my data as explained in this <a href="https://www.infoq.com/privacy-notice">Privacy Notice</a>.</span></label>
</span>
</div>
<input type="submit" role="button" value="Subscribe" class="button button__medium button__red" onclick="return footerNewsletter.validateEmail('Invalid email address');"/>
</form>
<p class="meta">
<a href="/privacy-notice/" target="_blank">We protect your privacy.</a>
</p>
<span class="success" style="display:none;" id="footerNewsletterMessage"></span>
</div>
<script type="text/javascript">
var footerNewsletter = new Newsletter('Enter your e-mail address',
'email-newsletter-infoq', 'footerNewsletterType','footerNewsletterMessage', 'fnt', 'input_email_h', 'input-simple-newsletter-country', 'cmpi','footer_except_homepage');
</script>
</div>
<div data-col="1/2" data-bg="certification" class="event__container" style="padding-right: 20px;">
<a href="https://certification.qconferences.com/?utm_source=infoq&amp;utm_medium=referral&amp;utm_campaign=largefooterad_onlinecohorts26" target="_blank" class="qconplus__events-slider" style="">
<div>
<div class="logo-certification"></div>
</div>
<p class="intro" style="margin-bottom: 0;color: white !important;">
<b>InfoQ Online Certification Programs</b>
<br>For Senior Engineers, Architects, and Technical Leaders
</p>
<ul style="margin-bottom: 0;">
<li style="color: white !important;"><b>Architect</b> with Luca Mezzalira | September 14</li>
<li style="color: white !important;"><b>Engineering Leadership</b> with Michelle Brush | September 18</li>
<li style="color: white !important;"><b>AI-Assisted Engineering</b> with Zichuan Xiong & Premanand Chandrasekaran | September 18</li>
<li style="color: white !important;"><b>AI Security &amp; Privacy Engineering</b> with Katharine Jarmul | October 14</li>
</ul>
<br>
<p class="intro" style="margin-bottom: 0;color: white !important;">As your role becomes more senior, the work changes. You're no longer just implementing decisions; you're shaping the systems, trade-offs, and technical direction other teams depend on.<br>These 5-week online programs give you a structured way to work through real decisions from your current role with senior peers from other companies.</p>
<strong class="button button__green cta__button">RESERVE YOUR PLACE</strong>
<small style="color: white;"><br>Live online. 4 hours a week, for 5 weeks.</small>
</a>
</div>
</div>
</section>
<section data-nosnippet class="section container footer__subfooter align__left">
<div class="container__inner columns">
<div data-col="1/4" class="columns footer__links-wrap">
<div class="footer__links">
<a href="/" class="nuxt-link-exact-active active" title="Home">Home</a>
<a href="/reginit.action" class="emphasis" title="Create account">Create account</a>
<button aria-label="Log In" class="login" data-ref="mainLogin">Log In</button>
<a rel="noreferrer noopener" href="http://qconferences.com/" target="_blank" title="QCon Conferences">QCon Conferences</a>
<a rel="noreferrer noopener" href="https://events.infoq.com/" target="_blank">Events</a>
<a href="/write-for-infoq/" title="Write for InfoQ">Write for InfoQ</a>
<a href="/infoq-editors/" title="InfoQ Editors">InfoQ Editors</a>
<a href="/about-infoq/" title="About InfoQ">About InfoQ</a>
<a href="https://c4media.com/" target="_blank" title="About C4Media">About C4Media</a>
<a rel="noreferrer noopener" href="https://get.infoq.com/infoq-mediakit/" title="Media Kit" target="_blank">
Media Kit
</a>
<a href="https://devmarketing.c4media.com/?utm_source=infoq" title="InfoQ Developer Marketing Blog" target="_blank">InfoQ Developer Marketing Blog</a>
<a rel="noreferrer noopener" href="https://c4media.com/diversity" title="Diversity" target="_blank">Diversity</a>
</div>
</div>
<div data-col="1/4" class="events__list">
<h4 class="heading footer__heading">Events</h4>
<ul class="qcons__list no-style">
<li><span class="icon event__type conference"></span>
<div class="qcon__detail">
<h5 class="heading">
<a rel="noreferrer noopener" href="https://certification.qconferences.com/architecture?utm_source=infoq&utm_medium=referral&utm_campaign=footer_onlinecohortarchitecture26" target="_blank">Online InfoQ Architect Certification</a>
</h5>
<span class="meta date">September 14, 2026</span>
</div>
</li>
<li><span class="icon event__type conference"></span>
<div class="qcon__detail">
<h5 class="heading">
<a rel="noreferrer noopener" href="https://certification.qconferences.com/ai-assisted-engineering?utm_source=infoq&utm_medium=referral&utm_campaign=footer_onlinecohortaiassistedengineering26" target="_blank">Online InfoQ AI-Assisted Engineering Certification</a>
</h5>
<span class="meta date">September 18, 2026</span>
</div>
</li>
<li><span class="icon event__type conference"></span>
<div class="qcon__detail">
<h5 class="heading">
<a rel="noreferrer noopener" href="https://certification.qconferences.com/ai-security-privacy?utm_source=infoq&utm_medium=referral&utm_campaign=footer_onlinecohortaisecurity26" target="_blank">Online InfoQ AI Security & Privacy Engineering Program</a>
</h5>
<span class="meta date">October 14, 2026</span>
</div>
</li>
<li><span class="icon event__type conference"></span>
<div class="qcon__detail">
<h5 class="heading">
<a rel="noreferrer noopener" href="https://qconsf.com/?utm_source=infoq&utm_medium=referral&utm_campaign=footer_qsf26" target="_blank">QCon San Francisco</a>
</h5>
<span class="meta date">November 16-20, 2026</span>
</div>
</li>
<li><span class="icon event__type conference"></span>
<div class="qcon__detail">
<h5 class="heading">
<a rel="noreferrer noopener" href="https://newyork.qcon.ai/?utm_source=infoq&utm_medium=referral&utm_campaign=footer_qainy26" target="_blank">QCon AI New York</a>
</h5>
<span class="meta date">December 15-16, 2026</span>
</div>
</li>
<li><span class="icon event__type conference"></span>
<div class="qcon__detail">
<h5 class="heading">
<a rel="noreferrer noopener" href="https://qconlondon.com/?utm_source=infoq&utm_medium=referral&utm_campaign=footer_qlondon27" target="_blank">QCon London 2027</a>
</h5>
<span class="meta date">April 13-16, 2027</span>
</div>
</li>
</ul>
</div>
<div data-col="1/4" class="footer__social-wrap">
<h4 class="heading text-left footer__heading">Follow us on </h4>
<div class="social__links columns social__links__row">
<a href="https://www.youtube.com/infoq"><span class="icon icon__large icon__social icon__youtube"></span><div><span class="social__count">Youtube</span><span class="social__followers">232K Followers</span></div></a>
<a href="http://www.linkedin.com/company/infoq"><span class="icon icon__large icon__social icon__linkedin"></span><div><span class="social__count">Linkedin</span><span class="social__followers">26K Followers</span></div></a>
<a rel="noreferrer noopener" href="https://www.instagram.com/infoqdotcom/" target="_blank"><span class="icon icon__large icon__social icon__instagram"></span><div><span class="social__count">Instagram</span><span class="social__followers">New</span></div></a>
<a href="#" id="footerNewsletterRssLink"><span class="icon icon__large icon__social icon__rss"></span><div><span class="social__count">RSS</span><span class="social__followers">19K Readers</span></div></a>
<a rel="noreferrer noopener" href="http://twitter.com/infoq" target="_blank"><span class="icon icon__large icon__social icon__twitter"></span><div><span class="social__count">X</span><span class="social__followers">57.1k Followers</span></div></a>
<a rel="noreferrer noopener" href="https://www.facebook.com/InfoQ-75911537320" target="_blank"><span class="icon icon__large icon__social icon__fb"></span><div><span class="social__count">Facebook</span><span class="social__followers">21K Likes</span></div></a>
<a rel="noreferrer noopener" href="https://bsky.app/profile/infoq.com" target="_blank"><span class="icon icon__large icon__social icon__bluesky"></span><div><span class="social__count">Bluesky</span><span class="social__followers">New</span></div></a>
</div>
</div>
<div data-col="1/4" class="footer__stayin-wrap">
<h4 class="heading text-left footer__heading">Stay in the know</h4>
<div class="stayIn_panel_container">
<a href="/podcasts/" class="stayIn_panel" target="_blank" rel="noreferrer noopener"><span>The InfoQ Podcast</span><picture><source srcset="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLXBvZGNhc3QuanBnIiwiZWRpdHMiOiB7IndlYnAiOiB7ICJxdWFsaXR5Ijo4MH19fQ==" type="image/webp"><source srcset="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLXBvZGNhc3QuanBnIiwiZWRpdHMiOiB7ImpwZWciOiB7ICJxdWFsaXR5Ijo4MH19fQ==" type="image/webp"><img src="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLXBvZGNhc3QuanBnIiwiZWRpdHMiOiB7ImpwZWciOiB7ICJxdWFsaXR5Ijo4MH19fQ==" loading="lazy" width="65px" height="64px" alt="The InfoQ Podcast Logo - Stay in the know"></picture></a>
<a href="/podcasts/#engineering_culture" class="stayIn_panel" target="_blank" rel="noreferrer noopener"><span>Engineering Culture Podcast</span><picture><source srcset="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLXBvZGNhc3QtZW5naW5lZXJpbmctY3VsdHVyZS5qcGciLCJlZGl0cyI6IHsid2VicCI6IHsgInF1YWxpdHkiOjgwfX19" type="image/webp"><source srcset="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLXBvZGNhc3QtZW5naW5lZXJpbmctY3VsdHVyZS5qcGciLCJlZGl0cyI6IHsianBlZyI6IHsgInF1YWxpdHkiOjgwfX19" type="image/webp"><img src="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLXBvZGNhc3QtZW5naW5lZXJpbmctY3VsdHVyZS5qcGciLCJlZGl0cyI6IHsianBlZyI6IHsgInF1YWxpdHkiOjgwfX19" loading="lazy" width="65px" height="64px" alt="Engineering Culture Podcast Logo - Stay in the knoww"></picture></a>
<a href="/software-architects-newsletter/" class="stayIn_panel" target="_blank" rel="noreferrer noopener"><span>The Software Architects' Newsletter</span><picture><source srcset="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLWFyY2hpdGVjdC1uZXdzbGV0dGVyLmpwZyIsImVkaXRzIjogeyJ3ZWJwIjogeyAicXVhbGl0eSI6ODB9fX0=" type="image/webp"><source srcset="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLWFyY2hpdGVjdC1uZXdzbGV0dGVyLmpwZyIsImVkaXRzIjogeyJqcGVnIjogeyAicXVhbGl0eSI6ODB9fX0=" type="image/webp"><img src="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLWFyY2hpdGVjdC1uZXdzbGV0dGVyLmpwZyIsImVkaXRzIjogeyJqcGVnIjogeyAicXVhbGl0eSI6ODB9fX0=" loading="lazy" width="65px" height="64px" alt="The Software Architects' Newsletter Logo - Stay in the know"></picture></a>
</div>
</div>
</div>
</section>
<section data-nosnippet class="container footer__bottom section white align__left">
<div class="container__inner columns">
<div data-col="2/3" class="column">
<div class="footer__contact contact columns">
<span data-col="1/6">
General Feedback
<a href="mailto:feedback@infoq.com">feedback@infoq.com</a>
</span>
<span data-col="1/6">
Advertising
<a href="mailto:sales@infoq.com">sales@infoq.com</a>
</span>
<span data-col="1/6">
Editorial
<a href="mailto:editors@infoq.com">editors@infoq.com</a>
</span>
<span data-col="1/6">
Marketing
<a href="mailto:marketing@infoq.com">marketing@infoq.com</a>
</span>
</div>
</div>
<div class="column" data-col="1/3">
<p class="footer__more">
InfoQ.com and all content copyright &#169; 2006-2026 C4Media Inc.<br/>
<a href="/privacy-notice" target="_blank">Privacy Notice</a>, <a href="/terms-and-conditions " target="_blank">Terms And Conditions</a>, <a href="/cookie-policy " target="_blank">Cookie Policy</a>
</p>
</div>
</div>
</section>
</footer>
<!-- ####### SITE END ######### -->
</div>
<div class="intbt">
<a href="/int/bt/" title="bt">BT</a>
</div>
<script type="text/javascript">
$.when(humanDetectionAsync()).then(
function(status) {
$.getScript("/scripts/__hd.ifq?hdt=pgarAk5gUTjmy6ja&ha=" + status);
}
);
</script>
<script type="text/javascript">
var pageFullyLoaded = false;
// this event is fired by frontend once all the necessary things have been done(mobile display, moving vcr boxes around when needed...)
infoq.event.on('loaded', function(e) {
pageFullyLoaded = true;
});
infoq.event.on('pageWidthChanged', function(e) {
// re-execute tracking vcr impressions when this event happens (it only happens when elements are added/removed from page)
// doTrackVcrImpressions takes into account the data-trk-impr="true" if =false the element was already tracked.
// this is needed when switching from mobile to desktop or when layout on mobile changes and desktop version is displayed. New elements become visible.
Tracker.safeExec(Tracker.doTrackVcrImpressions);
// these 2 need to be called also because we might be on a content page. In case we are not nothing happens
VCR.doTrackingCustomRightbar();
VCR.doTrackingCustomRightbarForPresentations()
});
//check to see if error page
if(window.device !== undefined) {
var intervalImpressions = setInterval(function() {
var shouldTrack = false;
// no vcr widgets on index pages
if(window.isIndexPage) {
if(window.sponsoredPodcastDone === undefined || window.sponsoredPodcastDone) {
shouldTrack = true;
}
} else
//on homepage, bottom widget + 2 native widgets
if(InfoQConstants.pageType == "HOMEPAGE" && window.finishedVcrOptional1 && window.finishedRelatedVcr && (window.finishedVcrOptional2 === undefined || window.finishedVcrOptional2)) {
shouldTrack = true;
} else
// rightbar widgets + native widgets + content vcr widgets
if((InfoQConstants.pageType == "NEWS_PAGE" || InfoQConstants.pageType == "ARTICLE_PAGE")
&& ((window.finishedRightbarVcr || window.finishedRightbarVcr === undefined) && window.contentVcrFinished)) {
shouldTrack = true;
} else if ((InfoQConstants.pageType == "PRESENTATION_PAGE")
&& (window.contentVcrFinished || window.contentVcrFinished === undefined)
&& (window.finishedRightbarVcr || window.finishedRightbarVcr === undefined)
) {
shouldTrack = true;
} else
// native widgets + content widgets
if(window.contentVcrFinished && (window.finishedVcrOptional1 || window.finishedVcrOptional1 === undefined)
&& (window.finishedVcrOptional2 || window.finishedVcrOptional2 === undefined)) {
shouldTrack = true;
}
// we start tracking only after the page is fully loaded, frontend signals that they finished everything related to page display.
if(shouldTrack && pageFullyLoaded) {
clearInterval(intervalImpressions);
Tracker.safeExec(Tracker.doTrackVcrImpressions);
// start tracking viewable impressions also only after everything is ready
function callbackRouter(entries, observer) {
var targets = new Array();
entries.forEach(function (entry) {
var target = entry.target;
if (target.dataset.trkView === 'false') return;
if (entry.intersectionRatio > 0) {
target.dataset.trkView = false;
targets.push(target);
}
});
Tracker.doTrackViewableImpressions(targets);
}
var elementsForTrackingViewableImpressions = document.querySelectorAll('[data-trk-view="true"]')
var observer = new IntersectionObserver(callbackRouter, { threshold: 0.3 });
elementsForTrackingViewableImpressions.forEach(observer.observe.bind(observer));
}
}, 500);
}
$(document).ready(function () {
// desktop notifications widget
Tracker.encodeNotificationLinks($(".f_notificationWidget"));
// mobile notifications widget
Tracker.encodeNotificationLinks($(".h_notifications"));
// desktop notifications page
Tracker.encodeNotificationLinks($(".notification-page"));
// mobile notifications page
Tracker.encodeNotificationLinks($(".notifications_page"));
});
</script>
<script type="text/javascript">
if(window.location.hash){
var hash = window.location.hash.substring(1);
if(hash == 'subscribe'){
$('html,body').animate({scrollTop: $('.ftxt3 > .newsletter').offset().top}, 'slow');
}
}
</script>
<script type="text/javascript">
var newsletterSubscriptionURL ='/newsletter/subscribe.action';
DynamicLinks.updateRssLinks('XnLuCjIf6z9BHCGZ6Nd4R76O32258ENr');
ContentSummary.setSelectedTab('en');
//when user enters the main content area show default topics in the topics bar
$("#content-wrapper").mouseenter(function() {
showDefaultTopics();
});
Bookmarks.contentTitle = "How to Avoid Cascading Failures in Distributed Systems";
Bookmarks.apiUrl = '/widgets/bookmark.action';
Bookmarks.isContentBookmarked = "false";
</script>
<script type="text/javascript">
var $buoop = {vs:{i:6,f:1,o:10.1,s:1}}
$buoop.ol = window.onload;
$(document).ready(function() {
try {if ($buoop.ol) $buoop.ol();}catch (e) {}
var e = document.createElement("script");
e.setAttribute("type", "text/javascript");
e.setAttribute("src", "https://cdn.infoq.com/statics_s2_20260910111657/scripts/lib/browser-update-org/update.js");
document.body.appendChild(e);
});
</script>
<script>
!function(f,b,e,v,n,t,s)
{if(f.fbq)return;n=f.fbq=function(){n.callMethod?
n.callMethod.apply(n,arguments):n.queue.push(arguments)};
if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
n.queue=[];t=b.createElement(e);t.async=!0;
t.src=v;s=b.getElementsByTagName(e)[0];
s.parentNode.insertBefore(t,s)}(window,document,'script',
'https://connect.facebook.net/en_US/fbevents.js');
fbq('init', '842388869148196');
fbq('track', 'PageView');
</script>
<noscript>
<img height="1" width="1" src="https://www.facebook.com/tr?id=842388869148196&ev=PageView&noscript=1"/>
</noscript>
<script type='text/javascript'>
try {
mixpanel.track('page viewed', {
'page name' : document.title,
'url' : window.location.pathname
});
}
catch(err) {
}
</script>
</body>
</html>
<!-- s2 -->