Files
nexus/sreweekly/articles/202/04-managing-failure-modes-in-microservice-architectures.html
2026-09-12 17:23:01 +08:00

2991 lines
216 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en" xmlns="http://www.w3.org/1999/xhtml">
<head>
<!-- Google Optimize Anti-flicker snippet (recommended) -->
<style>.async-hide { opacity: 0 !important} </style>
<style>.promo{background:#222528;position:fixed;z-index:1001!important;left:0;top:0;right:0;max-height:48px;min-height:48px;padding-top:0!important;padding-bottom:0!important}.promo,.promo p{-webkit-box-align:center;-ms-flex-align:center;align-items:center}.promo p{font-size:.8125rem;line-height:1rem;color:#fff;margin-bottom:0;margin-top:0;margin:0 auto;display:-webkit-box;display:-ms-flexbox;display:flex;font-weight:700}@media only screen and (max-width:650px){.promo p{font-size:.6875rem}}.promo span{overflow:hidden;display:-webkit-box;-webkit-line-clamp:3;-webkit-box-orient:vertical}.promo a{color:#fff!important;text-decoration:underline!important}.promo a.btn{background:#d0021b;padding:7px 20px;text-decoration:none!important;font-weight:700;margin-left:10px;margin-right:10px;white-space:nowrap;border-radius:5px}@media only screen and (max-width:650px){.promo a.btn{font-size:.6875rem;padding:7px 10px}}.promo.container{padding-top:8px;padding-bottom:8px}@media only screen and (min-width:1050px){.promo.container{padding-top:0;padding-bottom:0}}.promo .actions{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.promo .actions__left{-ms-flex-preferred-size:100%;flex-basis:100%;-webkit-box-align:center;-ms-flex-align:center;align-items:center}.promo .actions__right{-ms-flex-preferred-size:40px;flex-basis:40px;margin-top:0;margin-bottom:0;-webkit-box-pack:end;-ms-flex-pack:end;justify-content:flex-end}.promo.hidden{display:none}.promo.show{display:-webkit-box!important;display:-ms-flexbox!important;display:flex!important}.promo.show .container__inner{-webkit-box-flex:1;-ms-flex:1;flex:1}.promo.fixed{position:fixed}.promo.show+header.header{margin-top:48px}.header{background:#fff;-webkit-box-shadow:0 1px 0 #dde2e5;box-shadow:0 1px 0 #dde2e5}.header .actions__left,.header__bottom__events{max-width:100%!important;margin:0}.header .header__events-all{margin:0;display:-webkit-box;display:-ms-flexbox;display:flex;position:relative}.header .header__events-all .header__event-slot{-webkit-box-flex:1;-ms-flex:1 100%;flex:1 100%;border-right:1px solid #dde2e5;margin-top:3px;margin-bottom:3px;padding-left:15px;text-align:left;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center;padding-right:5px;min-width:300px}.header .header__events-all .header__event-slot:hover{text-decoration:none!important}.header .header__events-all .header__event-slot img{min-width:40px;height:40px}.header .header__events-all .header__event-slot div{margin-left:10px}.header .header__events-all .header__event-slot span{font-weight:700!important;font-size:.75rem;margin-bottom:0!important;margin-top:0;display:block;line-height:1.125rem;text-align:left}.header .header__events-all .header__event-slot p{font-weight:400;font-size:.625rem;line-height:130%!important;color:#495057!important;margin:0}@media only screen and (min-width:1050px){.header .header__events-all .header__event-slot p{font-size:.625rem}}.header .header__events-all .header__event-slot:first-child{padding-left:0}.header .header__events-all .header__event-slot:last-child{padding-right:0;border-right:0}@media only screen and (min-width:1050px){.header__bottom,.header__middle,.header__top{position:relative;white-space:nowrap}}.header__top{padding:10px 0}@media only screen and (min-width:800px){.header__top{padding:5px 0}}.header__bottom{padding:5px 0;z-index:29;max-width:100%}.header__bottom a{font-size:.8125rem}.header__bottom .trending{margin-left:0;margin-right:0}.header__bottom .actions{-webkit-box-align:start;-ms-flex-align:start;align-items:flex-start}.header__bottom .actions__left{max-width:calc(100% + 8px)}@media only screen and (min-width:800px){.header__bottom .actions__left{max-width:calc(100% + 24px)}}.header__bottom .actions__right{-webkit-box-flex:1;-ms-flex:1 1 auto;flex:1 1 auto;display:none}@media only screen and (min-width:1050px){.header__bottom .actions__right{display:-webkit-box;display:-ms-flexbox;display:flex}}.header__middle{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-flow:row wrap;flex-flow:row wrap}@media only screen and (min-width:1050px){.header__middle{-ms-flex-wrap:nowrap;flex-wrap:nowrap}}.header__top .actions__left{-webkit-box-align:center;-ms-flex-align:center;align-items:center;-ms-flex-line-pack:center;align-content:center}@media only screen and (min-width:1050px){.header__top .actions__right{max-width:430px}}.no-style.header__nav li:nth-child(3){font-weight:700}.no-style.header__nav li:nth-child(3) a{color:#0e5ef1!important}.header__bottom__events::after{background:-webkit-gradient(linear,left top,right top,color-stop(0,rgba(255,255,255,0)),to(#fff));background:linear-gradient(90deg,rgba(255,255,255,0) 0,#fff 100%);content:'';position:absolute;height:60px;right:10px;width:25px}@media only screen and (min-width:1050px){.header__bottom__events::after{display:none}}.contribute-link{font-weight:400;font-size:.6875rem;color:#000!important;position:relative;padding-left:10px}.contribute-link:hover{color:#00791d!important;text-decoration:none!important}.contribute-link::before{content:'';width:1px;height:12px;position:absolute;top:50%;-webkit-transform:translateY(-50%);transform:translateY(-50%);background:rgba(0,0,0,.1);left:-1px}.my-0{margin-top:0!important;margin-bottom:0!important}.header__desc.my-0{margin-left:0}.header__bottom__events .actions__left{max-width:100%!important;overflow-x:scroll;-ms-overflow-style:none;scrollbar-width:none;display:block;scroll-behavior:smooth;min-width:100%}.header__bottom__events .actions__left::-webkit-scrollbar{display:none}.header__bottom__events .actions__left:-webkit-scrollbar-thumb{background:#fff}.logo{line-height:1rem}.header{position:relative;z-index:41;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.header .input:focus,.header input[type=password]:focus,.header input[type=text]:focus{border:1px solid #00791d}.header a:not(.button):not(.login__action):not(.active){text-decoration:none;color:#222}.header a:not(.button):not(.login__action):not(.active):hover{text-decoration:underline;color:#222}.header__items{display:none;-ms-flex-wrap:wrap;flex-wrap:wrap}@media only screen and (min-width:1050px){.header__items{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row}.header__items nav{position:relative;background:0 0;padding:0;left:0;top:0;line-height:inherit;display:block;-webkit-box-shadow:none;box-shadow:none;max-width:100%;max-height:80px}}.header__items>div{width:100%;margin-bottom:32px;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-flow:column nowrap;flex-flow:column nowrap}@media only screen and (min-width:552px){.header__items>div:not(:nth-last-child(-n+2)){margin-bottom:32px}}@media only screen and (min-width:552px) and (max-width:1050px){.header__items>div{-webkit-box-flex:0;-ms-flex-positive:0;flex-grow:0;-ms-flex-negative:0;flex-shrink:0;-ms-flex-preferred-size:calc(99.7% * 1/2 - (32px - 32px * 1/2));flex-basis:calc(99.7% * 1/2 - (32px - 32px * 1/2));max-width:calc(99.7% * 1/2 - (32px - 32px * 1/2));width:calc(99.7% * 1/2 - (32px - 32px * 1/2))}.header__items>div:nth-child(1n){margin-right:32px;margin-left:0}.header__items>div:last-child{margin-right:0}.header__items>div:nth-child(2n){margin-right:0;margin-left:auto}}@media only screen and (min-width:800px) and (max-width:1050px){.header__items>div{-webkit-box-flex:0;-ms-flex-positive:0;flex-grow:0;-ms-flex-negative:0;flex-shrink:0;-ms-flex-preferred-size:calc(99.7% * 1/4 - (32px - 32px * 1/4));flex-basis:calc(99.7% * 1/4 - (32px - 32px * 1/4));max-width:calc(99.7% * 1/4 - (32px - 32px * 1/4));width:calc(99.7% * 1/4 - (32px - 32px * 1/4))}.header__items>div:nth-child(1n){margin-right:32px;margin-left:0}.header__items>div:last-child{margin-right:0}.header__items>div:nth-child(4n){margin-right:0;margin-left:auto}}@media only screen and (min-width:1050px){.header__items>div{margin-bottom:0!important;margin-right:0!important;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row}}.header__items .language__switcher{display:none}.header__items .language__switcher .li-nav.active>a,.header__items .language__switcher>li.active>a{color:#fff!important}.header__items .language__switcher .li-nav.active>a:hover,.header__items .language__switcher>li.active>a:hover{color:#fff!important;background:#0e5ef1}.header__items .language__switcher .li-nav:hover>a,.header__items .language__switcher>li:hover>a{color:#fff}.header+main{display:block;min-height:210px;-webkit-transition:margin .15s ease;transition:margin .15s ease;margin-top:0!important}.header--hide .header+main{margin-top:50px}.header--hide .header__toggle{opacity:0;top:20px;visibility:hidden}.header--hide .header__logo{max-height:0}.header--hide .header__middle,.header--hide .header__top{max-height:0;overflow:hidden;padding-top:0;padding-bottom:0;border-color:transparent}.header--hide .header__bottom .vue-portal-target{top:3px;right:0;bottom:auto;left:auto;position:absolute}.header__middle,.header__top{-webkit-transition:all .15s ease;transition:all .15s ease}.header__middle{border-bottom:1px solid rgba(0,0,0,.1);z-index:33}.header__middle .vue-portal-target{width:100%}@media only screen and (min-width:1050px){.header__middle .vue-portal-target{display:none}}@media only screen and (min-width:1050px){.header__middle{line-height:5.75rem;text-align:left;padding:0;z-index:30}.header__middle .widget__heading{display:none}}.header__top>.actions__left{display:none}@media only screen and (min-width:1050px){.header__top>.actions__left{display:-webkit-box;display:-ms-flexbox;display:flex}}.header__top>.actions__right{-webkit-box-flex:1;-ms-flex-positive:1;flex-grow:1;margin-right:0;margin-top:0;margin-bottom:0;-webkit-box-pack:end;-ms-flex-pack:end;justify-content:flex-end}.header__top>.actions__right .dropdown__holder{width:calc(100vw - 16px);max-height:80vh}@media only screen and (min-width:600px){.header__top>.actions__right .dropdown__holder{width:auto}}.header__top>.actions__right>*{margin:0}.header__top>.actions__right .search{display:none}@media only screen and (min-width:1050px){.header__top>.actions__right>*{display:-webkit-box;display:-ms-flexbox;display:flex;white-space:nowrap}.header__top>.actions__right .search{display:block}}.header__top .user__login{display:block}.header__top .user__login>.button,.header__top .user__login>button{border-top-right-radius:0;border-bottom-right-radius:0}@media only screen and (min-width:1050px){.header__top{position:relative;right:auto;width:100%}}.header__logo{max-width:165px;position:absolute;top:8px;overflow:hidden;-webkit-transition:all .1s ease;transition:all .1s ease;z-index:32;line-height:2.25rem;height:36px;width:100px;margin-left:50px;-ms-flex-preferred-size:190px;flex-basis:190px}@media only screen and (min-width:800px){.header__logo{top:4px}}@media only screen and (min-width:1050px){.header__logo{position:relative;top:0;overflow:visible;margin-right:20px;margin-left:0;line-height:3.125rem;height:50px}.header__logo>*{width:165px}}@media only screen and (min-width:1280px){.header__logo{margin-right:30px}}.header__desc,.header__more>button{text-transform:capitalize;color:#666;letter-spacing:0;font-size:0;font-weight:400;line-height:1.5rem;vertical-align:top;font-smoothing:antialiased;font-family:-apple-system,BlinkMacSystemFont,"Segoe UI Variable","Segoe UI",system-ui,ui-sans-serif,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji";-webkit-transition:font-size .15s ease-in-out;transition:font-size .15s ease-in-out}@media only screen and (min-width:1050px){.header__desc,.header__more>button{font-size:.6875rem}}.header__desc{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;position:relative;width:auto}.header__topics{white-space:nowrap;float:right;position:relative}.header__topics *{display:inline-block;vertical-align:top}.header__topics a{font-size:.8125rem}@media only screen and (min-width:1050px){.header__topics{float:none}}.header__more{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;vertical-align:top}.header__more:before{left:-12px}.header__more:after,.header__more:before{content:'';width:1px;height:12px;position:absolute;top:50%;-webkit-transform:translateY(-50%);transform:translateY(-50%);background:rgba(0,0,0,.1)}.header__more:after{right:-12px}.header__more:hover{cursor:pointer;color:rgba(0,0,0,.75)}.header__user{display:inline-block;vertical-align:top;white-space:nowrap;margin-left:8px}@media only screen and (min-width:1050px){.header__user{margin-left:0}}.header__user>div{display:inline-block;vertical-align:top}.header__user-nav a:not(.button){text-transform:uppercase;font-size:.75rem;font-weight:600}.header__user-nav a:not(.button):not(.active){opacity:.5}.header__user-nav a:not(.button):not(.active):hover{opacity:.75}.header__user-nav a:not(.button):hover{text-decoration:none}.header__user-nav a:not(.button).active{color:#222;cursor:default;text-decoration:none}.header__user-nav a:not(.button):before{margin-right:0}.header__user-nav a:not(.button):not(:last-child){margin-right:16px}@media only screen and (min-width:1050px){.header__user-nav a:not(.button):not(:last-child){margin-right:32px}}@media only screen and (min-width:1050px){.header__user-nav+.header__topics{margin-left:56px}}.header__search{display:none;vertical-align:top;margin-right:0}@media only screen and (min-width:1050px){.header__search{display:inline-block}}.header__search,.header__user{line-height:inherit}.header__nav{border-bottom:1px solid rgba(0,0,0,.1)}@media only screen and (min-width:800px){.header__nav{border-bottom:0}}.header__nav .button{margin-left:0}.header__nav .button__more{margin-right:20px}.header__nav .li-nav,.header__nav>li{-webkit-transition:all .15s ease;transition:all .15s ease}@media only screen and (min-width:1050px){.header__nav .li-nav,.header__nav>li{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;vertical-align:middle;position:static;border:none;min-height:65px}.header__nav .li-nav:hover.has--subnav .nav .li-nav,.header__nav .li-nav:hover.has--subnav .nav>li,.header__nav>li:hover.has--subnav .nav .li-nav,.header__nav>li:hover.has--subnav .nav>li{white-space:normal}.header__nav .li-nav:hover.has--subnav .nav__subnav,.header__nav>li:hover.has--subnav .nav__subnav{opacity:1;top:100%;visibility:visible;-webkit-transition-delay:.25s;transition-delay:.25s}.header__nav .li-nav:hover.has--subnav .nav__category,.header__nav>li:hover.has--subnav .nav__category{width:100%}.header__nav .li-nav:hover.has--subnav .nav__category:after,.header__nav .li-nav:hover.has--subnav .nav__category:before,.header__nav>li:hover.has--subnav .nav__category:after,.header__nav>li:hover.has--subnav .nav__category:before{-webkit-transition-delay:.25s;transition-delay:.25s;opacity:1}}@media only screen and (min-width:1080px){.header__nav .li-nav:not(:last-child),.header__nav>li:not(:last-child){margin-right:23px;margin-left:23px}}@media only screen and (min-width:1280px){.header__nav .li-nav:not(:last-child),.header__nav>li:not(:last-child){margin-right:23px;margin-left:23px}}@media only screen and (min-width:1338px){.header__nav .li-nav:not(:last-child),.header__nav>li:not(:last-child){margin-right:23px;margin-left:23px}}.header__nav .li-nav>a,.header__nav>li>a{font-size:.875rem;line-height:1.5rem;padding:12px 0;color:#000;display:inline-block;max-width:100%;position:relative;z-index:61;white-space:normal}@media only screen and (min-width:1050px){.header__nav .li-nav>a,.header__nav>li>a{padding:8px 0 0;font-weight:700}}.header__nav .li-nav>a:after,.header__nav .li-nav>a:before,.header__nav>li>a:after,.header__nav>li>a:before{content:'';position:absolute;bottom:-1px;left:50%;width:0;height:0;opacity:0;-webkit-transform:translateX(-50%);transform:translateX(-50%);border-style:solid;border-width:0 5px 5px 5px;-webkit-transition:opacity .15s ease-in-out;transition:opacity .15s ease-in-out;border-color:transparent transparent #fff transparent}.header__nav .li-nav>a:before,.header__nav>li>a:before{left:50%;bottom:0;border-width:0 6px 6px 6px;border-color:transparent transparent #f5f7f8 transparent}@media only screen and (min-width:1050px){.header__nav .li-nav>a,.header__nav>li>a{font-size:1rem}}@media only screen and (min-width:1800px){.header__nav .li-nav>a,.header__nav>li>a{font-size:1.125rem}}.header--open{overflow:hidden}@media only screen and (min-width:600px){.header--open{overflow:visible}}.header--open .content-items{max-height:215px;margin:12px 0 24px}.header--open .search{margin-top:16px;display:block}.header--open .header__toggle:before{z-index:10}.header--open .header__toggle>span:nth-child(1){top:50%;-webkit-transform:rotate(45deg);transform:rotate(45deg)}.header--open .header__toggle>span:nth-child(2){opacity:0}.header--open .header__toggle>span:nth-child(3){top:50%;-webkit-transform:rotate(-45deg);transform:rotate(-45deg)}.header--open .header__items,.header--open .header__items .language__switcher{display:-webkit-box;display:-ms-flexbox;display:flex}.header--open .header__top{z-index:100;position:absolute;left:0;right:8px;background:#fff}@media only screen and (min-width:800px){.header--open .header__top{right:20px}}.header--open .header__top .search{display:none}.header--open .header__logo{z-index:101}.header--open .header__middle{padding-top:60px}.header--open .header__bottom{display:none}.header--open .header__container{max-height:90vh;overflow-x:hidden;overflow-y:auto}@media only screen and (min-width:1050px){.header--open .header__container{overflow:visible}}.header .subnav{position:absolute;-webkit-box-shadow:0 5px 25px 1px rgba(0,0,0,.15);box-shadow:0 5px 25px 1px rgba(0,0,0,.15);background:#fff;visibility:hidden;line-height:1.75rem;max-width:100%;width:100%;left:0;opacity:0;overflow:hidden;border:1px solid #f5f7f8;border-radius:2px;-webkit-transition:all .15s ease-in-out;transition:all .15s ease-in-out;top:105%;z-index:60;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-flow:row nowrap;flex-flow:row nowrap;-webkit-box-align:stretch;-ms-flex-align:stretch;align-items:stretch}.header .subnav .subnav__categories{-webkit-box-flex:0;-ms-flex:0 1 280px;flex:0 1 280px;padding:24px 0;background:#f5f7f8;margin-right:0!important}.header .subnav .subnav__categories>li{display:block;font-size:.9375rem;padding:2px 48px 2px 24px}.header .subnav .subnav__categories>li a{display:block;font-weight:700}.header .subnav .subnav__categories>li:hover{background:#e1e1e1}.header .subnav .subnav__heading{margin-bottom:20px}.header .subnav .subnav__content{position:relative;-ms-flex-item-align:start;align-self:flex-start;padding:24px;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-flow:row wrap;flex-flow:row wrap}@media only screen and (min-width:1050px){.header .subnav .subnav__content{padding-left:32px;padding-right:32px;margin:0 auto;-webkit-box-flex:1;-ms-flex:1 1 600px;flex:1 1 600px}}.header .subnav .subnav__content .heading__container{-ms-flex-preferred-size:100%;flex-basis:100%}.header .subnav .subnav__content:before{content:'';position:absolute;left:0;top:0;bottom:-100%;width:1px;background:rgba(0,0,0,.1)}.languagesEdition .active{font-weight:700}.header__more.dropdown:after{content:'';display:inline-block;vertical-align:middle;-ms-flex-item-align:center;-ms-grid-row-align:center;align-self:center;background-repeat:no-repeat;background-position:center;margin-bottom:2px;width:7px;height:7px;-webkit-transition:-webkit-transform .15s ease;transition:-webkit-transform .15s ease;transition:transform .15s ease;transition:transform .15s ease,-webkit-transform .15s ease;background-color:#fff!important;background-size:contain}.header__more.dropdown button{color:#000!important;margin-right:-20px;padding-right:20px;z-index:1000}.header__more.dropdown .dropdown__holder{width:165px}.header__more.dropdown .dropdown__content{padding:13px;padding-top:5px;padding-bottom:5px}.header__more.dropdown .languagesEdition li{border-bottom:1px solid #e6e6e6}.header__more.dropdown .languagesEdition li:last-child{border:0}.logo__data{display:none;color:#666;font-size:.6875rem}@media only screen and (min-width:1050px){.logo__data{display:block;line-height:.8125rem}}.container{padding-left:12px;padding-right:12px;margin:0 auto;min-width:320px;-webkit-transition:padding .15s ease-in-out;transition:padding .15s ease-in-out}@media only screen and (min-width:600px){.container{padding-left:16px;padding-right:16px}}@media only screen and (min-width:800px){.container{padding-left:20px;padding-right:20px}}@media only screen and (min-width:1250px){.container{padding-left:60px;padding-right:60px}}@media only screen and (min-width:1400px){.container{padding-left:20px;padding-right:20px}}.container.white{background:#fff}.container__inner{max-width:1290px;margin:0 auto;-webkit-transition:max-width .15s ease-out;transition:max-width .15s ease-out}.search{display:block;position:relative;z-index:33;width:100%;max-width:100%;margin:0}.search:before{position:absolute;top:50%;right:24px;-webkit-transform:translateY(-50%);transform:translateY(-50%);margin-right:0;z-index:34}.search__bar{display:block;border-radius:2px;position:relative;z-index:33}.search__bar #search{margin-bottom:0;max-width:100%;background:#fff}.search__go{top:50%;right:0;bottom:0;left:auto;position:absolute;-webkit-transform:translateY(-50%);transform:translateY(-50%);z-index:32;-webkit-appearance:none;-moz-appearance:none;appearance:none;width:36px;height:36px;line-height:2.25rem;-webkit-box-shadow:none;box-shadow:none;display:block;background:0 0;border:0;font-size:0}@media only screen and (min-width:600px){.search__go{z-index:35}}.header #search,.search__go:hover{cursor:pointer}.header #search{height:36px;position:relative;max-width:100%;background-color:#f5f7f8!important}@media only screen and (min-width:600px){.header #search{font-size:.8125rem;min-width:165px;max-width:100%;opacity:1}}.header #search:hover{cursor:auto}@media only screen and (min-width:1050px){.header #search{margin-left:auto;border-top-right-radius:0;border-bottom-right-radius:0;border-right:0}.header #search:focus{min-width:215px}}.header #search:focus{cursor:auto}.header #search .field__desc{display:none}.header #searchForm{width:100%;margin-top:8px}.header #searchForm:before{right:8px}@media only screen and (min-width:600px){.header #searchForm{margin-top:16px}}@media only screen and (min-width:1050px){.header #searchForm{margin-top:0}}</style>
<script>(function(a,s,y,n,c,h,i,d,e){s.className+=' '+y;h.start=1*new Date;
h.end=i=function(){s.className=s.className.replace(RegExp(' ?'+y),'')};
(a[n]=a[n]||[]).hide=h;setTimeout(function(){i();h.end=null},c);h.timeout=c;
})(window,document.documentElement,'async-hide','dataLayer',4000,
{'GTM-W9GJ5DL':true});</script>
<script type="text/javascript">
var loggedIn = false;
if (loggedIn) {
var userCountryId = '';
}
</script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
//CookieControl tool recomendation
// Call the default command before gtag.js or Tag Manager runs to
// adjust how the tags operate when they run. Modify the defaults
// per your business requirements and prior consent granted/denied, e.g.:
gtag('consent', 'default', {
'ad_storage': 'denied',
'ad_user_data': 'denied',
'ad_personalization': 'denied',
'analytics_storage': 'denied',
wait_for_update: 500
});
gtag('set', 'ads_data_redaction', true);
gtag('set', 'url_passthrough', true);
if((typeof loggedIn != "undefined") && loggedIn){
window.dataLayer.push({'logged_in': 'true'});
} else {
window.dataLayer.push({'logged_in': 'false'});
}
window.dataLayer.push({'show_queryz': ''});
gtag('config', 'G-VMVPD4D2JY');
</script>
<!-- Google Tag Manager -->
<script>
var gtmProfile="GTM-W9GJ5DL";
(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer', gtmProfile);</script>
<!-- End Google Tag Manager -->
<title>Managing Failure Modes in Microservice Architectures - InfoQ</title>
<link rel="canonical" href="https://www.infoq.com/presentations/microservices-failure-modes/"/>
<link rel="alternate" href="https://www.infoq.com/presentations/microservices-failure-modes/" hreflang="en"/>
<link rel="alternate" href="https://www.infoq.com/presentations/microservices-failure-modes/" hreflang="x-default" />
<meta http-equiv="pragma" content="no-cache" />
<meta http-equiv="cache-control" content="private,no-cache,no-store,must-revalidate" />
<meta http-equiv="expires" content="0" />
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
<meta name="copyright" content="&copy; 2006 C4Media" />
<meta name="google-site-verification" content="0qInQx_1WYOeIIbxnh7DnXlw1XOxNgAYakO2k4GhNnY" />
<meta name="msapplication-TileColor" content="#ffffff"/>
<meta name="msapplication-TileImage" content="/styles/static/images/logo/logo.jpg"/>
<meta property="wb:webmaster" content="3eac1729a8bbe046" />
<meta content="width=device-width,initial-scale=1" name="viewport">
<meta http-equiv="X-UA-Compatible" content="IE=10, IE=edge">
<link rel="stylesheet" type="text/css" media="screen" href="https://cdn.infoq.com/statics_s1_20260910111632/styles/style.css"/>
<link rel="stylesheet" type="text/css" media="print" href="https://cdn.infoq.com/statics_s1_20260910111632/styles/print.css"/>
<link rel="preload" type="text/css" href="https://cdn.infoq.com/statics_s1_20260910111632/styles/style_en.css" as="style" onload="this.onload=null;this.rel='stylesheet'">
<link rel="preload" href="https://cdn.infoq.com/statics_s1_20260910111632/styles/icons.css" as="style" onload="this.onload=null;this.rel='stylesheet'">
<link rel="preload" type="text/css" media="screen" href="https://cdn.infoq.com/statics_s1_20260910111632/styles/style_extra.css" as="style" onload="this.onload=null;this.rel='stylesheet'">
<noscript>
<link rel="stylesheet" type="text/css" href="https://cdn.infoq.com/statics_s1_20260910111632/styles/style_en.css"/>
<link rel="stylesheet" href="https://cdn.infoq.com/statics_s1_20260910111632/styles/icons.css">
<link rel="stylesheet" type="text/css" media="screen" href="https://cdn.infoq.com/statics_s1_20260910111632/styles/style_extra.css"/>
</noscript>
<link rel="stylesheet" type="text/css" href="https://cdn.infoq.com/statics_s1_20260910111632/styles/fonts/font.css"/>
<link rel="shortcut icon" href="https://cdn.infoq.com/statics_s1_20260910111632/favicon.ico" type="image/x-icon" />
<link rel="apple-touch-icon" href="https://cdn.infoq.com/statics_s1_20260910111632/apple-touch-icon.png"/>
<script type="text/javascript">
var device='desktop';
var InfoQConstants = {};
InfoQConstants.language = 'en';
InfoQConstants.countryCode = '';
InfoQConstants.scp = 'eyJTdGF0ZW1lbnQiOiBbeyJSZXNvdXJjZSI6IioiLCJDb25kaXRpb24iOnsiRGF0ZUxlc3NUaGFuIjp7IkFXUzpFcG9jaFRpbWUiOjE3ODkxNzk1NjV9LCJJcEFkZHJlc3MiOnsiQVdTOlNvdXJjZUlwIjoiMC4wLjAuMC8wIn19fV19';
InfoQConstants.scs = 'b2rpQAEAeU5~CRlEliW1pDfh5crr6t~sxCKD4L6SQBLaFifyU~IlQCtK8HbiCqEmNZ8pPyWsv8x2Ffj8Qdx0V6t8~7oMEUcDGQhNul5i0dmlwsfidM78xj49sE6AJ34cOQDrHr84adoAVRw0Gnl-2shJjkcGQB08tuypLsJ07ik_';
InfoQConstants.sck = 'APKAIMZVI7QH4C5YKH6Q';
InfoQConstants.scd = '.infoq.com';
InfoQConstants.sct = '7200';
InfoQConstants.pageUrl = (typeof window.location != 'undefined' && window.location && typeof window.location.href != 'undefined' && window.location.href) ? window.location.href : "URL_UNAVAILABLE";
InfoQConstants.cet='3oCTVQJcuE725ucu';
InfoQConstants.userDetectedCountryCode = 'US';
InfoQConstants.bpadb = '3PoHelVdDuUlnG1iFrPq';
</script>
<script type="text/javascript" src="https://cdn.infoq.com/statics_s1_20260910111632/scripts/combinedJs.min.js"></script>
<script type="text/javascript">
var JSi18n = JSi18n || {}; // define only if not already defined
JSi18n.error='Error';
JSi18n.login_unverifiedAccount='Unverified account';
JSi18n.contentSummary_showPresentations_1='';
JSi18n.contentSummary_showPresentations_2='';
JSi18n.contentSummary_showPresentations_3='';
JSi18n.contentSummary_showInterviews_1='';
JSi18n.contentSummary_showInterviews_2='';
JSi18n.contentSummary_showInterviews_3='';
JSi18n.contentSummary_showMinibooks_1='';
JSi18n.contentSummary_showMinibooks_2='';
JSi18n.login_sendingRequest='Sending request ...';
JSi18n.bookmark_saved='<q>&nbsp;&nbsp;&nbsp;Saved&nbsp;&nbsp;&nbsp;&nbsp;</q>';
JSi18n.bookmark_error='<q style=color:black;>&nbsp;&nbsp;&nbsp;Error&nbsp;&nbsp;&nbsp;&nbsp;</q>';
JSi18n.categoryManagement_showpopup_viewAllLink_viewAllPrefix='View All';
JSi18n.categoryManagement_showpopup_viewAllLink_viewAllSuffix='';
JSi18n.categoryManagement_showpopup_includeExcludeLink_Exclude='Exclude';
JSi18n.categoryManagement_showpopup_includeExcludeLink_Include='Include';
JSi18n.login_invalid_email='Please specify a valid email';
JSi18n.login_email_not_found = 'No user found with that email';
JSi18n.content_datetime_format='MMM dd, yyyy';
// used by frontend
JSi18n.FE = {
labels: {
follow: "Follow",
followTopic: "Follow Topic",
unfollow: "Unfollow",
unfollowTopic: "Unfollow Topic",
following: "Following",
followers: "Followers",
like: "Like",
liked: "Liked",
authors: "Peers",
users : "Users",
topics: "Topics",
hide: "Hide Item",
hidden: "%s was hidden on your profile page.",
undo: "Undo",
showLess: "Show less",
showMore: "Show more",
moreAuthors: "And %s more",
bookmarked: "Content Bookmarked",
unbookmarked: "Content Unbookmarked",
characterLimit: "Characters Remaining"
}
}
var usersInPage = JSON.parse('[{\"id\":\"62671158\",\"ref\":\"author-Monica-Beckwith\",\"url\":\"\/profile\/Monica-Beckwith\",\"followedByCurrentUser\":false,\"minibio\":\"\",\"name\":\"Monica Beckwith\",\"bio\":\"\",\"followers\":1833,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s1_20260910111632\/images\/profiles\/QhDv7pXEUK1sdLsYArFmnLxecH4rYhAc.jpg\"},{\"id\":\"45723890\",\"ref\":\"author-Rags-Srinivas\",\"url\":\"\/profile\/Rags-Srinivas\",\"followedByCurrentUser\":false,\"minibio\":\"\",\"name\":\"Rags Srinivas\",\"bio\":\"\",\"followers\":356,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s1_20260910111632\/images\/profiles\/t4nyfgw1THkp4wMZ9EZ59RGJx8Hf9Rk8.jpg\"},{\"id\":\"87551248\",\"ref\":\"author-Steef~Jan-Wiggers\",\"url\":\"\/profile\/Steef~Jan-Wiggers\",\"followedByCurrentUser\":false,\"minibio\":\"Cloud Queue Lead Editor | Domain Architect | Cloud Expert\",\"name\":\"Steef-Jan Wiggers\",\"bio\":\"\",\"followers\":895,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s1_20260910111632\/images\/profiles\/BhZx03k3Hj0pZVXmTzGqItwZxtJ06oIb.jpg\"},{\"id\":\"80977916\",\"ref\":\"author-Thomas-Betts\",\"url\":\"\/profile\/Thomas-Betts\",\"followedByCurrentUser\":false,\"minibio\":\"Senior Laureate Application Architect at Blackbaud\",\"name\":\"Thomas Betts\",\"bio\":\"\",\"followers\":1171,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s1_20260910111632\/images\/profiles\/pSqI6HrU3k9rmmVjwS34OHG0bOMYiE6a.jpg\"},{\"id\":\"72028228\",\"ref\":\"author-Sergio-De-Simone\",\"url\":\"\/profile\/Sergio-De-Simone\",\"followedByCurrentUser\":false,\"minibio\":\"\",\"name\":\"Sergio De Simone\",\"bio\":\"\",\"followers\":777,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s1_20260910111632\/images\/profiles\/NovciOoQOAYWqYqRQBFo97SuMm0xbUiC.jpg\"},{\"id\":\"126467140\",\"ref\":\"author-Renato-Losio\",\"url\":\"\/profile\/Renato-Losio\",\"followedByCurrentUser\":false,\"minibio\":\"Cloud Expert | AWS Data Hero \",\"name\":\"Renato Losio\",\"bio\":\"\",\"followers\":913,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s1_20260910111632\/images\/profiles\/ptroF8HdI2vWXm0NDaKeS0JdiPxMOAra.jpg\"},{\"id\":\"101764598\",\"ref\":\"author-Adrian-Cockcroft\",\"url\":\"\/profile\/Adrian-Cockcroft\",\"followedByCurrentUser\":false,\"minibio\":\"\",\"name\":\"Adrian Cockcroft\",\"bio\":\"\",\"followers\":43,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s1_20260910111632\/images\/profiles\/Z1WrQYKtfrWWi8zkA1YCInu0KM6Ape6p.jpg\"},{\"id\":\"126464202\",\"ref\":\"author-Johan-Janssen\",\"url\":\"\/profile\/Johan-Janssen\",\"followedByCurrentUser\":false,\"minibio\":\"Architect at ASML\",\"name\":\"Johan Janssen\",\"bio\":\"\",\"followers\":536,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s1_20260910111632\/images\/profiles\/Fb4eZ0mtvMf6MhsmBIhsUVueV4xAs2FD.jpg\"},{\"id\":\"39485652\",\"ref\":\"author-Daniel-Bryant\",\"url\":\"\/profile\/Daniel-Bryant\",\"followedByCurrentUser\":false,\"minibio\":\"InfoQ News Manager | Building Platforms at Syntasso\",\"name\":\"Daniel Bryant\",\"bio\":\"\",\"followers\":2751,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s1_20260910111632\/images\/profiles\/tSe5dczMaSGtRUm18VkTR2tcF4W3SogA.jpg\"},{\"id\":\"63268344\",\"ref\":\"author-Chris-Swan\",\"url\":\"\/profile\/Chris-Swan\",\"followedByCurrentUser\":false,\"minibio\":\"Engineer, Atsign\",\"name\":\"Chris Swan\",\"bio\":\"\",\"followers\":1873,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s1_20260910111632\/images\/profiles\/8PE76nOujWAoCM8yqLn9Hfv2HqW3VlIP.jpg\"},{\"id\":\"4927376\",\"ref\":\"author-Karsten-Silz\",\"url\":\"\/profile\/Karsten-Silz\",\"followedByCurrentUser\":false,\"minibio\":\"Full-Stack Java Developer\",\"name\":\"Karsten Silz\",\"bio\":\"\",\"followers\":399,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s1_20260910111632\/images\/profiles\/p6zmOdOcqXiRj09iiZNeDZap7f0IglQW.jpg\"},{\"id\":\"343314\",\"ref\":\"author-Jonathan-Allen\",\"url\":\"\/profile\/Jonathan-Allen\",\"followedByCurrentUser\":false,\"minibio\":\"Software Architect\",\"name\":\"Jonathan Allen\",\"bio\":\"\",\"followers\":1833,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s1_20260910111632\/images\/profiles\/Wk_C09_mzwK23YkTkKMXResJv3LKUN5D.jpg\"},{\"id\":\"114725059\",\"ref\":\"author-Holly-Cummins\",\"url\":\"\/profile\/Holly-Cummins\",\"followedByCurrentUser\":false,\"minibio\":\"Senior Technical Staff Member, IBM\",\"name\":\"Holly Cummins\",\"bio\":\"\",\"followers\":615,\"imgSrc\":\"https:\/\/cdn.infoq.com\/statics_s1_20260910111632\/images\/profiles\/cRsuGlFgKyGmGfEHvafpMO63CxbrEm22.jpg\"}]');
var topicsInPage = JSON.parse('[{\"name\":\"Architecture & Design\",\"id\":\"6816\",\"url\":\"\/architecture-design\",\"followers\":10383,\"followedByCurrentUser\":false},{\"name\":\"Culture & Methods\",\"id\":\"6817\",\"url\":\"\/culture-methods\",\"followers\":3994,\"followedByCurrentUser\":false},{\"name\":\"Resilience\",\"id\":\"14722\",\"url\":\"\/Resilience\",\"followers\":90,\"followedByCurrentUser\":false},{\"name\":\".NET Core\",\"id\":\"15683\",\"url\":\"\/Net-Core\",\"followers\":8030,\"followedByCurrentUser\":false},{\"name\":\"InfoQ\",\"id\":\"134\",\"url\":\"\/InfoQ\",\"followers\":95,\"followedByCurrentUser\":false},{\"name\":\"Machine Learning\",\"id\":\"5449\",\"url\":\"\/MachineLearning\",\"followers\":14751,\"followedByCurrentUser\":false},{\"name\":\"Microservices\",\"id\":\"15274\",\"url\":\"\/microservices\",\"followers\":22348,\"followedByCurrentUser\":false},{\"name\":\"Architecture\",\"id\":\"1389\",\"url\":\"\/architecture\",\"followers\":3366,\"followedByCurrentUser\":false},{\"name\":\"QCon San Francisco 2019\",\"id\":\"17901\",\"url\":\"\/qcon-sanfrancisco-2019\",\"followers\":13,\"followedByCurrentUser\":false},{\"name\":\"AI, ML & Data Engineering\",\"id\":\"16690\",\"url\":\"\/ai-ml-data-eng\",\"followers\":6077,\"followedByCurrentUser\":false},{\"name\":\"QCon Software Development Conference\",\"id\":\"597\",\"url\":\"\/qcon\",\"followers\":261,\"followedByCurrentUser\":false},{\"name\":\"Java9\",\"id\":\"7097\",\"url\":\"\/Java9\",\"followers\":5540,\"followedByCurrentUser\":false},{\"name\":\"Cloud\",\"id\":\"7451\",\"url\":\"\/Cloud\",\"followers\":2170,\"followedByCurrentUser\":false},{\"name\":\"DevOps\",\"id\":\"6043\",\"url\":\"\/Devops\",\"followers\":5131,\"followedByCurrentUser\":false},{\"name\":\"Reactive Programming\",\"id\":\"15453\",\"url\":\"\/reactive-programming\",\"followers\":12430,\"followedByCurrentUser\":false},{\"name\":\"Transcripts\",\"id\":\"17662\",\"url\":\"\/transcripts\",\"followers\":21,\"followedByCurrentUser\":false},{\"name\":\"Cloud Native Architecture\",\"id\":\"17983\",\"url\":\"\/Cloud-Native-Architecture\",\"followers\":91,\"followedByCurrentUser\":false},{\"name\":\"Development\",\"id\":\"6815\",\"url\":\"\/development\",\"followers\":4154,\"followedByCurrentUser\":false}]');
var userContentLikesInPage = [];
var userCommentsLikesInPage = [];
var currentUserId = 2;
</script>
<!-- Latest Version -->
<script src="https://cc.cdn.civiccomputing.com/9/cookieControl-9.x.min.js" type="text/javascript"></script>
<script>
const config = {
apiKey: '8910ea974a96ffb6f927952b4ae9b9b0cc3e5973',
product: 'PRO_MULTISITE',
// general settings
consentCookieExpiry: 90,
encodeCookie: true,
sameSiteCookie: true, // if false, cookie set as SameSite=None;secure;
sameSiteValue: ';secure', // either 'Strict', 'Lax', or 'None'
subDomains: true,
initialState: 'notify', // 'notify','top','box' require pro licence
notifyOnce: false,
setInnerHTML: true,
//layout settings
layout: 'slideout',
position: 'left',
theme: 'light',
acceptButton: true,
rejectButton: true,
closeOnGlobalChange: true,
closeStyle: 'icon',
toggleType: 'slider',
notifyDismissButton: true,
settingsStyle: 'link',
excludedCountries: ['US'],
/*accessibility: {
disableSiteScrolling: true,
},*/
statement: {
description: 'For more detailed information about the cookies we use, see our',
name: 'Cookie Policy',
url: 'https://www.infoq.com/cookie-policy',
updated: '01/01/2024',
},
// cookies starting from 'cookie_expire' are from live.infoq.com but since we use the same tool on the same domain we need to specify those too so
// that infoq.com cookieControl does not delete live.infoq.com cookies(also infoq.com cookies have been specified in live.infoq.com cookieControl configs)
necessaryCookies: ['RegUserCookie', 'UserCookie', 'IdpCookie', 'ConversionTrackingV2_','PSAdialog','*P13NWN*','topbarSurvey','__bkm','JSESSIONID','mp_','_mixpanel','CloudFront-Key-Pair-Id','CloudFront-Policy','CloudFront-Signature','cookie_expire','discount_promo_closed','discount_promo_code','discount_promo_submitted','exit_survey_popup','referrer_popup','voting_popup_*','AWSALB','AWSALBCORS','aws-waf-token','iq_sub_preview'],
optionalCookies: [
{
name: 'analytics',
label: 'Analytics',
description: 'Analytical cookies help us to improve our website by collecting and reporting information on its usage.',
cookies: ['_ga', '_ga*', '_gid', '_gat', '__utma', '__utmt', '__utmb', '__utmc', '__utmz', '__utmv'],
onAccept: function(){
gtag('consent', 'update', {'analytics_storage': 'granted'});
},
onRevoke: function(){
gtag('consent', 'update', {'analytics_storage': 'denied'});
}
},
{
name: 'marketing',
label: 'Advertising',
description: 'We use advertising cookies to display advertisements to you for our products.',
onAccept: function(){
gtag('consent', 'update', {'ad_storage': 'granted', 'ad_personalization': 'granted', 'ad_user_data': 'granted'});
},
onRevoke: function(){
gtag('consent', 'update', {'ad_storage': 'denied', 'ad_personalization': 'denied', 'ad_user_data': 'denied'});
}
}
],
text : {
// main preference panels
title: '<h3>Our use of cookies</h3>',
intro: 'We use necessary cookies to make our site work. Functional cookies help enhance the performance and functionality of the site. '+
'We\'d also like to set analytics cookies to help us improve your experience by measuring how you use the site. '+
'These will be set only if you accept. ',
acceptSettings: 'I Accept',
rejectSettings: 'I Do Not Accept',
necessaryTitle : '<h3>Necessary Cookies</h3>',
necessaryDescription : 'Necessary cookies enable core functionality ' +
'such as page navigation and access to secure areas. '+
'The website cannot function properly without '+
'these cookies, and can only be disabled by changing '+
'your browser preferences.',
closeLabel: 'Close Cookie Control',
cornerButton: 'Set cookie preferences',
// main preference panel controls
on: 'On',
off : 'Off',
thirdPartyTitle : 'Some cookies require your attention',
thirdPartyDescription : 'Consent for the following cookies could not be '+
'automatically revoked. Please follow the link(s) '+
'below to opt out manually.',
// notification panels (only accessible for pro licences)
notifyTitle : 'Your choice regarding cookies on this site',
notifyDescription : 'We use cookies to optimise site functionality and '+
'give you the best possible experience.',
accept : 'I Accept',
reject: 'I Do Not Accept',
settings : 'Settings',
},
branding : {
removeAbout: true,
},
};
// do not load this for local envs only. for testing on local envs remove/modify this condition
if(InfoQConstants.pageUrl.indexOf('local')===-1){
CookieControl.load( config );
}
</script>
<script type="application/ld+json">
{
"@context": "http://schema.org",
"@type": "NewsArticle",
"mainEntityOfPage": {
"@type": "WebPage",
"@id": "https://www.infoq.com/presentations/microservices-failure-modes/"
},
"headline": "Managing Failure Modes in Microservice Architectures",
"image": ["https://res.infoq.com/presentations/microservices-failure-modes/en/mediumimage/Adrian-Cockcroft-QConSF2019-m-1578439545067.jpg"
],
"datePublished": "2020-01-07T17:35:00+0000",
"dateModified": "2020-01-07T17:35:00+0000",
"author": [
{
"@type": "Person",
"name": "Adrian Cockcroft",
"url": "https://www.infoq.com/profile/Adrian-Cockcroft/"
}
],
"publisher": {
"@type": "Organization",
"name": "InfoQ",
"logo": {
"@type": "ImageObject",
"url": "https://assets.infoq.com/resources/en/logo_scrubber_big_4_3.jpg"
}
},
"description": "Adrian Cockcroft explores how to apply some industry standard techniques (including Failure Modes and Effects Analysis) to cloud native microservices architectures.",
"video":
{
"@type": "VideoObject",
"name": "Managing Failure Modes in Microservice Architectures",
"description": "Adrian Cockcroft explores how to apply some industry standard techniques (including Failure Modes and Effects Analysis) to cloud native microservices architectures.",
"thumbnailUrl": [
"https://res.infoq.com/presentations/microservices-failure-modes/en/mediumimage/Adrian-Cockcroft-QConSF2019-m-1578439545067.jpg"
],
"uploadDate": "2020-01-07T17:35:00+0000",
"duration": "PT51M33S",
"contentUrl": "https://videog.infoq.com/presentations/19-nov-managingfailuremodesinm-2.mp4"
},
"isAccessibleForFree": "False",
"hasPart":
{
"@type": "WebPageElement",
"isAccessibleForFree": "False",
"cssSelector" : ".paywallPdf"
}
}
</script>
<meta name="ifq:pageType" content="PRESENTATION_PAGE"/>
<script type="text/javascript">
InfoQConstants.pageType = 'PRESENTATION_PAGE';
</script>
<meta property="og:type" content="website" />
<meta property="og:image" content="https://res.infoq.com/presentations/microservices-failure-modes/en/headerimage/Adrian-Cockcroft-QConSF2019-h-1578439528337.jpg"/>
<meta property="twitter:image" content="https://res.infoq.com/presentations/microservices-failure-modes/en/headerimage/Adrian-Cockcroft-QConSF2019-h-1578439528337.jpg"/>
<meta property="og:title" content="Managing Failure Modes in Microservice Architectures "/>
<meta property="og:description" content="Adrian Cockcroft explores how to apply some industry standard techniques (including Failure Modes and Effects Analysis) to cloud native microservices architectures. He looks at how chaos engineering techniques are driving the industry from annual datacenter disaster recovery testing of monolithic applications to continuous resilience assurance for cloud native microservices." />
<meta property="og:site_name" content="InfoQ"/>
<meta property="og:url" content="https://www.infoq.com/presentations/microservices-failure-modes/"/>
<meta name="twitter:card" content="summary_large_image" />
<meta name="twitter:title" content="Managing Failure Modes in Microservice Architectures "/>
<meta name="twitter:description" content="Adrian Cockcroft explores how to apply some industry standard techniques (including Failure Modes and Effects Analysis) to cloud native microservices architectures. He looks at how chaos engineering techniques are driving the industry from annual datacenter disaster recovery testing of monolithic applications to continuous resilience assurance for cloud native microservices." />
<link rel="image_src" href="https://res.infoq.com/presentations/microservices-failure-modes/en/headerimage/Adrian-Cockcroft-QConSF2019-h-1578439528337.jpg"/>
<meta name="keywords" content="microservices failure modes,Architecture &amp; Design,QCon San Francisco 2019,Resilience,Transcripts,Microservices,QCon Software Development Conference,Cloud Native Architecture,Architecture,InfoQ,Cloud,"/>
<meta name="description" content="Adrian Cockcroft explores how to apply some industry standard techniques (including Failure Modes and Effects Analysis) to cloud native microservices architectures."/>
<meta name="tprox" content="1578418500000" />
<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/swfobject/2.1/swfobject.js"></script>
<script type="text/javascript">
var slides = new Array('https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl1-1578439542817.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl2-1578439541602.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl3-1578439543122.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl4-1578439538846.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl5-1578439540278.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl6-1578439536535.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl7-1578439532963.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl8-1578439530956.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl9-1578439529025.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl10-1578439530205.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl11-1578439528757.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl12-1578439530706.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl13-1578439541890.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl14-1578439543742.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl15-1578439540523.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl16-1578439538080.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl17-1578439535040.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl18-1578439532713.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl19-1578439533997.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl20-1578439532417.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl21-1578439533234.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl22-1578439537062.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl23-1578439535771.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl24-1578439537819.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl25-1578439539785.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl26-1578439543996.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl27-1578439542145.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl28-1578439531322.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl29-1578439529300.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl30-1578439541332.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl31-1578439543392.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl32-1578439539235.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl33-1578439540807.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl34-1578439534765.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl35-1578439537328.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl36-1578439535312.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl37-1578439531882.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl38-1578439530459.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl39-1578439544531.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl40-1578439536258.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl41-1578439538340.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl42-1578439539536.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl43-1578439541056.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl44-1578439544252.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl45-1578439529925.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl46-1578439532164.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl47-1578439534493.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl48-1578439537572.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl49-1578439534248.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl50-1578439540033.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl51-1578439533506.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl52-1578439536814.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl53-1578439536014.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl54-1578439529641.jpg','https://imgopt.infoq.com/fit-in/1288x0/filters:quality(80)/presentations/microservices-failure-modes/en/slides/sl55-1578439531626.jpg'),
TIMES = new Array(0,25,74,134,174,193,231,310,355,420,557,668,684,817,927,987,1017,1044,1084,1131,1193,1246,1288,1381,1548,1662,1693,1749,1792,1859,1987,2099,2121,2148,2176,2206,2233,2248,2332,2356,2382,2402,2524,2638,2674,2729,2778,2806,2815,2853,2927,2979,3026,3038,3044,3093),
EXPRESSINSTALL_SWF = 'https://cdn.infoq.com/statics_s1_20260910111632/scripts/video/flash/expressinstall.swf',
SVMPLAYER_SWF = 'https://cdn.infoq.com/statics_s1_20260910111632/scripts/video/flash/svmplayer.swf',
isWideScreen = true,
// i18n text for the to full view/minimize button
i18nFullScreen = 'Full',
i18nMinimize = 'Minimize',
demoTimings = '';
var playerSplash = 'https://cdn.infoq.com/statics_s1_20260910111632/styles/static/images/logo/logo_scrubber_16_9.jpg';
var vhf = 'cnRtcGU6Ly92aWRlb2YuaW5mb3EuY29tL2NmeC9zdC8=';
var jsclassref = 'cHJlc2VudGF0aW9ucy8xOS1ub3YtbWFuYWdpbmdmYWlsdXJlbW9kZXNpbm0tMi5tcDQ=';
</script>
<script type="text/javascript" src="https://cdn.infoq.com/statics_s1_20260910111632/scripts/video/player/PlayerPresentation_mobile.js"></script>
<script type="text/javascript">
$(document).ready(function() {
init();
});
</script>
<script type='text/javascript' src='https://cdn.infoq.com/statics_s1_20260910111632/scripts/video/videoPlayercombinedJsPresentation.js'></script>
<script type='text/javascript' src='https://cdn.infoq.com/statics_s1_20260910111632/scripts/video/PresentationUI.js'></script>
<script type='text/javascript' src='https://cdn.infoq.com/statics_s1_20260910111632/scripts/ui/jquery.tipsy.js'></script>
<link rel="stylesheet" href="https://cdn.infoq.com/statics_s1_20260910111632/styles/tipsy.css" type="text/css" />
<script type="text/javascript">
P.s = 'https://videoh.infoq.com/presentations/19-nov-managingfailuremodesinm-2.mp4';
</script>
<script type="text/javascript" src="https://cdn.infoq.com/statics_s1_20260910111632/scripts/relatedVcr.min.js"></script>
<script type="application/javascript">
var communityIds = "2498";
var topicIds = "6446,3450,6218,3960,593,6526,876,131,2961";
VCR.loadAllVcrs(communityIds, topicIds);
</script>
<script type="text/javascript" src='https://cdn.infoq.com/statics_s1_20260910111632/scripts/subscription.js'></script>
<script type="text/javascript" src="https://cdn.infoq.com/statics_s1_20260910111632/scripts/infoq.js"></script>
<script type="text/javascript">
document.addEventListener('DOMContentLoaded', function() {
if (!window || !window.infoq) return
infoq.init()
})
</script>
<script defer data-domain="infoq.com" src="https://plausible.io/js/script.file-downloads.hash.outbound-links.pageview-props.tagged-events.js"></script>
<script>window.plausible = window.plausible || function() { (window.plausible.q = window.plausible.q || []).push(arguments) }</script>
</head>
<body >
<!-- Google Tag Manager (noscript) -->
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-W9GJ5DL"
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager (noscript) -->
<div class="intbt">
<a href="/int/bt/" title="bt">BT</a>
</div>
<script type="text/javascript">
var allCountries = [{"id":3,"name":"Afghanistan"},{"id":244,"name":"Åland"},{"id":6,"name":"Albania"},{"id":61,"name":"Algeria"},{"id":13,"name":"American Samoa"},{"id":1,"name":"Andorra"},{"id":9,"name":"Angola"},{"id":5,"name":"Anguilla"},{"id":11,"name":"Antarctica"},{"id":4,"name":"Antigua and Barbuda"},{"id":12,"name":"Argentina"},{"id":7,"name":"Armenia"},{"id":16,"name":"Aruba"},{"id":15,"name":"Australia"},{"id":14,"name":"Austria"},{"id":17,"name":"Azerbaijan"},{"id":31,"name":"Bahamas"},{"id":24,"name":"Bahrain"},{"id":20,"name":"Bangladesh"},{"id":19,"name":"Barbados"},{"id":35,"name":"Belarus"},{"id":21,"name":"Belgium"},{"id":36,"name":"Belize"},{"id":26,"name":"Benin"},{"id":27,"name":"Bermuda"},{"id":32,"name":"Bhutan"},{"id":29,"name":"Bolivia"},{"id":254,"name":"Bonaire, Sint Eustatius, and Saba"},{"id":18,"name":"Bosnia and Herzegovina"},{"id":34,"name":"Botswana"},{"id":33,"name":"Bouvet Island"},{"id":30,"name":"Brazil"},{"id":104,"name":"British Indian Ocean Territory"},{"id":28,"name":"Brunei Darussalam"},{"id":23,"name":"Bulgaria"},{"id":22,"name":"Burkina Faso"},{"id":25,"name":"Burundi"},{"id":114,"name":"Cambodia"},{"id":46,"name":"Cameroon"},{"id":37,"name":"Canada"},{"id":52,"name":"Cape Verde"},{"id":121,"name":"Cayman Islands"},{"id":40,"name":"Central African Republic"},{"id":207,"name":"Chad"},{"id":45,"name":"Chile"},{"id":47,"name":"China"},{"id":53,"name":"Christmas Island"},{"id":38,"name":"Cocos (Keeling) Islands"},{"id":48,"name":"Colombia"},{"id":116,"name":"Comoros"},{"id":39,"name":"Congo (Democratic Republic)"},{"id":41,"name":"Congo (People\u0027s Republic)"},{"id":44,"name":"Cook Islands"},{"id":49,"name":"Costa Rica"},{"id":43,"name":"Cote D\u0027Ivoire"},{"id":97,"name":"Croatia"},{"id":51,"name":"Cuba"},{"id":253,"name":"Curaçao"},{"id":54,"name":"Cyprus"},{"id":55,"name":"Czech Republic"},{"id":58,"name":"Denmark"},{"id":57,"name":"Djibouti"},{"id":59,"name":"Dominica"},{"id":60,"name":"Dominican Republic"},{"id":213,"name":"East Timor"},{"id":62,"name":"Ecuador"},{"id":64,"name":"Egypt"},{"id":203,"name":"El Salvador"},{"id":87,"name":"Equatorial Guinea"},{"id":66,"name":"Eritrea"},{"id":63,"name":"Estonia"},{"id":68,"name":"Ethiopia"},{"id":72,"name":"Falkland Islands (Malvinas)"},{"id":74,"name":"Faroe Islands"},{"id":71,"name":"Fiji"},{"id":70,"name":"Finland"},{"id":75,"name":"France"},{"id":80,"name":"French Guiana"},{"id":170,"name":"French Polynesia"},{"id":208,"name":"French Southern Territories"},{"id":76,"name":"Gabon"},{"id":84,"name":"Gambia"},{"id":79,"name":"Georgia"},{"id":56,"name":"Germany"},{"id":81,"name":"Ghana"},{"id":82,"name":"Gibraltar"},{"id":88,"name":"Greece"},{"id":83,"name":"Greenland"},{"id":78,"name":"Grenada"},{"id":86,"name":"Guadeloupe"},{"id":91,"name":"Guam"},{"id":90,"name":"Guatemala"},{"id":249,"name":"Guernsey"},{"id":85,"name":"Guinea"},{"id":92,"name":"Guinea-Bissau"},{"id":93,"name":"Guyana"},{"id":98,"name":"Haiti"},{"id":95,"name":"Heard Island and McDonald Islands"},{"id":96,"name":"Honduras"},{"id":94,"name":"Hong Kong"},{"id":99,"name":"Hungary"},{"id":107,"name":"Iceland"},{"id":103,"name":"India"},{"id":100,"name":"Indonesia"},{"id":106,"name":"Iran"},{"id":105,"name":"Iraq"},{"id":101,"name":"Ireland"},{"id":245,"name":"Isle of Man"},{"id":102,"name":"Israel"},{"id":108,"name":"Italy"},{"id":109,"name":"Jamaica"},{"id":111,"name":"Japan"},{"id":250,"name":"Jersey"},{"id":110,"name":"Jordan"},{"id":122,"name":"Kazakhstan"},{"id":112,"name":"Kenya"},{"id":115,"name":"Kiribati"},{"id":243,"name":"Kosovo"},{"id":120,"name":"Kuwait"},{"id":113,"name":"Kyrgyzstan"},{"id":123,"name":"Laos"},{"id":132,"name":"Latvia"},{"id":124,"name":"Lebanon"},{"id":129,"name":"Lesotho"},{"id":128,"name":"Liberia"},{"id":133,"name":"Libya"},{"id":126,"name":"Liechtenstein"},{"id":130,"name":"Lithuania"},{"id":131,"name":"Luxembourg"},{"id":143,"name":"Macau"},{"id":139,"name":"Macedonia"},{"id":137,"name":"Madagascar"},{"id":151,"name":"Malawi"},{"id":153,"name":"Malaysia"},{"id":150,"name":"Maldives"},{"id":140,"name":"Mali"},{"id":148,"name":"Malta"},{"id":138,"name":"Marshall Islands"},{"id":145,"name":"Martinique"},{"id":146,"name":"Mauritania"},{"id":149,"name":"Mauritius"},{"id":238,"name":"Mayotte"},{"id":152,"name":"Mexico"},{"id":73,"name":"Micronesia"},{"id":136,"name":"Moldova"},{"id":135,"name":"Monaco"},{"id":142,"name":"Mongolia"},{"id":246,"name":"Montenegro"},{"id":147,"name":"Montserrat"},{"id":134,"name":"Morocco"},{"id":154,"name":"Mozambique"},{"id":141,"name":"Myanmar"},{"id":155,"name":"Namibia"},{"id":164,"name":"Nauru"},{"id":163,"name":"Nepal"},{"id":161,"name":"Netherlands"},{"id":8,"name":"Netherlands Antilles"},{"id":156,"name":"New Caledonia"},{"id":166,"name":"New Zealand"},{"id":160,"name":"Nicaragua"},{"id":157,"name":"Niger"},{"id":159,"name":"Nigeria"},{"id":165,"name":"Niue"},{"id":158,"name":"Norfolk Island"},{"id":118,"name":"North Korea"},{"id":144,"name":"Northern Mariana Islands"},{"id":162,"name":"Norway"},{"id":167,"name":"Oman"},{"id":173,"name":"Pakistan"},{"id":180,"name":"Palau"},{"id":178,"name":"Palestinian Territory"},{"id":168,"name":"Panama"},{"id":171,"name":"Papua New Guinea"},{"id":181,"name":"Paraguay"},{"id":169,"name":"Peru"},{"id":172,"name":"Philippines"},{"id":176,"name":"Pitcairn"},{"id":174,"name":"Poland"},{"id":179,"name":"Portugal"},{"id":177,"name":"Puerto Rico"},{"id":182,"name":"Qatar"},{"id":183,"name":"Reunion"},{"id":184,"name":"Romania"},{"id":185,"name":"Russian Federation"},{"id":186,"name":"Rwanda"},{"id":193,"name":"Saint Helena"},{"id":117,"name":"Saint Kitts and Nevis"},{"id":125,"name":"Saint Lucia"},{"id":251,"name":"Saint Martin"},{"id":175,"name":"Saint Pierre and Miquelon"},{"id":229,"name":"Saint Vincent and the Grenadines"},{"id":247,"name":"Saint-Barthélemy"},{"id":236,"name":"Samoa"},{"id":198,"name":"San Marino"},{"id":202,"name":"Sao Tome and Principe"},{"id":187,"name":"Saudi Arabia"},{"id":199,"name":"Senegal"},{"id":248,"name":"Serbia"},{"id":189,"name":"Seychelles"},{"id":197,"name":"Sierra Leone"},{"id":192,"name":"Singapore"},{"id":252,"name":"Sint Maarten"},{"id":196,"name":"Slovakia"},{"id":194,"name":"Slovenia"},{"id":188,"name":"Solomon Islands"},{"id":200,"name":"Somalia"},{"id":239,"name":"South Africa"},{"id":89,"name":"South Georgia and the South Sandwich Islands"},{"id":119,"name":"South Korea"},{"id":255,"name":"South Sudan"},{"id":67,"name":"Spain"},{"id":127,"name":"Sri Lanka"},{"id":190,"name":"Sudan"},{"id":201,"name":"Suriname"},{"id":195,"name":"Svalbard and Jan Mayen"},{"id":205,"name":"Swaziland"},{"id":191,"name":"Sweden"},{"id":42,"name":"Switzerland"},{"id":204,"name":"Syria"},{"id":220,"name":"Taiwan"},{"id":211,"name":"Tajikistan"},{"id":221,"name":"Tanzania"},{"id":210,"name":"Thailand"},{"id":209,"name":"Togo"},{"id":212,"name":"Tokelau"},{"id":216,"name":"Tonga"},{"id":218,"name":"Trinidad and Tobago"},{"id":215,"name":"Tunisia"},{"id":217,"name":"Turkey"},{"id":214,"name":"Turkmenistan"},{"id":206,"name":"Turks and Caicos Islands"},{"id":219,"name":"Tuvalu"},{"id":223,"name":"Uganda"},{"id":222,"name":"Ukraine"},{"id":2,"name":"United Arab Emirates"},{"id":77,"name":"United Kingdom"},{"id":224,"name":"United States Minor Outlying Islands"},{"id":226,"name":"Uruguay"},{"id":225,"name":"USA"},{"id":227,"name":"Uzbekistan"},{"id":234,"name":"Vanuatu"},{"id":228,"name":"Vatican City (Holy See)"},{"id":230,"name":"Venezuela"},{"id":233,"name":"Vietnam"},{"id":231,"name":"Virgin Islands (British)"},{"id":232,"name":"Virgin Islands (U.S.)"},{"id":235,"name":"Wallis and Futuna"},{"id":65,"name":"Western Sahara"},{"id":237,"name":"Yemen"},{"id":241,"name":"Zaire"},{"id":240,"name":"Zambia"},{"id":242,"name":"Zimbabwe"}];
var gdprCountriesIds = [196,194,191,184,179,174,161,148,132,131,130,108,101,99,97,88,77,75,70,67,63,58,56,55,54,37,23,21,14];
</script>
<section data-nosnippet class="section container subscribe-box hidden">
<div class="container__inner">
<div class="actions">
<div class="actions__left">
<h2>InfoQ Software Architects' Newsletter</h2>
<span><p>A monthly overview of things you need to know as an architect or aspiring architect.</p>
<p><a href="https://www.infoq.com/software-architects-newsletter#placeholderPastIssues">View an example</a></p>
</span>
<div class="newsletter__subscribe">
<form class="form gdpr" name="dataCollectCampaignNewsletterForm" id="dataCollectCampaignNewsletterForm" action="#" onsubmit="dataCollectNewsletter.saveSubscription(); return false;">
<div class="field newsletter__mail input__text input__no-label input__medium email">
<label for="email-dataCollectnewsletter-infoq" class="label field__label">Enter your e-mail address</label>
<input id="email-dataCollectnewsletter-infoq" name="footerNewsletterEmail" placeholder="Enter your e-mail address" class="input field__input" type="email"/>
<input type="text" name="emailH" id="input_email_h_d" aria-required="false" style="display:none !important" tabindex="-1" autocomplete="off"/>
<input type="hidden" id="fnt_d" name="fnt_d" value="3oCTVQJcuE725ucu"/>
<input type="hidden" id="dataCollectNewsletterType" name="dataCollectNewsletterType" value="regular"/>
<input type="hidden" id="cmpi_d" name="cmpi_d" value="4"/>
</div>
<div class="hidden">
<span class="input__select field country">
<label for="input-dataCollect-newsletter-country" class="label field__label">Select your country</label>
<select id="input-dataCollect-newsletter-country" class="select field__input">
<option value="" class="select__option">Select a country</option>
</select>
<p class="input__message field__desc"></p>
</span>
<span class="input__checkbox field hidden">
<input type="checkbox" id="gdpr-consent-campaign">
<label for="gdpr-consent-campaign" class="label"><span>I consent to InfoQ.com handling my data as explained in this <a href="https://www.infoq.com/privacy-notice">Privacy Notice</a>.</span></label>
</span>
</div>
<input type="submit" role="button" value="Subscribe" class="button button__medium button__red" onclick="return dataCollectNewsletter.validateEmail('Invalid email address');"/>
</form>
<p class="meta">
<a href="/privacy-notice/" target="_blank">We protect your privacy.</a>
</p>
<span class="success" style="display:none;" id="dataCollectNewsletterMessage"></span>
</div>
</div>
<div class="actions__right">
<button aria-label="Close" class="close closeBox button button__unstyled button__icon icon icon__close-black icon--only">Close</button>
</div>
</div>
</div>
</section>
<script type="text/javascript">
var dataCollectNewsletter = new Newsletter('Enter your e-mail address',
'email-dataCollectnewsletter-infoq', 'dataCollectNewsletterType','dataCollectNewsletterMessage', 'fnt_d', 'input_email_h_d', 'input-dataCollect-newsletter-country', 'cmpi_d','popup_all_pages');
</script>
<div class="infoq" id="infoq">
<!-- ####### SITE START ######### -->
<section class="section container promo hidden">
<div class="container__inner">
<div class="actions">
<div class="actions__left">
<p>
<span>Online InfoQ AI-Assisted Engineering Certification (Oct 19): Your agent got faster. Did your code get better?</span>
<a class="btn" href="/url/pb/b23b7e3c-f8b5-4f46-bbf3-cddf2aeb2a9e/" target="_blank" rel="nofollow">
Register Now
</a>
</p>
</div>
<div class="actions__right">
<button aria-label="Close"
class="close button button__unstyled button__icon icon icon__close-white icon--only close-top-promo">Close
</button>
</div>
</div>
</div>
</section>
<header class="header">
<button aria-label="Toggle Navigation" tabindex="0" class="burger header__toggle button">Toggle Navigation <span></span><span></span><span></span></button>
<div class="header__container container">
<div class="container__inner">
<div data-nosnippet class="actions header__top">
<div class="actions__left">
<p class="header__desc my-0">
Facilitating the Spread of Knowledge and Innovation in Professional Software Development
</p>
<div class="button__dropdown dropdown header__more my-0 dropdown__center">
<button aria-label="English edition" class="button button__unstyled button__small">English edition </button>
<div class="dropdown__holder">
<!---->
<div class="dropdown__content">
<ul class="no-style dropdown__nav languagesEdition">
<li class="active"><a href="#" onclick="return false;">English edition</a></li>
<li><a href="https://www.infoq.cn">Chinese edition</a></li>
<li><a href="/jp/">Japanese edition</a></li>
<li><a href="/fr/">French edition</a></li>
</ul>
</div>
<!---->
</div>
</div>
<a class="my-0 contribute-link" role="button" href="/write-for-infoq/" title="Write for InfoQ">
Write for InfoQ
</a>
</div>
<div class="actions__right">
<div>
<form id="searchForm" name="search-form" action="/search.action" enctype="multipart/form-data" class="search icon__search icon icon__green">
<div class="field search__bar input__text input__no-label input__small">
<label for="search" class="label field__label">Search</label>
<input name="queryString" type="text" id="search" value="" placeholder="Search" class="input field__input">
<input type="hidden" name="page" value="1"/>
<input type="hidden" size="15" name="searchOrder">
</div>
<input value="Search" type="submit" class="search__go">
</form>
</div>
<div class="action_button__group">
<a href="/reginit.action?" onclick="return UserActions_Login.addLoginInfo(this);" class="button button__green button__small">Register</a>
<a href="/social/keycloakLogin.action?fl=login" onclick="return UserActions_Login.addLoginInfo(this);" class="button button__small button__outline">Sign in</a>
</div>
<div class="modal_auth_required">
<div class="modal_auth_required__actions">
<h2 class="h2">Unlock the full InfoQ experience</h2>
<p>Unlock the full InfoQ experience by logging in! Stay updated with your favorite authors and topics, engage with content, and download exclusive resources.</p>
<a href="/social/keycloakLogin.action?fl=login" onclick="return UserActions_Login.addLoginInfo(this);" class="button button__has-transition">Log In</a>
<div class="separator"><span>or</span></div>
<h3 class="h3">Don't have an InfoQ account?</h3>
<a href="/reginit.action?" onclick="return UserActions_Login.addLoginInfo(this);" class="button__outline button__has-transition">Register</a>
</div>
<div class="modal_auth_required__content">
<ul class="newsletter__features">
<li><strong>Stay updated on topics and peers that matter to you</strong>Receive instant alerts on the latest insights and trends.</li>
<li><strong>Quickly access free resources for continuous learning</strong>Minibooks, videos with transcripts, and training materials.</li>
<li><strong>Save articles and read at anytime</strong>Bookmark articles to read whenever youre ready.</li>
</ul>
</div>
<form id="login-form" data-vv-scope="login" class="login__form form" action="#" data-gtm-form-interact-id="0">
<input type="hidden" id="loginWidgetOrigin" name="loginWidgetOrigin" value="mainLogin">
<input type="hidden" name="fromP13N" id="isP13n" value="">
<input type="hidden" name="fromP13NId" id="p13n-id" value="">
<input type="hidden" name="fromP13NType" id="p13n-type" value="">
</form>
</div>
</div>
</div>
<div class="header__middle">
<div class="logo header__logo">
<a href="/" class="logo__symbol active">
Logo - Back to homepage
</a>
</div>
<div class="content-items">
<a href="/news/" class="icon icon__news">News</a>
<a href="/articles/" class="icon icon__articles">Articles</a>
<a href="/presentations/" class="icon icon__presentations">Presentations</a>
<a href="/podcasts/" class="icon icon__podcasts">Podcasts</a>
<a href="/minibooks/" class="icon icon__guides">Guides</a>
</div>
<div class="header__items columns">
<div>
<h3 class="widget__heading">Topics</h3>
<nav class="nav header__nav topics" data-trk-ref="header_personas">
<div class="has--subnav li-nav">
<a href="/development/" title="Development" class="nav__category">Development</a>
<div class="nav__subnav subnav">
<ul class="subnav__categories no-style">
<li><a href="/java/" title="Java">Java</a></li>
<li><a href="/kotlin/" title="Kotlin">Kotlin</a></li>
<li><a href="/dotnet/" title=".Net">.Net</a></li>
<li><a href="/c_sharp/" title="C#">C#</a></li>
<li><a href="/swift/" title="Swift">Swift</a></li>
<li><a href="/golang/" title="Go">Go</a></li>
<li><a href="/rust/" title="Rust">Rust</a></li>
<li><a href="/javascript/" title="JavaScript">JavaScript</a></li>
</ul>
<div class="subnav__content" data-id="6815">
<div class="heading__container actions">
<div class="actions__left">
<h3 class="heading section__heading">Featured in Development</h3>
</div>
</div>
<ul data-size="large" data-horizontal="true" data-tax="" taxonomy="articles" class="cards no-style">
<li>
<div class="card__content">
<div class="card__data">
<h4 class="card__title">
<a href="/presentations/solopreneur-journey">A Solopreneur's Journey: From Engineer to Puzzle Master and Storyteller</a>
</h4>
<p class="card__excerpt">Joe Cassavaugh shares his journey from software engineer to successful solopreneur with a $2M+ indie franchise. He explains how he scaled production to 10 games in 5 years, adopted Unity to boost velocity 4-6x, optimized content pipelines, and leveraged refactoring patterns. He discusses key trade-offs between corporate engineering and solopreneurship for senior devs and leaders.</p>
<div class="card__footer"></div>
</div>
<a href="/presentations/solopreneur-journey" class="card__header">
<img loading="lazy" alt="A Solopreneur's Journey: From Engineer to Puzzle Master and Storyteller" src="https://imgopt.infoq.com/fit-in/100x100/filters:quality(80)/presentations/solopreneur-journey/en/smallimage/joe-cassavaugh-thumbnail-1788338336404.jpg" class="card__image"/>
</a>
</div>
</li>
</ul>
<a href="/development/" class="button__more button button__large button__arrow arrow__right">All in development</a>
</div>
</div>
</div>
<div class="has--subnav li-nav">
<a href="/architecture-design/" title="Architecture &amp; Design" class="nav__category">Architecture &amp; Design</a>
<div class="nav__subnav subnav">
<ul class="subnav__categories no-style">
<li><a href="/architecture/" title="Architecture">Architecture</a></li>
<li><a href="/enterprise-architecture/" title="Enterprise Architecture">Enterprise Architecture</a></li>
<li><a href="/performance-scalability/" title="Scalability/Performance">Scalability/Performance</a></li>
<li><a href="/design/" title="Design">Design</a></li>
<li><a href="/Case_Study/" title="Case Studies">Case Studies</a></li>
<li><a href="/microservices/" title="Microservices">Microservices</a></li>
<li><a href="/servicemesh/" title="Service Mesh">Service Mesh</a></li>
<li><a href="/DesignPattern/" title="Patterns">Patterns</a></li>
<li><a href="/Security/" title="Security">Security</a></li>
</ul>
<div class="subnav__content" data-id="6816">
<div class="heading__container actions">
<div class="actions__left">
<h3 class="heading section__heading">Featured in Architecture &amp; Design</h3>
</div>
</div>
<ul data-size="large" data-horizontal="true" data-tax="" taxonomy="articles" class="cards no-style">
<li>
<div class="card__content">
<div class="card__data">
<h4 class="card__title">
<a href="/minibooks/next-gen-architecture-ai-era">Next-Gen Architecture Playbook: Insights and Patterns for the AI Era</a>
</h4>
<p class="card__excerpt">This eMag examines how architects can lead with clarity in a rapidly evolving engineering world, distilling industry insights into field-tested practices for teams. Together, these stories reveal a core theme: the technology leader’s role is expanding from building systems to guiding how tech behaves and learns, while enabling engineers and organizations to bring out their best.</p>
<div class="card__footer"></div>
</div>
<a href="/minibooks/next-gen-architecture-ai-era" class="card__header">
<img loading="lazy" alt="Next-Gen Architecture Playbook: Insights and Patterns for the AI Era" src="https://imgopt.infoq.com/fit-in/100x100/filters:quality(80)/minibooks/next-gen-architecture-ai-era/en/smallimage/thumb-image-emag-next-gen-architecture-playbook-1787922032442.jpg" class="card__image"/>
</a>
</div>
</li>
</ul>
<a href="/architecture-design/" class="button__more button button__large button__arrow arrow__right">All in architecture-design</a>
</div>
</div>
</div>
<div class="has--subnav li-nav">
<a href="/ai-ml-data-eng/" title="AI Infrastructure" class="nav__category">AI Infrastructure</a>
<div class="nav__subnav subnav">
<ul class="subnav__categories no-style">
<li><a href="/bigdata/" title="Big Data">Big Data</a></li>
<li><a href="/machinelearning/" title="Machine Learning">Machine Learning</a></li>
<li><a href="/nosql/" title="NoSQL">NoSQL</a></li>
<li><a href="/database/" title="Database">Database</a></li>
<li><a href="/data-analytics/" title="Data Analytics">Data Analytics</a></li>
<li><a href="/streaming/" title="Streaming">Streaming</a></li>
</ul>
<div class="subnav__content" data-id="16690">
<div class="heading__container actions">
<div class="actions__left">
<h3 class="heading section__heading">Featured in AI, ML &amp; Data Engineering</h3>
</div>
</div>
<ul data-size="large" data-horizontal="true" data-tax="" taxonomy="articles" class="cards no-style">
<li>
<div class="card__content">
<div class="card__data">
<h4 class="card__title">
<a href="/articles/when-spec-driven-development-pays-off">When Spec-Driven Development Pays Off</a>
</h4>
<p class="card__excerpt">AI coding assistants have become a core part of software development. AI-generated code has shown productivity gains, but it's also contributing to security weaknesses and familiar bug patterns. In this article, author Nitin Garg highlights the bottleneck has moved from code generation to code verification, and how to detect &amp; mitigate it when the AI-generated behavior diverges from the intent.</p>
<div class="card__footer"></div>
</div>
<a href="/articles/when-spec-driven-development-pays-off" class="card__header">
<img loading="lazy" alt="When Spec-Driven Development Pays Off" src="https://imgopt.infoq.com/fit-in/100x100/filters:quality(80)/articles/when-spec-driven-development-pays-off/en/smallimage/When-Spec-Driven-Development-Pays-Off-thumb-1788790605567.jpg" class="card__image"/>
</a>
</div>
</li>
</ul>
<a href="/ai-ml-data-eng/" class="button__more button button__large button__arrow arrow__right">All in ai-ml-data-eng</a>
</div>
</div>
</div>
<div class="has--subnav li-nav">
<a href="/culture-methods/" title="Culture &amp; Methods" class="nav__category">Culture &amp; Methods</a>
<div class="nav__subnav subnav">
<ul class="subnav__categories no-style">
<li><a href="/agile/" title="Agile">Agile</a></li>
<li><a href="/diversity/" title="Diversity">Diversity</a></li>
<li><a href="/leadership/" title="Leadership">Leadership</a></li>
<li><a href="/lean/" title="Lean/Kanban">Lean/Kanban</a></li>
<li><a href="/personal-growth/" title="Personal Growth">Personal Growth</a></li>
<li><a href="/scrum/" title="Scrum">Scrum</a></li>
<li><a href="/sociocracy/" title="Sociocracy">Sociocracy</a></li>
<li><a href="/software_craftsmanship/" title="Software Craftmanship">Software Craftmanship</a></li>
<li><a href="/team-collaboration/" title="Team Collaboration">Team Collaboration</a></li>
<li><a href="/testing/" title="Testing">Testing</a></li>
<li><a href="/ux/" title="UX">UX</a></li>
</ul>
<div class="subnav__content" data-id="6817">
<div class="heading__container actions">
<div class="actions__left">
<h3 class="heading section__heading">Featured in Culture &amp; Methods</h3>
</div>
</div>
<ul data-size="large" data-horizontal="true" data-tax="" taxonomy="articles" class="cards no-style">
<li>
<div class="card__content">
<div class="card__data">
<h4 class="card__title">
<a href="/presentations/rust-refactoring">Accelerating Performance by Incrementally Integrating Rust Into Existing Codebase</a>
</h4>
<p class="card__excerpt">Lily Mara explains how to avoid high-risk software rewrites through incremental FFI refactoring. She shares how engineering teams can replace Python bottlenecks with Rust via PyO3, demonstrating how to achieve dramatic function-level speedups, seamless integration testing, and meaningful infrastructure cost savings without microservice overhead.</p>
<div class="card__footer"></div>
</div>
<a href="/presentations/rust-refactoring" class="card__header">
<img loading="lazy" alt="Accelerating Performance by Incrementally Integrating Rust Into Existing Codebase" src="https://imgopt.infoq.com/fit-in/100x100/filters:quality(80)/presentations/rust-refactoring/en/smallimage/lily-mara-thumbnail-1788338447565.jpeg" class="card__image"/>
</a>
</div>
</li>
</ul>
<a href="/culture-methods/" class="button__more button button__large button__arrow arrow__right">All in culture-methods</a>
</div>
</div>
</div>
<div class="has--subnav li-nav">
<a href="/devops/" class="nav__category">DevOps</a>
<div class="nav__subnav subnav">
<ul class="subnav__categories no-style">
<li><a href="/infrastructure/" title="Infrastructure">Infrastructure</a></li>
<li><a href="/continuous_delivery/" title="Continuous Delivery">Continuous Delivery</a></li>
<li><a href="/automation/" title="Automation">Automation</a></li>
<li><a href="/containers/" title="Containers">Containers</a></li>
<li><a href="/cloud-computing/" title="Cloud">Cloud</a></li>
<li><a href="/observability/" title="Observability">Observability</a></li>
</ul>
<div class="subnav__content" data-id="6043">
<div class="heading__container actions">
<div class="actions__left">
<h3 class="heading section__heading">Featured in DevOps</h3>
</div>
</div>
<ul data-size="large" data-horizontal="true" data-tax="" taxonomy="articles" class="cards no-style">
<li>
<div class="card__content">
<div class="card__data">
<h4 class="card__title">
<a href="/presentations/ai-platform-engineering-roundtable">Platform Engineering in the Age of AI</a>
</h4>
<p class="card__excerpt">The panelists explain how platform teams adapt to support AI-assisted engineering, highlighting which capabilities belong in the platform. They discuss trade-offs between standardization and developer autonomy, while sharing strategies to manage AI tooling, security guardrails, and shifting workflows.</p>
<div class="card__footer"></div>
</div>
<a href="/presentations/ai-platform-engineering-roundtable" class="card__header">
<img loading="lazy" alt="Platform Engineering in the Age of AI" src="https://imgopt.infoq.com/fit-in/100x100/filters:quality(80)/presentations/ai-platform-engineering-roundtable/en/smallimage/thumb-1788861349861.jpg" class="card__image"/>
</a>
</div>
</li>
</ul>
<a href="/devops/" class="button__more button button__large button__arrow arrow__right">All in devops</a>
</div>
</div>
</div>
<div class="li-nav">
<a rel="noreferrer noopener" href="https://events.infoq.com/" class="nav__category" title="Events" target="_blank">Events</a>
</div>
</nav>
</div>
<div>
<h3 class="widget__heading">Helpful links</h3>
<ul class="no-style header__nav">
<li>
<a href="/about-infoq" title="About InfoQ">
About InfoQ
</a>
</li>
<li>
<a href="/infoq-editors" title="InfoQ Editors">
InfoQ Editors
</a>
</li>
<li>
<a href="/write-for-infoq" title="Write for InfoQ">
Write for InfoQ
</a>
</li>
<li>
<a href="https://c4media.com/" target="_blank" title="About C4Media">
About C4Media
</a>
</li>
<li>
<a rel="noreferrer noopener" href="https://c4media.com/diversity" title="Diversity" target="_blank">Diversity</a>
</li>
</ul>
</div>
<div>
<h3 class="widget__heading">Choose your language</h3>
<ul class="language__switcher no-style">
<li class="active"><a href="#" onclick="return false;" title="InfoQ English">En</a></li>
<li><a href="https://www.infoq.cn">中文</a></li>
<li><a href="/jp/">日本</a></li>
<li><a href="/fr/">Fr</a></li>
</ul>
</div>
</div>
</div>
<div data-nosnippet class="actions header__bottom header__bottom__events">
<div class="actions__left">
<div class="events-list" role="list" aria-label="Upcoming events">
<div class="event-item" role="listitem" data-event-type="certification" data-sort-date="2026-09-14">
<a class="event-card" href="https://certification.qconferences.com/architecture?utm_source=infoq&amp;utm_medium=referral&amp;utm_campaign=homepageheader_onlinecohortarchitecture26" target="_blank" rel="noopener noreferrer" aria-label="Online InfoQ Architect Certification, Sep 14, 2026">
<div class="event-date">
<div class="event-date-box" aria-hidden="true"><span>Sep</span><span>14,</span><span>2026</span></div>
</div>
<div class="event-content">
<p class="event-title">Architect Certification</p>
<p class="event-description">Distributed systems, decentralized decisions, platform engineering, and AI architecture. <br/><strong>Online. Register Now.</strong></p>
</div>
</a>
</div>
<div class="event-item" role="listitem" data-event-type="certification" data-sort-date="2026-09-18">
<a class="event-card" href="https://certification.qconferences.com/ai-assisted-engineering?utm_source=infoq&utm_medium=referral&utm_campaign=homepageheader_onlinecohortaiassistedengineering26" target="_blank" rel="noopener noreferrer" aria-label="AI-Assisted Engineering Certification, Sep 18, 2026">
<div class="event-date">
<div class="event-date-box" aria-hidden="true"><span>Sep</span><span>18,</span><span>2026</span></div>
</div>
<div class="event-content">
<p class="event-title">AI-Assisted Engineering Certification</p>
<p class="event-description">Your coding agent moves fast. Who checks whether it's making the codebase better? <br/><strong>Online. Register Now.</strong></p>
</div>
</a>
</div>
<div class="event-item" role="listitem" data-event-type="certification" data-sort-date="2026-10-14">
<a class="event-card" href="https://certification.qconferences.com/ai-security-privacy?utm_source=infoq&amp;utm_medium=referral&amp;utm_campaign=homepageheader_onlinecohortaisecurity26" target="_blank" rel="noopener noreferrer" aria-label="Online InfoQ AI Security &amp; Privacy Engineering Certification, August 26, 2026">
<div class="event-date">
<div class="event-date-box" aria-hidden="true"><span>Oct</span><span>14,</span><span>2026</span></div>
</div>
<div class="event-content">
<p class="event-title">AI Security &amp; Privacy Engineering Certification</p>
<p class="event-description">Secure and govern production AI systems, from sensitive data to guardrails, evals, and audits.<br><strong>Online. Register now.</strong></p>
</div>
</a>
</div>
<div class="event-item" role="listitem" data-event-type="qcon" data-sort-date="2026-11-16">
<a class="event-card" href="https://qconsf.com/?utm_source=infoq&amp;utm_medium=referral&amp;utm_campaign=homepageheader_qsf26" target="_blank" rel="noopener noreferrer" aria-label="QCon San Francisco, Nov 16-20, 2026">
<div class="event-date">
<div class="event-date-box" aria-hidden="true"><span>Nov</span><span>16-20,</span><span>2026</span></div>
</div>
<div class="event-content">
<p class="event-title">QCon San Francisco</p>
<p class="event-description">What's working across AI, architecture, and leadership, from the teams doing it. <br/><strong>Register. Early bird ends August 11.</strong></p>
</div>
</a>
</div>
<div class="event-item" role="listitem" data-event-type="qcon" data-sort-date="2026-12-15">
<a class="event-card" href="https://newyork.qcon.ai/?utm_source=infoq&utm_medium=referral&utm_campaign=homepageheader_qainy26" target="_blank" rel="noopener noreferrer" aria-label="QCon AI New York, December 15-16, 2026">
<div class="event-date">
<div class="event-date-box" aria-hidden="true"><span>Dec</span><span>15-16,</span><span>2026</span></div>
</div>
<div class="event-content">
<p class="event-title">QCon AI New York</p>
<p class="event-description">Production AI across agents, context, evals, security, and infrastructure, from the senior engineers building it.<br/><strong>Registration open.</strong></p>
</div>
</a>
</div>
<div class="event-item" role="listitem" data-event-type="qcon" data-sort-date="2027-04-13">
<a class="event-card" href="https://qconlondon.com/?utm_source=infoq&amp;utm_medium=referral&amp;utm_campaign=homepageheader_qconlondon27" target="_blank" rel="noopener noreferrer" aria-label="QCon London, April 13-16, 2027">
<div class="event-date">
<div class="event-date-box" aria-hidden="true"><span>Apr</span><span>13-16,</span><span>2027</span></div>
</div>
<div class="event-content">
<p class="event-title">QCon London</p>
<p class="event-description">What early-adopter teams have proven in production, across 15 engineering tracks. <br/><strong> Register. Early bird ends August 11.</strong></p>
</div>
</a>
</div>
</div>
</div>
<!---->
</div>
</div>
</div>
</header>
<!-- ####### CONTENT START ######### -->
<main>
<div>
<section class="container section white">
<div class="container__inner">
<article data-type="presentation" class="article grid transcript rightbar">
<p class="crumbs">
<span data-nosnippet><a href="/" title="InfoQ Homepage">InfoQ Homepage</a></span>
<span data-nosnippet><a href="/presentations" title="Presentations">Presentations</a></span>
<span data-nosnippet class="active">Managing Failure Modes in Microservice Architectures</span>
</p>
<div class="article__category architecture-design">
<a
href="/architecture-design/"
class="article__tag box--info" title="Architecture &amp; Design">
Architecture &amp; Design
</a>
</div>
<div data-nosnippet class="notice">
<div class="box--warning">
<a target="_blank" rel="nofollow" href="https://qconsf.com/?utm_source=infoq&amp;utm_medium=referral&amp;utm_campaign=infoqyellowbox_qsf26 ">QCon San Francisco (Nov 16-20): Deep technical sessions. Peer conversations that change how you think. </a>
</div>
</div>
<h1>Managing Failure Modes in Microservice Architectures</h1>
<div class="article__actions">
<div class="actions__left actions__sidebar article__actions">
<button id="toggleLikeContent" class="icon button button__icon like button__gray icon__like" aria-label="Like">Like</button>
<div class="bookmarkReading">
<button id="bookmarkBtn" data-ref="bookmarkPage" aria-label="Read later" class="login button__gray button button__icon icon icon__bookmark icon--only"> </button>
<!-- new dropdown rading list -->
<div class="my-0 button__dropdown dropdown reading_list dropdown__center" id="showBookmarks">
<button aria-label="Reading list" class="button button__unstyled button__small"></button>
<div class="dropdown__holder">
<div class="dropdown__content">
<ul class="no-style dropdown__nav">
<li class="active"><a href="/showbookmarks.action">Reading list</a></li>
</ul>
</div>
</div>
</div>
<div id="toastContainer" class="toast-none toast-hide">
<div class="toast"><span></span></div>
</div>
</div>
<script type="text/javascript">
if(loggedIn){
$('#showBookmarks').css('display', 'flex');
}
function performBookmark() {
Bookmarks.toggleBookmark('presentations', 'microservices-failure-modes');
}
infoq.event.on('bookmarkRequested', function(e) {
Bookmarks.toggleBookmark('presentations', 'microservices-failure-modes');
});
infoq.event.on("loaded", function(){
if(loggedIn){
var href = window.location.href;
if(href.indexOf("#bookmarkPage") != -1){
$('#bookmarkBtn').click();
}
}
});
$(document).ready(function() {
if(Bookmarks.isContentBookmarked == 'true'){
$('#bookmarkBtn').addClass('button__green');
$('#bookmarkBtn').removeClass('button__gray');
}else{
$('#bookmarkBtn').removeClass('button__green');
$('#bookmarkBtn').addClass('button__gray');
}
});
</script>
</div>
</div>
<div class="presentation__switch">
<label class="label">View Presentation</label>
<ul class="no-style">
<li><button class="button button__unstyled button__tooltip tooltip__top button__icon icon icon__pres-2-cols icon--only active" aria-label="Vertical" data-tip="Vertical" data-tip-placement="top" data-key="pres-2-cols">Vertical</button></li>
<li><button class="button button__unstyled button__tooltip tooltip__top button__icon icon icon__pres-1-col icon--only" aria-label="Horizontal" data-tip="Horizontal" data-tip-placement="top" data-key="pres-1-col">Horizontal</button></li>
<li><button class="button button__unstyled button__tooltip tooltip__top button__icon icon icon__pres-fs icon--only" aria-label="Full" data-tip="Full" data-tip-placement="top" data-key="pres-fs">Full</button></li>
</ul>
</div>
<div class="speed">
<label class="label">Speed:</label>
<ul id="presentation_speed" class="no-style presentation_speed">
<li><button class="button button__unstyled active" aria-label="1x" onclick="P.adjustSpeed(1.0, $(this));return false;">1x</button></li>
<li><button class="button button__unstyled" aria-label="1.25x" onclick="P.adjustSpeed(1.25, $(this));return false;">1.25x</button></li>
<li><button class="button button__unstyled" aria-label="1.5x" onclick="P.adjustSpeed(1.5, $(this));return false;">1.5x</button></li>
<li><button class="button button__unstyled" aria-label="2x" onclick="P.adjustSpeed(2.0, $(this));return false;">2x</button></li>
</ul>
</div>
<div class="player">
<div id="player">
<script type="text/javascript">
SlideSynchronizer.setTimes(TIMES);
SlideSynchronizer.playerId="video";
</script>
<video id="video" playsinline controls controlsList="nodownload" width="100%" height="100%" onplay="SlideSynchronizer.start();"></video>
</div>
<div class="actions player__actions">
<div class="actions__left">
<label class="label">Download</label>
<ul class="no-style download">
<li data-ref="downloadPresentationMp3">
<button id="mp3" class="button button__unstyled download" data-type="mp3" aria-label="MP3">MP3</button>
</li>
<form method="get" action="/mp3download.action" target="_blank" id="mp3Form">
<input type="hidden" name="filename" value="presentations/infoq-19-nov-managingfailuremodesinm-2.mp3"/>
</form>
<li data-ref="downloadPdf">
<button id="slides" class="button button__unstyled download" data-type="slides" aria-label="Slides">Slides</button>
</li>
<form method="get" action="/pdfdownload.action" target="_blank" id="pdfForm">
<input type="hidden" name="filename" value="presentations/QConSF2019-Adrian-Cockcroft-Managing-Failure-Modes-in-Microservice-Architectures.pdf"/>
</form>
</ul>
</div>
<div class="actions__right"><span>51:33</span></div>
</div>
</div>
<div id="slideContainer" class="intro slides">
<div id="slide"></div>
</div>
<div class="summary">
<h2 class="heading">Summary</h2>
<p>Adrian Cockcroft explores how to apply some industry standard techniques (including Failure Modes and Effects Analysis) to cloud native microservices architectures. He looks at how chaos engineering techniques are driving the industry from annual datacenter disaster recovery testing of monolithic applications to continuous resilience assurance for cloud native microservices.</p>
</div>
<div data-nosnippet class="bio">
<h2 class="heading">Bio</h2>
<p>Adrian Cockcroft joined Amazon as their VP of Cloud Architecture Strategy in 2016 and leads their open source community engagement team. He played a crucial role in developing the cloud ecosystem as Cloud Architect at Netflix and later as a Technology Fellow at Battery Ventures. Prior to this, he held positions as Distinguished Engineer at eBay and Sun Microsystems.</p>
</div>
<div data-nosnippet class="about-conference">
<h2 class="heading">About the conference</h2>
<p>Software is changing the world. QCon empowers software development by facilitating the spread of knowledge and innovation in the developer community. A practitioner-driven conference, QCon is designed for technical team leads, architects, engineering directors, and project managers who influence innovation in their teams.</p>
</div>
<div data-nosnippet class="event__list-box box__border">
<h3>INFOQ EVENTS</h3>
<ul class="cards no-style webinarEvents">
<li data-col="1/1" data-type="webinar" hoverables=""><img alt="" data-src="sponsorship/eventsnotice/4b6b261f-58c2-44bd-856a-2189a99d6743/resources/1HarnessWebinarSept17-Transcript-1786634790914.png" style="width: 288px; height: 145px;" src="https://imgopt.infoq.com/fit-in/3000x4000/filters:quality(85)/filters:no_upscale()/sponsorship/eventsnotice/4b6b261f-58c2-44bd-856a-2189a99d6743/resources/1HarnessWebinarSept17-Transcript-1786634790914.png" rel="share"> <span class="date">September 17th, 2026, 1 PM EDT</span>
<h4 class="title"><a href="https://www.infoq.com/url/t/3370b6d4-7e2c-4875-8258-5601b235e3c3/?label=Harness-Transcripts" rel="nofollow" target="_blank">Beyond the PR: The New Control Plane for Agentic Software Delivery</a></h4>
<p class="card__excerpt"><a href="https://www.infoq.com/url/t/3370b6d4-7e2c-4875-8258-5601b235e3c3/?label=Harness-Transcripts" rel="nofollow" target="_blank">Presented by: Mohit Suman - Staff Product Manager at Harness</a></p>
<footer>
<div class="authors"><span><a data-id="author-SPONSORED BY SPONSOR" href="https://www.infoq.com/url/t/3370b6d4-7e2c-4875-8258-5601b235e3c3/?label=Harness-Transcripts" rel="nofollow" target="_blank">SPONSORED BY HARNESS</a></span></div>
</footer>
</li>
</ul>
<ul class="cards no-style webinarEvents">
<li data-col="1/1" data-type="webinar" hoverables=""><img alt="" data-src="sponsorship/eventsnotice/f2bd6c51-c5d2-41cf-957d-198b2d3a83a3/resources/1DatadogWebinarOct8-Transcripts-1787826934657.png" style="width: 288px; height: 145px;" src="https://imgopt.infoq.com/fit-in/3000x4000/filters:quality(85)/filters:no_upscale()/sponsorship/eventsnotice/f2bd6c51-c5d2-41cf-957d-198b2d3a83a3/resources/1DatadogWebinarOct8-Transcripts-1787826934657.png" rel="share"> <span class="date">October 8th, 2026, 12 PM EDT</span>
<h4 class="title"><a href="https://www.infoq.com/url/t/517ac1f3-a6c1-442e-84ed-c00c52ee13d4/?label=Datadog-Transcripts" rel="nofollow" target="_blank">When AI Accelerates Development, Can Your CI Pipeline Keep Up?</a></h4>
<p class="card__excerpt"><a href="https://www.infoq.com/url/t/517ac1f3-a6c1-442e-84ed-c00c52ee13d4/?label=Datadog-Transcripts" rel="nofollow" target="_blank">Presented by: Eric Metaj - Product Marketing Manager, Software Delivery at Datadog, and Rohin Chandra - Product Manager, CI/CD Optimization at Datadog</a></p>
<footer>
<div class="authors"><span><a data-id="author-SPONSORED BY SPONSOR" href="https://www.infoq.com/url/t/517ac1f3-a6c1-442e-84ed-c00c52ee13d4/?label=Datadog-Transcripts" rel="nofollow" target="_blank">SPONSORED BY DATADOG</a></span></div>
</footer>
</li>
</ul>
</div>
<div class="article__content">
<div class="article__data">
<div id="presentationNotes">
<h2 class="expanded">Transcript</h2>
<p>Cockcroft: This is new content. It&#39;s pretty much the first time I&#39;ve presented it. I was working on my slides yesterday. This is really the first of an installment of things. I&#39;m setting out the groundwork for some work I&#39;m expecting to do a lot more on 2020. First, I&#39;m going to start with, why do I care about this? This is what&#39;s really happening. I&#39;m starting to see more of AWS customers, not just move like the front-end or the back-end for a mobile app or something like that to the cloud, they&#39;re closing all the data centers and they&#39;re moving everything to the cloud. This includes airlines - I&#39;m a power user of airlines, I want all the airlines to keep flying - finance, healthcare, manufacturing.</p>
<p>We have substantial businesses that are moving their critical infrastructure, and they&#39;re moving that to the cloud. As they do that, we&#39;ve got to develop the patterns that make that work reliably. We got to understand the failure modes. We&#39;ve got to have a much more sophisticated discussion about what could go wrong and what to do. That&#39;s why I&#39;m poking at this, and if anyone wants to talk about that if you&#39;re in a safety-critical industry, I&#39;m very happy to continue the conversation. In the past, we had disaster recovery. Back in the &#39;70s, in this study, you had a mainframe over here, and you could have another mainframe over there, and you&#39;d copy stuff back and forth. If something went wrong, you could switch to the other one. That was the traditional disaster recovery model. That then became more institutionalized and that&#39;s why I have a backup data center. I&#39;ll talk a bit more about that later.</p>
<p>Then, between Amazon and Netflix, and a few other people, we came up with chaos engineering, a decade or so ago. We were starting to induce failure. The difference in chaos engineering is it&#39;s API-driven. Everything was highly automated. We could start using automation to do some of the disaster recovery and do it a little bit more proactively. What I think we&#39;re heading towards is continuous resilience. I had this question - nobody seems to want chaos in production - do you want chaos in production? If I call it continuous resilience, will you let me do it in production? It&#39;s really just changing the name. We&#39;re doing exactly the same thing. When we&#39;re doing it in a production, we&#39;ll be doing continuous resilience because I think that&#39;s easier to sell to management. Nobody doesn&#39;t want continuous resilience, but they&#39;re not sure about anything with the word chaos in it. It was a bit of a branding thing here. What I&#39;m really talking about here is trying to productize and automate the things that the unicorn companies have done with chaos engineering, and take that to be a common pattern that&#39;s more productized that we&#39;ll be able to use across the industry as building blocks. We&#39;re in that transition right now.</p>
<h2 class="expanded">Availability, Safety and Security</h2>
<p>Here&#39;s the real problem: you can only be as strong as your weakest link. Whether it&#39;s security or availability that one chink in your armor is the thing that will break. The only way you can find that chink is by having somebody go there and test every link to find where the weaknesses are. This really doesn&#39;t just apply to availability, safety, and security. They all have similar characteristics. If you think about it, they have hard to measure near misses because you don&#39;t see the near misses, they&#39;re invisible. They&#39;re the ones you need to watch out for. They have complex dependencies that are hard to model and then have catastrophic failure modes. When it goes wrong, it really goes wrong. We get stories almost every day, something went wrong, ran out of capacity at launch or there&#39;s failure, or security failure, or an outage, or something like that.</p>
<p>They also have similar mitigations, and the way you protect against all these things is to have some layered defense in depth. If one thing breaks, there&#39;s another thing behind it to save you, and behind that maybe there&#39;s another thing. Depending on how critical it is, there&#39;s layers of defense. That applies to security, availability, safety. They&#39;re all the same.</p>
<p>There&#39;s bulkhead. If something goes wrong, you want to contain it. You want to stop it from spreading beyond that. That&#39;s one of the reasons that microservices work. If a microservice crash, it doesn&#39;t crash everything. If a monolith crashes, the whole thing&#39;s gone. That bulkhead is part of it. Also, you really want to minimize dependencies and privilege. If you think about a monolith that touches credit cards, the whole monolith has to be PCI compliant and is subject to all these controls. If you have a set of microservices, the ones that touch credit cards, probably in a different account, manage very differently than the ones doing personalization or the ones choosing what color the screen should be or whatever, the user interface. You can iterate and go much faster and have much lighter weight controls on the parts of your system that have minimum privileges and dependencies and you can really concentrate on the pieces that need it. That&#39;s another reason for spreading apart things and not bundling everything into one big monolith.</p>
<p>They also break each other. How many times a security failure is taking down your system? In fact, if there&#39;s a breach going on, the first thing they do is shut everything down, so you&#39;re not available. That&#39;s one of the things. Then the other thing is, if a system that&#39;s managing safety crashes, you don&#39;t have the safety. If your self-driving car controller crashes, then you might drive you off a cliff or something. You don&#39;t want these things to happen. These things are all linked across. We can think about all the different linkages. What I&#39;m going to concentrate on today is the availability piece. We&#39;ll extend this and some of these techniques can be used for security and safety as well. I&#39;m particularly interested in safety right now in safety-critical industries.</p>
<h2 class="expanded">What Should Your System Do When Something Fails?</h2>
<p>What should your system do when something fails? There are a few choices. One of the things you can do is stop, if you&#39;re not sure what you should do, if it might not be safe to continue, so you should stop. The other thing is maybe try and carry on with reduced functionality. That sounds nice. What actually usually happens is, it collapses horribly, because the least well-tested code in your system is the error handling code, and the least well-tested processes you have are the disaster recovery processes. It&#39;s just the fact of life. These things are the ones that no one wants to touch because if anything goes wrong, then it makes it even worse. We&#39;ve got lots of examples of a small failure triggering a bigger failure and blowing up to take out an entire system, an entire company. We&#39;ve seen airports and airlines go down because of a single route or a single power supply failure, or a small error in one place. Nuclear power station failures, those kinds of things. Think about this.</p>
<p>Think about a permissions look up. It&#39;s a fairly minor thing. You&#39;re running through your code and you say, &quot;Should this customer be allowed to do this thing?&quot; and you get an exception back so you can&#39;t tell because the subscriber service is down. Should you stop or continue? It depends. What are you about to do? Are you about to move a billion dollars from one bank account to another or you&#39;re about to show somebody a movie? If you&#39;re gonna show someone a movie, just keep going. We eventually trained most of the Netflix engineers to do permissive failure because the cost of showing them that movie if you weren&#39;t sure if they were a subscriber in good standing was negligible. It&#39;s probably not even a cent. If you&#39;re in banking, and you&#39;re not quite sure what the state the world is in, you stop. That&#39;s the way they operate.</p>
<p>There&#39;s a really good paper on this. You saw Pat [Helland] yesterday morning. He&#39;s got lots of great papers. &quot;Memories, Guesses, and Apologies&quot; is a high-level view of what databases are. Databases are things that try to remember what you told them. Then they try to guess what you told them before. If they can&#39;t figure out what to do next, they try to apologize for it. The apologies part is actually the bit that&#39;s interesting here. It talks about, sometimes rather than writing a whole lot of code to try and capture every condition, you say &quot;Call this number.&quot; Punting to customer service, punting to a human, it&#39;s actually one of the best strategies for handling failure modes. You can overdo it, but it&#39;s actually better than running through a whole lot of code which is very poorly tested and probably going to make it worse.</p>
<p>Quite often the thing to do is handle the easy stuff, when it gets complicated punt to a human. Humans have a very adaptable way of being able to manage failure modes. There&#39;s a whole section on this. There&#39;s a book called &quot;The Accidental Anarchist&quot; by Sidney Dekker, which is really worth reading. If you look at the way processes are written and the way the processes are actually operated in safety-critical industries, they&#39;re different. If you automate the way it was written down, you&#39;re not actually doing the right automation. The human element is there to make it safe.</p>
<p>Who here has a backup datacenter? A few of you. How often do your failover apps do it? If you work in banking, the answer should be at least once a year because the auditors come by and make you do it. If you don&#39;t work in banking, the answers are usually never or very occasionally. I have one person who gave the best answer I&#39;ve ever had, which is every weekend. &quot;If the week number is even, we&#39;re on this datacenter. If the week number is odd, we&#39;re running on this datacenter. We flip it every weekend. If anything goes wrong on a Wednesday, we push the button, we flip, and everyone carries on and it&#39;s perfect for us.&quot; That is the best operational discipline I&#39;ve ever heard of, and it happened once.</p>
<p>At Netflix, the last time I heard was every couple of weeks, they do a region evacuation test. They&#39;re testing stuff reasonably often. One of the big banks we were working with did a one-time region evacuation test. There was a lot of work and it went well but they don&#39;t do it every few weeks. It&#39;s something that people do to test. That&#39;s apps, and then that whole datacenter at once. Does that ever happen? Typically, the order doesn&#39;t make you test your critical apps. They don&#39;t pull the plug on the datacenter. This is very rare. I call this &quot;Availability Theater.&quot; You&#39;ve spent a lot of money building a big datacenter, and if you ever had to use it, you know that you&#39;d be in terrible shape and it&#39;s horrible and all kinds of things would go wrong. The only time you actually test datacenter failure is when it actually fails. That&#39;s when you discover how little of it actually works.</p>
<h2 class="expanded">A Fairy Tale</h2>
<p>We had this nice fairy tale we&#39;re operating on, once upon a time, in theory. If everything works perfectly, we got a plan to survive some of the disasters we thought of in advance. It&#39;s not a great story. It&#39;s not a good bedtime story to keep you up at night. Here&#39;s a few things. Think about your company&#39;s domain name. Think about what happened if that domain name no longer existed or was taken down because somebody forgot to renew it. How much of your systems would work?</p>
<p>It turns out there was a SaaS provider who had this happen a couple of years ago. All of their internal email was down, so that means all of the accounts they had to do anything were on that email address. All of their product was on the same domain. What was left? One Twitter handle. CEO apologizing on Twitter for about two days, solid. I&#39;ve already got this, &quot;I&#39;m sorry, we&#39;re trying to fix it.&quot; That was the only communication channel they had left that was branded to that vendor. Don&#39;t be that company. Everyone hopefully is being frightened now. DNS is the fastest way of taking down just about any service. We&#39;ll look a bit at that later.</p>
<p>This has happened to everybody in the room pretty much that&#39;s been running anything. I&#39;m just guaranteeing that you have had a certificate site. It took down Netflix a bunch of times. It&#39;s happened to AWS. We have so much process around trying to find these stupid certificates and make sure they don&#39;t all expire at the same time, and have tracking systems for them. The operational effort going into just this one thing is huge. We&#39;re pretty good at it. Every now and again, some new service comes up that makes it sneak another one in, but the mature ones are pretty good at this. That happens all the time.</p>
<p>It turns out computers don&#39;t work underwater. It&#39;s very sad, really. Then when the water goes away, they&#39;re full of bits of sand, and seaweed, and dead fish, and things. They still don&#39;t work. A friend of mine had a lovely time rebuilding a datacenter that was no longer in a basement in Jersey City. Another trick: don&#39;t put the generators in the basement. That&#39;s also not a good plan. Not putting the fuel supply for the generators in the basement is also not a good plan. Try to keep stuff above flood level, if possible. Also, hopefully, it&#39;s not due tomorrow. Watch out for that.</p>
<p>There&#39;s a good book here, &quot;Drift into Failure.&quot; The point of this book is that at every stage you&#39;re making the right decision, you&#39;re doing the right thing. It actually isn&#39;t good to test your backup systems because you know they&#39;re not gonna work and it&#39;s gonna be expensive and it might not go wrong. The locally right thing to do is to just concentrate on making your product better and hope that you don&#39;t get the failure. That is actually the rational thing to do. These rational decisions get put into a chain until every possible mitigation has gone away and then you do get this massive failure. I tell people don&#39;t read the book on the plane because it&#39;s mostly full of plane crashes and people dying in hospital, and stuff like that. While I was flying to Chicago for GOTO Chicago a year or so ago, I wrote an entire section of this book about a plane crash while I was on a plane and made myself do it, and then gave this talk about plane crashes. I had to learn about that failure.</p>
<p>What you need to do is learn to capture and learn from near misses. The airline industry is very good at this. Every time you get stuck at the gate because one tiny thing was wrong with your plane, that tiny thing is being logged centrally shared with every airline, shared with every manufacturer. They all share the engines. They share the airframes. All of that information goes into a central database that is managed for the safety of the airline industry. That is one reason that putting ourselves in metal tubes and shooting ourselves across continents, which is one of the scariest things we could probably ever think about doing is actually very safe. It&#39;s because it&#39;s so dangerous that they got very good at making it safe. That&#39;s why, when things aren&#39;t that dangerous, you don&#39;t spend so much effort on it.</p>
<h2 class="expanded">Chaos Architecture</h2>
<p>This is the architecture l think about. You&#39;ve got infrastructure at the bottom. You&#39;re going to have redundancy there. There&#39;s some switching between things. If you&#39;ve got two ways to do something, you have to have a way to decide one of them isn&#39;t working and switch to the other. That switch has to be really robust and reliable. That&#39;s the problem with disaster recovery. Your switching processes, and code, and practices are not well-tested. You&#39;ve got an unreliable switch between your primary and backup datacenter. You might as well just not have the backup datacenter.</p>
<p>Above that, you got the application. Make sure your code doesn&#39;t fall over when it gets annoying error messages and things. Then at the top, you&#39;ve got people because given a perfectly good system, people will mess it up. If you&#39;re not trained, you don&#39;t know what it&#39;s doing. You&#39;ll reboot the wrong machine, you&#39;re not practiced. You need game days and you need to practice the exercises that will let you know what to do when there&#39;s a failure.</p>
<p>This is defense in depth for availability. You need experienced staff that have been on calls. They know how to handle the main failure modes. You&#39;ve got robust applications that you&#39;ve done a bit of fuzz testing, and you&#39;ve tried to break a few times. You have a dependable switching fabric, and there&#39;s a redundant service foundation.</p>
<p>This part of the talk, I&#39;ve given a few times. It&#39;s an intro to why this is interesting. This is the first time I&#39;ve presented it. I&#39;m just bridging into it. One of the things about failures is that, as you get good at handling them, they start to become more and more strange. The easy ones, you&#39;ve sorted. Then, every error you get is something you&#39;ve never seen before in some weird combination of things. The best defense is to get fast at figuring it out.</p>
<p>I did a talk about five years ago at Monitorama, where I was complaining about the fact that a lot of monitoring tools gather data every minute. Then they actually sit on the data for a few minutes and about six, seven minutes after, something actually happened in the real world, you finally have enough points on a graph to say, &quot;That doesn&#39;t look good.&quot; Then you&#39;ve tried to fix it, and then you have to wait another six or seven minutes before the graph gets better again. Minutes is maybe fine, but at the rate we&#39;re running out with online web services, that could be too long. It&#39;s very hard. The feedback loop is too slow.</p>
<p>If you have one-second updates with some of your critical measures, then within 30 seconds, you can tell something&#39;s gone wrong. One of the effects here is you&#39;ve deployed some code, you go, &quot;I can&#39;t even tell whether this is good or not for five minutes, I&#39;m going to go get a cup of coffee.&quot; You&#39;re off by the coffee, and everyone starts running around because the site&#39;s down. &quot;What&#39;s going on?&quot; Eventually, you wander back and discover you broke the site and you were off chatting to people over a cup of coffee. If you know you&#39;re going to be able to see it in 30 seconds, you&#39;ll wait to make sure it&#39;s Ok before you go and get the cup of coffee. 30 seconds is the limit. 10 seconds is great for responsiveness and 30 seconds is the maximum, and after that, you&#39;re going to be off doing something else. A human attention span is an important characteristic of these safety systems.</p>
<h2 class="expanded">Observability</h2>
<p>What we want to do then is make it possible to see what&#39;s going on. Observability is a very important characteristic. The word observability was really defined in this paper in 1961, by Kalman, who did a lot of work in control theory. If you can tell how a system is going to behave by looking at its inputs and outputs, then it&#39;s observable. That&#39;s actually a strong property of a control system. What that means is that if it&#39;s doing something inside, you have to have a way to look inside it and do logging or expose some of the internal behaviors so that you can predict what its outputs going to be. That&#39;s really what we&#39;re doing. Adding logging to a system is just making it more observable. Sticking printfs in your code, that&#39;s observability.</p>
<p>All these things are models though. What you have outside the system is a model of how you think it&#39;s going to behave. All models are wrong, some models are useful. The key here is to have one that&#39;s simple enough that it&#39;s tractable to work with, but complicated enough that it captures the behavior of the system.</p>
<p>This book is voted the most interesting book right now. They run workshops. There&#39;s a workshop next March, I&#39;ve actually applied to go to it. I want to go hang out with Nancy Leverson and the team at MIT. I&#39;m really into Systems Thinking in general, I&#39;ve done talks on that. Using system thinking to look at safety, there&#39;s a lot of interesting work here. The systems that are described in this book, a couple of fairly insignificant systems, like the air traffic control system, the nuclear launch system, they worked. This is the team that worked on making those planes not crash into each other, and not have accidentally launch nuclear missiles all the time. The software in those systems needs to be done right. There&#39;s a lot of examples in there.</p>
<p>I&#39;m going to dig into this a little bit and try and explain how this works. This is a screenshot from the book. It&#39;s got three levels in it. If you look at this, the bottom&#39;s the control process. That&#39;s the thing that does the work. Above it, you&#39;ve got the Automated Controller. A lot of the code we write is actually controlling things. It&#39;s controlling the process. Think of auto-scalers or a lot of the security code is permission checking, &quot;should you do this thing?&quot; All of that is control process. Above that is a human, and the human is trying to understand, is the automatic control working right and is the thing itself working right?</p>
<p>I&#39;m going to change the labels on this to maybe make it make more sense. Here&#39;s a Financial Services application at the bottom, customers are making a requests to it. At some point, it says, &quot;Whatever you&#39;ve requested has some completed action.&quot; It&#39;s just some part of the flow of some financial services application. Above that, there&#39;s a control plane, which is maybe auto-scaling the number of machines that are running it or it&#39;s doing fraud analysis to decide whether the system should be allowed to do things or something like that. You&#39;ve got all these controls that you&#39;ve put around your code. Above it, there&#39;s a human watching this. If the human sees throughput goes to zero, they&#39;re supposed to pay attention and go, &quot;What happened?&quot; Maybe there&#39;s an alert get sent to them, then they say, &quot;Is the control plane telling me the wrong thing? Is the metric telling me the wrong thing or is it really going to zero?&quot; You don&#39;t know. Your model of automation is your model of how that control plane is working, and the model of control process is your understanding of how the actual thing works. If you imagine getting somebody that&#39;s never seen the system before and putting him in charge of it, and they get the alert, they don&#39;t have the model. You&#39;re saying, &quot;What should we do?&quot; &quot;I don&#39;t know. Call someone. You phone a friend.&quot; This written trained procedures are about trying to get people to understand the model of the behavior of the control system, and the model of the thing being controlled. There&#39;s two separate models, and a lot of the problems we have in the usability of that interface.</p>
<p>This is the way STPA works. Instead of looking at the boxes and saying, &quot;What could go wrong with this box?&quot; we look at the wires. We look at the connections between them. We look at the information flows. In this case, let&#39;s say the human controller sees throughput go to zero, while the throughput goes to zero and the human is supposed to generate a control action, but there are all these different things, human was not paying attention, so not provided. They didn&#39;t do the action so maybe the system runs out for a while, or they do an unsafe action rebooting everything in sight maybe.</p>
<p>The system was actually running fine, but they got freaked out by something and did something that wasn&#39;t necessary. Maybe the reason that the throughput went down was the Super Bowl just started on TV, which is something that causes about half of Netflix&#39;s traffic to go away. In fact, we had a big outage once in Brazil and Mexico at the same time because Brazil and Mexico were playing each other at soccer, and everyone stopped all the TVs with big DoS attack on all the people trying to watch Netflix on TV.</p>
<p>There are external reasons why throughput might drop, and you might get freaked out, and do something that isn&#39;t needed. That&#39;s an unsafe action or safe but too early. Then you might say, &quot;The system&#39;s down, you provide the right action, but too late.&quot; You could do things in the wrong order. Maybe you see throughput go up, and the system getting unhappy, and you want to increase the limit on auto-scaler but you don&#39;t increase it enough. That&#39;s a sort of a stop too soon. You didn&#39;t provide enough of a fix or you took too long. Then there&#39;s conflicts. Maybe there isn&#39;t one human controller, there&#39;s a room full of them and they can&#39;t agree what to do. It&#39;s that kind of conflict, coordination problems between people. Maybe two people independently fix the system without talking to each other and the combination of the two break things. Or, it degrades over time, meaning maybe the written train procedure about how to fix this particular problem was written years ago, and the system&#39;s changed enough that it no longer reflects that system.</p>
<p>This is a good checklist for hazards. If you&#39;re looking at how to control your environment, and you&#39;re looking at that human interface, these are the standard list of hazards. Go through them one by one. Think about, how would this apply to my system? You can go through and you can make a list, and think &quot;What would we do here? How are we going to make sure only one person at a time is making a change or get everybody on a Con call and have that coordination happen on the Con call?&quot; You can&#39;t make a change unless you&#39;re dialed into the call. Those kinds of rules are there to give that coordination. We&#39;ve got to follow them.</p>
<p>Here&#39;s another area. What if the system, just for some reason stops reporting the right numbers? It&#39;s working fine, it&#39;s completing its actions, but it stopped reporting throughput. What happens? Maybe it gets frozen at the last number, so you just stop getting updates. Maybe you start getting zeros. Maybe the number overflows, and you&#39;re getting effectively random numbers or negative numbers for throughput or it&#39;s corrupted. There&#39;s some bit error somewhere along the way and some random number comes through, that might trigger an alarm. Particularly, if you&#39;ve got a control algorithm, that it&#39;s automatically applying things, if you feed a random number into it, you&#39;re going to get weird stuff. You got to be careful what those control algorithms are doing.</p>
<p>Out of order - sometimes things get delivered in the wrong order, that something gets delayed, something else gets past it, you can get issues with that. Maybe the system just starts sending things too quickly and your sensor, your reporting system, your monitoring system collapses under the load. That can happen a lot in a fast-growing business. Or it just gets delayed. Like I was pointing out before, if it takes eight minutes to notice something has gone wrong, that may be too long. Again, coordination problems may get bad over time. Say you&#39;ve got a memory leak and whatever is recording your sensors or garbage collection, and after a while the GCs poses start getting too slow and the system starts getting unreliable or it&#39;s fine when you first deploy it, it gets bad over time. Run through this list, try to figure it out. Go read the book, there&#39;s a lot more detail on how to do this.</p>
<p>I&#39;m just trying to give you a flavor for this mechanism because the thing about this is, I&#39;m not saying, &quot;Did the app fail?&quot; I&#39;m saying, &quot;Am I providing the right information to the other parts of the system? What could go wrong with that information?&quot; It&#39;s looking at the wires rather than the boxes in the diagram.</p>
<p>Another problem: model problems. My model of automation, I wasn&#39;t trained in how the system worked, so I have the wrong model. I think the system behaves in a certain way, and it actually behaves in a different way. Or, I don&#39;t have the right inputs, like the system isn&#39;t actually out of CPU, it&#39;s out of memory, but I have no data on the memory usage. I&#39;m not getting updates of similar issues. You can have a problem with the model in the human or the model in the control plane as well. Think about what could go wrong there. Here&#39;s a particular example of that: the Boeing 737 Max 8, well-known problem right now. The model of automation that was in the pilot&#39;s head had not been updated from the previous 737. This is one part of what went wrong. When the plane started nose-diving, they didn&#39;t know why. They couldn&#39;t understand that the anti-stall system, which was a new system, had a model of a control process and it was automating something. It was trying to manage the system, but their model of the automation had not been updated. There&#39;s obviously other reasons, but that was one of the big contributing reasons that they wanted to make the plane as similar as possible to the last one, so they didn&#39;t want to retrain everybody to make it cheaper to introduce, but that was a critical piece of training here for a system that turned out to crash the systems.</p>
<p>That&#39;s STPA. I think it&#39;s a really interesting way of looking at what goes wrong. It&#39;s very good that the operator interaction and thinking about the overall control stability, so it&#39;s a top-down method. You&#39;re looking at the system from above, you&#39;re working down. I&#39;m going to be developing more models on that in future versions of this talk. We&#39;re now going to do more of a bottom-up version looking at risk.</p>
<h2 class="expanded">Risk</h2>
<p>If we look at risk, typically financial risk, it&#39;s usually severity and probability. What&#39;s the probability of something going wrong and how bad would it be? Multiply those two together, you&#39;ve got some idea of risk. These are economic-financial risks that are usually thought about in this way. However, if you add engineering risk detectability matters. s is because if a plane crashes, you know the plane crashed but that&#39;s a big thing. If you lose money, you can count the money. If there&#39;s some weird thing happening deep in something and you can&#39;t see it, you&#39;re building up failures and you&#39;re building up risk that you can&#39;t see. One way to mitigate risk is to add observability to expose silent failures. Actually, it reduces the risk because you can tell they aren&#39;t happening and you can tell they&#39;re starting to happen and you can get in there and mitigate something before it actually takes you out. This is a component level and there&#39;s a bunch of prioritizations you can do here to basically figure out what&#39;s the highest risk you should work on.</p>
<p>FMEA is a technique invented in the 1960s. Also, it&#39;s an old engineering technique. Anyone who did mechanical engineering or electrical engineering, they taught it to you in college. They don&#39;t seem to teach it to software developers, but I think it&#39;s a useful technique because it gives you a way to prioritize, and list what you&#39;re doing, and discuss things. I&#39;m going to go through some FMEAs now.</p>
<p>I think the way I&#39;m layering this is four different FMEAs. There are four different groups of people that should get together to think about what could go wrong. The people that write the unique code for whatever business you&#39;re in, the unique business logic. Product managers need to be in the conversation because a lot of the what goes wrong touches customers and the behavior of the system. And then developers that are writing that code. That unique business logic is a layer that you want to think about just in isolation.</p>
<p>Underneath that, what I&#39;m calling the software platform team is all the code you got from somewhere else that you didn&#39;t write, what could go wrong with that? Some of that code is libraries, some of it is running systems, commercial off-the-shelf, software that you&#39;ve installed. Its cloud web services. All of that stuff that you depend on, what could go wrong with that. That includes things like cloud control planes and things like that, those sorts of services.</p>
<p>Underneath that infrastructure is hardware, like networks, buildings, machines, those kinds of failures, power, and all those sorts of stuff. Finally, Resilience Engineering. Things can go wrong with the teams that are trying to manage the processes of failure not knowing which phone number you&#39;re supposed to call into when there&#39;s an incident or not knowing how to log into the dashboards or having a monitoring system that fails along with the system. There&#39;s failures of observability which can actually leave you blind and that&#39;s a whole different set of failure modes, and whoever&#39;s managing that should go and figure out what could go wrong.</p>
<p>I&#39;ve got sample versions of these spreadsheets. I put stuff on my GitHub account. I put slides on GitHub, I put PowerPoint in GitHub. In this faith, thought that one day Microsoft might let me do a pull request against an individual slide in a PowerPoint since they own this thing, but no. Anyway, it&#39;s a theory. I keep mentioning that and maybe one day, they&#39;ll integrate that. This deck isn&#39;t up next, I&#39;m still working on it.</p>
<p>If we look at FMEA, there are severity levels and we do a ranking of 1 to 10. I modified this a bit for infrastructure. The effects on the left are the standard names. Hazardous without warning. What does that mean? It means an earthquake or a meteorite destroyed your whole building - no warning, people injured. No warning. What could take you out with no warnings? For earthquakes and meteorites, you don&#39;t get much warning.</p>
<p>The next level down, you get a couple of days warning because they knew that Hurricane Sandy was coming for several days. You have time to do something to the system, but it still could destroy everything. The next one up is floods. The building is sort of ok, but the stuff in is been busted. Then if you have a fire, you get partial failures. The fire suppression system will take it out. You probably still have some of your data, but you lost some.</p>
<p>The ones I put in red are where you&#39;ve got some permanent loss of information. You&#39;ve lost some capacity or you lost some data, it&#39;s not coming back. Those machines are dead, those disks are dead. The yellow ones are temporary. It&#39;s down, but if you put power or cooling back, it&#39;s going to come back. There are various levels of failure mode there. There&#39;s a bit on the bubble here, where quite often if you power cycle a machine, they don&#39;t all come back. Not every disc starts back up, not every machine starts back up. Sometimes that disruption of power cycling can trigger latent faults in the machine. There&#39;s a bit on the bubble there. The way of thinking about it, people say, &quot;Datacenter went down.&quot; Do you mean, is it ever going to come back or is it different? If it&#39;s a smoking hole in the ground, that&#39;s a different outage than it just lost power for a few hours.</p>
<p>Then we have some standard probability things, this is just a standard set, an exponential idea that how likely is this thing to happen? You guess something in this level. The idea here is to be relative. You&#39;re not trying to be very accurate. You&#39;re just trying to get a relative probability. Then likelihood, this is how well you can detect it. If you have no way of measuring this thing, you have to give it a high ranking for unlikely to detect. If you can probably see it, you&#39;re in the middle, and if you&#39;ve got really good monitoring observability alerts set up and people pay attention to the alerts, you&#39;ll be down at the bottom there. Don&#39;t get too hung up on picking the number. It&#39;s just trying to get a number.</p>
<h2 class="expanded">FMEA Example</h2>
<p>Let&#39;s look at an example. What could go wrong with this? You could say, Lookup, and then no response. Ok, DNS is down. That&#39;s one thing that could go wrong with the system. I got my mobile app and whatever. This is that &quot;I forgot to renew my domain&quot; or something.</p>
<p>Let&#39;s look at another one. Let&#39;s assume we&#39;ve managed to look it up. We&#39;re going to send in a request. There&#39;s no route to that, this is a fast fail. If there&#39;s no route to host, it comes back immediately. It says, &quot;No. You&#39;ve given me this IP address, there&#39;s no one at home.&quot; Maybe the service is down or whatever. There&#39;s something going wrong here, but there isn&#39;t a route, so the network routing is down. This is a fast fail. Fast fails are nice because you can track them quickly.</p>
<p>Connect to host - hang around, wait a bit, it takes a little while. Undeliverable, but it timed out getting there. You have to wait tens of seconds or something before you get a request back. This is another failure mode. For every microservice that calls every other microservices, you can get these kinds of failures in them. This is looking at that low-level thing. Looking at the protocol as a series of failure modes.</p>
<p>Here&#39;s another one: connect. Let&#39;s try again. It worked in the end, but you are kind of logging something about some time out, maybe. If you can measure the timeout, maybe log that. This is what that looks like in an FMEA. What I&#39;ve got here is I&#39;m trying to call an API endpoint, and these are those four situations I just ran through but set out in a spreadsheet, where it says service unknown, service unreachable, request undeliverable, delivered but no response. You&#39;ve got all these different things here. I&#39;ve made up some numbers for what I think the various severity is the first column, probability is the next one, and detectability is the third one. You multiply them together, so 5 times 1 times 10 is 50 because I have no way to know. If DNS is down, nothing can get to your site. In fact, your site will just be fine.</p>
<p>The dataset is down for some proportion of your customers, it&#39;s an invisible failure. The only way you find out is when customers start tweeting or phoning your support line. I&#39;ve seen that failure a few times. It&#39;s very hard to detect. How do you work around it? Maybe you use an endpoint monitor, like those DNS endpoint monitors. There&#39;s a number of ways you could try and catch that. I think people, in general, underestimate the number of ways DNS can take you out and it&#39;s an area people should put more effort into. Quite a high score 50 of that. The other ones are pretty unlikely and not going to be that much of a problem, but you can retry things.</p>
<p>The previous one was TCP level, &quot;Can I get the packet that...?&quot; Let&#39;s pop up a level and look at some authentication, &quot;I&#39;m this user.&quot; &quot;No, I don&#39;t care. I&#39;m not going to talk to you.&quot; We&#39;re going to log that, it took certain amount of time to say this user failed to authenticate for some reason. Here&#39;s how you do that as an FMEA. A couple of either it fails completely, or it&#39;s slow and unreliable. It&#39;s two different cases. they&#39;re reasonably high, you should probably make sure you log an alert and have some way of tracking these things. You can work through up one level. This is now authenticating, once I get through this, I now have a connection that&#39;s authenticated, I can actually send a request on.</p>
<p>Finally, I can say, &quot;Get my homepage.&quot; All kinds of things could come back here. Think about the things that could come back. Think about, you should log them. You should log how long they take, what you were doing, what went wrong. Quite often the logging doesn&#39;t have all the information you&#39;d want.</p>
<p>Here are a few examples of things that might go wrong: time bombs, things that die over time, like internal wrap-around memory leaks. Everyone here should be subscribed to the End of Unix Time. If you&#39;re on Facebook, there&#39;s an event you can subscribe to. It&#39;s 2038. It&#39;s when the number of seconds since 1970 goes negative as a 32 bit number, and that&#39;s Unix time. You can just subscribe to it and then we&#39;ll have a little party or something. Another 19 years. QCon 2038, we&#39;ll get together and get it on our zoomer frames and whatever. Date bombs, leap years, leap seconds, epochs wrapping around, Y2K - are you testing for those? How did those break your system? This is the stuff that could cause you to get a different response back.</p>
<p>Content bombs, incoming data that crashes the app. I have a few examples of that. I&#39;m just going to skip that. You want to fuzz the input. You want to generate random things to see if you can crash things. Config errors, versioning errors, retry storms, I have a whole talk on retry storm I used to give. This is where chaos testing at the application level works, where you get your copy of Gremlin and you try to break one microservice at a time or you get in there and just mess with it.</p>
<p>The nice thing about microservice is, it does one thing, and hopefully, you did it this way. It has one verb or one noun it implements. You ask it to do the thing and it either does it or doesn&#39;t do it, and you should be able to see how fast it does it and poke at it. That isolation of functionality makes microservices much more tractable for knowing how they behave under failure. When you have a monolith that does 100 different things, there&#39;s so much internal state and complexity. It&#39;s very hard to understand and to get good test coverage on it.</p>
<p>What I&#39;m focusing on here is the interactions between the microservices, and what can go wrong there, and how those would propagate. Popping down a level, if we look at, for example, the cloud service control plane. What could go wrong here? You&#39;re making a call to EC2 set, &quot;Give me an instance.&quot; It says, &quot;No.&quot; That&#39;s bad. &quot;Why? &quot;We don&#39;t have any more of those left.&quot; That&#39;s one answer. We try not to do that too often. Or maybe the control planes down or something&#39;s wrong. Something&#39;s unhappy with the system today.</p>
<p>You try calling to say, &quot;Can I have an increased limit?&quot;, try different instance type, or switch to a different zone, or a different region. That&#39;s the provisioning part of EC2. There&#39;s a bunch of other things that could happen. You could get one, but it doesn&#39;t stop or it could just take a long time to start. You can think that what would you do in your situation? Similarly on the networking. One of the availability patterns that works well is to pre-allocate. If you&#39;re doing multi regions, pre-allocate all the networking so that if in the middle of some outage handling, your networking is already there. The cost of the networking structures is very low. You should make sure that they are always set up everywhere you want to be, even if you&#39;re not currently using them. It&#39;s a good practice. That&#39;s that pre-allocate network structures in all regions.</p>
<p>There&#39;s something you can&#39;t increase the table size. There&#39;s all these kinds of things. You want to basically try to pre-allocate state so that your control plane isn&#39;t needed as much. If your systems are there and you can just send traffic to a backup system if you&#39;re doing a failover, that&#39;s nice. As long as you don&#39;t have to provision new stuff, that will be easier. There&#39;s a number of failures here where you can mitigate them by pre-allocating. That&#39;s the point really.</p>
<p>If we look at the infrastructural level, I talked about this a minute ago, the availability zone durability, permanent destruction of a zone, fire or flood. You really want to make sure the system can run on two out of three zones in a region. The zones AWS has are at least 10 kilometers apart. They&#39;re not in the same floodplain. They&#39;re not in the same power supply. They&#39;re not on the same network interfaces. They have as little as possible in common. Then our regions are typically states apart cross country or in different countries, so we get a lot more separation.</p>
<p>Then, if you&#39;re trying to connect across regions then there&#39;s a bunch of different things. It&#39;s the same setup here in terms of the requests types. I&#39;m running through the protocol, &quot;Can I get to it? Can I look it up? Can I get to it?&quot; It&#39;s the same set of things, but now we&#39;re thinking about it in terms of region connectivity, instead of thinking about it at application level. What can you do if you can&#39;t deliver packets to a region? After a while you go, &quot;I&#39;m just going to decide that this region is not behaving and I&#39;m the networking trap. I cannot get to it. The network to a region is down or the network to a zone is down. I can&#39;t find ways of routing around.&quot; You got to be able to block the traffic and show that your system knows how to reroute traffic to another road zone or another region. That&#39;s what we&#39;re talking about here.</p>
<p>These are just examples. If you can look at the spreadsheet and fill them out, put your own things in them, extend them. At operation&#39;s level, it&#39;s better, I think, to use STPA to look at the monitoring operations failure, most of those hazards. Look at the hazards between the user, and what they see, and what the monitoring system shows you, what your dashboards show you, what the system&#39;s got. There are some low-level things, like just authentication. Maybe your monitoring agent cert timed out, so it can&#39;t authenticate, or the user can&#39;t log in, or whatever. There&#39;s a bunch of failure modes here just to do with authentication.</p>
<p>I tend to look at authentication connectivity as like the TCP level connectivity, authentication, and then what application-level at this low-level the traffic between systems. That&#39;s a pattern to follow, to think, &quot;Have I covered everything that might go wrong?&quot;</p>
<p>STPA has this top-down view of control hazards and FMEA has a bottom-up, lets you prioritize failure modes. They get better coverage with STPA, in general, for the things that go wrong, particularly for human stuff. You really want to use both in conjunction. There&#39;s good places to use both. Some people get religious about one or the other. I think there&#39;s lots of tools in the toolbox. These are interesting ones that can be useful.</p>
<h2 class="expanded">Good Cloud Resilience Practices</h2>
<p>One of the reasons chaos engineering happened when it did is that we had cloud and we were able to automate a lot of stuff. I&#39;m going to go through some resilience practices just to wrap up here. One of them is this rule of 3. If there&#39;s three ways to succeed, then you&#39;re in good shape. if you want to replicate data, three zones in a region, that&#39;s what the AWS services do. If you want to failover from a primary region to a secondary region, it&#39;s actually good to have two different secondary regions because if one failure failover fails, you want to have another option to fail over to if you&#39;re really paranoid. Then workloads across three regions active, active, active, that&#39;s the Netflix pattern that we set up, back when I was there a long time ago.</p>
<p>Another thing, if you&#39;re failing over, it&#39;s good to fail up. If you&#39;re worried about having a large workload and you&#39;re doing a multi-region primary, secondary failover, it&#39;s good to failover from a small region to a big one. The regions aren&#39;t all the same size. There are some workloads which run fine in one region, in one of the brand new regions that just came out in some little country around the world, if you failover to that, you may find we don&#39;t actually have enough capacity there to run. We&#39;d like to try and make them all look infinite, but they aren&#39;t all infinite, and you want to do some capacity planning and estimating.</p>
<p>The other thing is, during a failover there&#39;s lots of extra work to do. If you suddenly discover your application is latency-sensitive and stopped working when it&#39;s further away, that&#39;s a really bad time to discover that. The best practice would be say, if you&#39;re a Wall Street bank is to run everything in Oregon, and then failover to Virginia if everything goes wrong. You&#39;re more likely to succeed that way. If the problem is trying to survive a disaster, that&#39;s a useful attribute. Then you got a bit more latency, maybe for your daily work.</p>
<p>Another thing is to build your resilience environment first. The first app that we got running really at Netflix was the Chaos Monkey, and everyone else had to put up with the Chaos Monkey because it got there first. We were enforcing autoscaling. We wanted to be able to scale down. If you think of an autoscaler scaling up and scaling down, to scale down, it has to be able to kill instances. The Chaos Monkey was there to enforce the ability to scale down horizontally scaled workloads. That was actually what it was for.</p>
<p>It was to make sure you didn&#39;t put stateful machine, stateful workloads in autoscalers. Then you can have this badge of honor gamified a bit. &quot;My app survived all of this chaos testing, and it&#39;s running in this super high availability environment, and your app didn&#39;t. Do you mean your app&#39;s not important?&quot; You can gamify it a bit.</p>
<p>If you think about continuous delivery, which hopefully, many of you are doing now, you need test-driven development, you need canaries. If you put a check-in and it flows all the way through to production, you&#39;ve got that automation. Think about continuous resilience as an automated testing environment which will stress test that something really does have all the disaster recovery failover mechanisms in there. You&#39;re not just testing the functionality, you&#39;re testing its ability to withstand all of these failover operations. If you get that in there and do it right, you&#39;ll have really high confidence that whatever you&#39;ve got in production is going to be really resilient and makes that failure mitigations a very well-tested code path. Like I say, &quot;I don&#39;t care if you call it resilience engineering or chaos engineering.&quot;</p>
<p>As these datacenters migrate to the cloud, these fragile manual disaster recovery processes, I think they can be standardized, automated. We&#39;re starting to move to continuous resilience and build something that is more of a productizable repeatable thing.</p>
<p>There&#39;s a paper which has a lot more information about how to set up networks and things written by Pawan Agnihotri, who&#39;s one of the essay managers in the Wall Street region. I contributed an early version of this material to that paper. There&#39;s a section in there on FMEA. A more expanded version, I posted yesterday on medium, so you should be able to find this long written out version, and there&#39;s links to the long versions the FMEAs.</p>
<p>&nbsp;</p>
<script>
$(function() {
jQuery(function() {
jQuery.scrollDepth();
});
});
</script>
<p><big><strong>See more <a href="https://www.infoq.com/transcripts/presentations/">presentations with transcripts</a></strong></big></p>
<p>&nbsp;</p>
</div>
</div>
</div>
<div class="aside">
<div class="recorded">
<p class="meta recorded__meta">Recorded at:</p>
<a href="https://qconsf.com/">
<img loading="lazy" alt="Software is changing the world. QCon empowers software development by facilitating the spread of knowledge and innovation in the developer community. A practitioner-driven conference, QCon is designed for technical team leads, architects, engineering directors, and project managers who influence innovation in their teams." src="https://res.infoq.com/presentations/microservices-failure-modes/en/promoimage/Qconlogo1-1572000206837.jpg">
</a>
</div>
<p class="date">Jan 07, 2020</p>
<div class="article__authors authors">
<div>
<p class="meta">by</p>
<ul class="no-style authors">
<li data-id="author-Adrian-Cockcroft">
<p class="meta author__bio">
<a href="/profile/Adrian-Cockcroft/" class="avatar author__avatar" aria-label="Adrian Cockcroft"></a>
<span class="author__name">
<a href="/profile/Adrian-Cockcroft/" class="author__link">Adrian Cockcroft</a>
</span>
</p>
</li>
</ul>
</div>
</div>
<div class="ads__area">
<script type="text/javascript">
var uriMapping = "presentations";
var showVcr = "false";
var fillWithVcr = "true";
var sponsorshipsJson = "{&quot;links&quot;:[{&quot;styleName&quot;:&quot;pdf&quot;,&quot;style&quot;:&quot;PDF&quot;,&quot;text&quot;:&quot;The Practical Agentic Coding Playbook&quot;,&quot;id&quot;:&quot;18089bbe-8e7c-4b03-a6ef-ca191b8d102e&quot;,&quot;target&quot;:&quot;https://www.infoq.com/vendorcontent/show.action?vcr=430ffe46-9047-4739-a817-b7d64d070721&amp;utm_source=infoq&amp;utm_medium=VCR&amp;utm_campaign=vcr_fixed_link&quot;,&quot;active&quot;:true}]}";
var sponsoredLinks = $.parseJSON($("<div/>").html(sponsorshipsJson).text()).links;
var numberOfSponsoredVcrIds = sponsoredLinks != null ? sponsoredLinks.length : 0;
var maxItems = 5 - numberOfSponsoredVcrIds;
var intervalVcrSponsorship = setInterval(function() {
if (window.vcrsLoaded) {
clearInterval(intervalVcrSponsorship);
if(showVcr || fillWithVcr) {
if(fillWithVcr) {
for(var index in window.vcrList) {
if(VCR.isVcrSponsored(sponsoredLinks, window.vcrList[index])) {
VCR.addToExcludedList(window.vcrList[index]);
}
}
}
var vcrs = VCR.getByTopicsAndCommunities(window.vcrList, topicIds, communityIds, maxItems, false, null);
if (vcrs != null && vcrs.length > 0 || (sponsoredLinks != null && sponsoredLinks.length > 0)) {
VCR.addToExcludedList(vcrs);
var $widget = jQuery('.relatedVendor');
getCommonElementsForPresentationsWithTranscripts(vcrs, uriMapping, "RIGHT_BAR_RVC");
$widget.css("display", "block");
} else {
var $widget = jQuery('.relatedVendor');
$widget.removeClass("f_rvcbox");
}
}
window.contentVcrFinished = true;
// search for infoq.event.on("contentVcrFinished",... to see how/where it is used
infoq.event.trigger("contentVcrFinished");
}
}, 200);
</script>
<input type="hidden" name="" value="2498" id="cont_item_primary_topic"/>
<div class="widget relatedVendor"
style="display: block"
>
<h4 class="widget__heading">Related Sponsors</h4>
<ul data-tax="prsp" data-horizontal="true" data-size="small"
class="prsp__items no-style cards showImageFirst">
<div class="f_rvcList"></div>
<li class="book">
<div class="book__info">
<h4 class="book__title">
<a href="/url/f/18089bbe-8e7c-4b03-a6ef-ca191b8d102e/" rel="nofollow">
The Practical Agentic Coding Playbook
</a>
</h4>
</div>
</li>
</ul>
<ul class="related-prsp" jsh="{&quot;topic&quot;:&quot;QCon Software Development Conference&quot;,&quot;id&quot;:&quot;12605306-ee86-4a5c-bc22-ab8b965bfc4a&quot;,&quot;title&quot;:&quot;Moderne - Architect Influence Program 08/11/2026 - 11/11/2026&quot;}">
<h4 class="widget__heading">Related Sponsor</h4>
<a href="/url/f/66abda2e-0d1f-4030-92f5-2d460e845635/" target="_blank" rel="nofollow">
<img loading="lazy" src="https://imgopt.infoq.com//fit-in/250x380/filters:quality(100)/filters:no_upscale()/sponsorship/topic/12605306-ee86-4a5c-bc22-ab8b965bfc4a/ModerneLogoMicrosite-1786453474094.png"/>
</a>
<p style="clear: both; padding: 10px 0 0 0;"><b>Automated code transformation at scale. Fix vulnerabilities, migrate frameworks, and modernize legacy code across every repo. <a href="/url/f/b66b8015-fc94-4daf-88dc-691741225dd6/" target="_blank" rel="nofollow">Try Moderne Free</a>.</b></p>
</ul>
</div>
<script type="text/javascript">
window.finishedRightbarVcr = false;
var whitepaperVcrsJson = null;
var topicSponsorshipJson = "{&quot;iconLink&quot;:&quot;/url/f/66abda2e-0d1f-4030-92f5-2d460e845635/&quot;,&quot;iconHref&quot;:&quot;https://imgopt.infoq.com//fit-in/275x500/filters:quality(100)/filters:no_upscale()/sponsorship/topic/12605306-ee86-4a5c-bc22-ab8b965bfc4a/ModerneLogoMicrosite-1786453474094.png&quot;,&quot;id&quot;:&quot;44704352-e094-4805-9d0b-c9675f61311a&quot;}";
var vcrOptionalListJson = null;
/* do not delete these two, as they are used further in the code */
var contentDatetimeFormat='MMM dd, yyyy';
var contentUriMapping="presentations";
JSi18n.relatedRightbar_relatedContent='Related Content';
JSi18n.relatedRightbar_sponsoredContent='Related Sponsors';
JSi18n.relatedRightbar_sponsoredBy='Sponsored by';
var topicIds = "6446,3450,6218,3960,593,6526,876,131,2961";
var communityIds = "2498";
var company = "Moderne";
// this event is fired by frontend once all the necessary things have been done(mobile display, moving vcr boxes around when needed...)
var canStartTrackingCustomRightbar = false;
infoq.event.on('loaded', function(e) {
canStartTrackingCustomRightbar = true;
});
var intervalRightbar = setInterval(function() {
if (window.vcrsLoaded) {
clearInterval(intervalRightbar);
if(company != null && company != "") {
whitepaperVcrsJson = VCR.filterByCompany(company, window.vcrList);
}
vcrOptionalListJson = VCR.getByTopicsAndCommunities(window.vcrList, topicIds, communityIds, 20, true, null);
var adsArea = $('.ads__area');
// 1. display whitepaperVcrs
var topicSponsorship = $.parseJSON($("<div/>").html(topicSponsorshipJson).text());
var whitepaperVcrItems = $.parseJSON($("<div/>").html(JSON.stringify(whitepaperVcrsJson)).text());
var index = 0;
var hasWhitepaper = whitepaperVcrItems !== null && whitepaperVcrItems.length > 0;
if (hasWhitepaper) {
index++;
var whitepaperDiv = getRightbarCustomVcrWidget(whitepaperVcrItems, index, topicSponsorship, "transcriptsright", 1);
$(whitepaperDiv).addClass('prspContent_side');
$(whitepaperDiv).removeClass('box');
$(whitepaperDiv).find('a.book__image img').addClass("book__cover");
adsArea.append(whitepaperDiv);
}
// 2. repeat the "VCR OPTIONAL box as many times as possible, but not more than 5 times"
var items = $.parseJSON($("<div/>").html(JSON.stringify(vcrOptionalListJson)).text());
var minimumBoxHeight = 2000;
var availableSpace = $('.aside').height() - $('.relatedVendor').height();
var numberOfBoxes = Math.floor(availableSpace / minimumBoxHeight);
if (numberOfBoxes > 5) {
numberOfBoxes = 5;
}
for (var i = 0; i<numberOfBoxes; i++) {
index++;
var relatedVcrDiv = getRightbarCustomVcrWidget(items, index, null, "transcriptsright", 1);
$(relatedVcrDiv).addClass('prspContent_side');
$(relatedVcrDiv).removeClass('box');
$(relatedVcrDiv).find('a.book__image img').addClass("book__cover");
adsArea.append(relatedVcrDiv);
}
window.finishedRightbarVcr = true;
}
}, 200);
// these two events can happen one before another async(no precedence any can be first or second). Make sure tracking starts when both happened
var intervalTrackingRightbar = setInterval(function() {
if(canStartTrackingCustomRightbar && window.finishedRightbarVcr){
clearInterval(intervalTrackingRightbar);
VCR.doTrackingCustomRightbarForPresentations();
}
}, 200);
</script>
</div>
</div>
<div class="widget article__fromTopic topics">
<div class="widget__head related__for-topic" data-id="7451" data-trk-ref="content_primary_topic">
<h4 class="heading related__heading">
This content is in the <a href='/Cloud/'>Cloud</a> topic
</h4>
</div>
<h5 class="heading related__inline">Related Topics:</h5>
<ul class="no-style topics related__topics topics__small" data-trk-ref="content_related_topic">
<li data-id="6816">
<a href="/architecture-design/" class="button related__topic button__small button__black">Architecture &amp; Design</a>
</li>
<li data-id="17901">
<a href="/qcon-sanfrancisco-2019/" class="button related__topic button__small button__black">QCon San Francisco 2019</a>
</li>
<li data-id="14722">
<a href="/Resilience/" class="button related__topic button__small button__black">Resilience</a>
</li>
<li data-id="17662">
<a href="/transcripts/" class="button related__topic button__small button__black">Transcripts</a>
</li>
<li data-id="15274">
<a href="/microservices/" class="button related__topic button__small button__black">Microservices</a>
</li>
<li data-id="597">
<a href="/qcon/" class="button related__topic button__small button__black">QCon Software Development Conference</a>
</li>
<li data-id="17983">
<a href="/Cloud-Native-Architecture/" class="button related__topic button__small button__black">Cloud Native Architecture</a>
</li>
<li data-id="1389">
<a href="/architecture/" class="button related__topic button__small button__black">Architecture</a>
</li>
<li data-id="134">
<a href="/InfoQ/" class="button related__topic button__small button__black">InfoQ</a>
</li>
<li data-id="7451">
<a href="/Cloud/" class="button related__topic button__small button__black">Cloud</a>
</li>
</ul>
</div>
<input type="hidden" name="" value="2498" id="cont_item_primary_topic"/>
<script type="text/javascript">
$(document).ready(function() {
$.ajax({
url: "/api/recommendationlinks.action",
contentType: "application/x-www-form-urlencoded; charset=utf-8",
type: 'POST',
data: {
"primaryTopicAlias": "architecture-design",
"topicIds": "6218,3960,3450,6446,6526",
"title": "Managing Failure Modes in Microservice Architectures",
"contentPath": "/presentations/microservices-failure-modes",
"language": "en"
},
success: displayRelatedEditorial,
async: false
});
});
function displayRelatedEditorial(data) {
$('.related__editorial h4').text("Related Editorial");
if (data && data.length > 0) {
if(data[0].fromEs) {
//change title and tracking params
var box_title="Architecture &amp; Design";
//replace html entity since it conflicts with style
box_title=box_title.replace("&amp;","&");
$('.related__editorial h4').text("Popular in " + box_title);
}
for (var i = 0; i < data.length; i++) {
if (i === 5) {
break;
}
if (data[i].url.indexOf("/presentations/microservices-failure-modes") !== -1) {
console.log("Removing the current item from list...");
continue;
}
var theLinkURL = data[i].url;
if(!theLinkURL.endsWith("/")) {
theLinkURL = theLinkURL + "/";
}
var link = $('<li><h5 class="rvc__title"><a title="" href="' + theLinkURL + '/">' + data[i].title + '</a></h5></li>');
$('.related__editorial ul').append(link);
}
$('.related__editorial').show();
displayWidget = true;
}else{
$('.related__editorial').parent("li").remove();
}
if(displayWidget==true){
$('.related__group').attr("data-cols", $('.related__group').find(">li").length);
$('.related__group').css("display", "flex");
}
}
</script>
<ul class="no-style related__group nocontent cards">
<li>
<div class="related__editorial">
<h4 class="heading">Related Editorial</h4>
<ul></ul>
</div>
</li>
<li class="bg-gray">
<div class="related__prsp">
<h4 class="heading">Popular across InfoQ</h4>
<ul>
<li>
<h5 class="rvc__title">
<a href="/news/2026/09/netflix-flink-autoscaler/" class="rvc__link">
Netflix Moves toward Open Source Flink Autoscaler for 30,000+ Streaming Jobs
</a>
</h5>
</li>
<li>
<h5 class="rvc__title">
<a href="/news/2026/09/java-news-roundup-aug31-2026/" class="rvc__link">
Java News Roundup: TornadoVM 6, JReleaser, LangChain4j, Java Operator SDK, JHipster, Yupiik Fusion
</a>
</h5>
</li>
<li>
<h5 class="rvc__title">
<a href="/news/2026/09/google-mantis-vulnerability-scan/" class="rvc__link">
Google Mantis: an Agentic Vulnerability Scanning Harness for Reducing False Positives
</a>
</h5>
</li>
<li>
<h5 class="rvc__title">
<a href="/news/2026/09/blume-docs-ai/" class="rvc__link">
Blume: Zero-Config Docs Framework That Turns a Markdown Folder into an AI-Ready Website
</a>
</h5>
</li>
<li>
<h5 class="rvc__title">
<a href="/news/2026/09/cern-debian-infra/" class="rvc__link">
CERN Renounces RHEL in Favor of Debian for its Accelerator Controls Infrastructure
</a>
</h5>
</li>
<li>
<h5 class="rvc__title">
<a href="/podcasts/personality-over-skillset/" class="rvc__link">
Personality Over Skillset: How Adam Wachtel Builds Engineering Teams
</a>
</h5>
</li>
</ul>
</div>
</li>
</ul>
</article>
</div>
</section>
</div>
<div id="imgPreload"></div>
<script type="text/javascript">
// global vars that can be used for this page, use this section to add more.
var contentTitle = "Managing Failure Modes in Microservice Architectures",
contentPath = "/presentations/microservices-failure-modes",
contentUUID = "a6a3086a-1e58-4146-9927-965f8b211e12",
authorUserCSVIds = "101764598";
</script>
<script type="text/javascript">
// needed for starting download after IDP login, processing starts after frontend modules are loaded
infoq.event.on("loaded", function(){
if(loggedIn){
var href = window.location.href;
if(href.indexOf("#downloadPdf") != -1){
//window.location.href = href.substring(0, href.indexOf('#'));
$('#slides').click();
} else if(href.indexOf("#downloadPresentationMp3") != -1){
//window.location.href = href.substring(0, href.indexOf('#'));
$('#mp3').click();
}
}
});
P.c(false, 'p', isWideScreen, 'https://cdn.infoq.com/statics_s1_20260910111632/styles/static/images/logo/logo_scrubber_16_9.jpg', 'https://cdn.infoq.com/statics_s1_20260910111632/styles/static/images/logo/logo_scrubber_4_3.jpg');
//wait for the player to be ready
document.getElementById("video").addEventListener('loadeddata', function() {
//check if we are requested to jump to a specific time in the video
var t = CommonUtils.getQueryParameterByName("t");
if(t !== null && t.length <= 9) {
SlideSynchronizer.start();
P.seekPlayerTime(t, true);
}
}, false);
</script>
</main>
<footer class="footer ">
<section data-nosnippet class="section container">
<div class="container__inner">
<ul data-cols="5" class="no-style columns boxes topic__boxes">
<li class="development">
<div class="box__header">
<a class="t_all_footer_more-boxes-header" href="/development/">Development</a>
</div>
<div class="box__content">
<ul class="no-style box__list small">
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/tinygo-devkit/" title="Advancing Embedded Go: Recoverable Panics, UEFI, Radio and Hardware Dev Kit">Advancing Embedded Go: Recoverable Panics, UEFI, Radio and Hardware Dev Kit</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/riscv-cpython/" title="CPython Officially Adds RISC-V Support as a Tier 3 Platform">CPython Officially Adds RISC-V Support as a Tier 3 Platform</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/async-lua-neovim/" title="vim.async&#039;s Addition Modernizes Neovim&amp;rsquo;s Async Architecture for Better Stability">vim.async's Addition Modernizes Neovim&rsquo;s Async Architecture for Better Stability</a></h5>
</li>
</ul>
</div>
</li>
<li class="architecture-design">
<div class="box__header">
<a class="t_all_footer_more-boxes-header" href="/architecture-design/">Architecture &amp; Design</a>
</div>
<div class="box__content">
<ul class="no-style box__list small">
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/netflix-conductor-4-workflow/" title="Netflix Reworks Conductor for 420 Million Monthly Workflow Executions and 10X Larger Workflows">Netflix Reworks Conductor for 420 Million Monthly Workflow Executions and 10X Larger Workflows</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/meta-organizational-agents/" title="Meta&#039;s Recipe for Building Agents as &amp;quot;Organizational Second Brains&amp;quot;">Meta's Recipe for Building Agents as &quot;Organizational Second Brains&quot;</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/netflix-flink-autoscaler/" title="Netflix Moves toward Open Source Flink Autoscaler for 30,000+ Streaming Jobs">Netflix Moves toward Open Source Flink Autoscaler for 30,000+ Streaming Jobs</a></h5>
</li>
</ul>
</div>
</li>
<li class="culture-methods">
<div class="box__header">
<a class="t_all_footer_more-boxes-header" href="/culture-methods/">Culture &amp; Methods</a>
</div>
<div class="box__content">
<ul class="no-style box__list small">
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/autonomous-software-teams/" title="Beyond Autonomous Teams in Software Product Development">Beyond Autonomous Teams in Software Product Development</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/presentations/rust-refactoring/" title="Accelerating Performance by Incrementally Integrating Rust Into Existing Codebase">Accelerating Performance by Incrementally Integrating Rust Into Existing Codebase</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/podcasts/personality-over-skillset/" title="Personality Over Skillset: How Adam Wachtel Builds Engineering Teams">Personality Over Skillset: How Adam Wachtel Builds Engineering Teams</a></h5>
</li>
</ul>
</div>
</li>
<li class="ai-ml-data-eng">
<div class="box__header">
<a class="t_all_footer_more-boxes-header" href="/ai-ml-data-eng/">AI, ML &amp; Data Engineering</a>
</div>
<div class="box__content">
<ul class="no-style box__list small">
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/nvidia-pair-ai-task-router/" title="NVIDIA Personal AI Router Distributes AI Tasks across Local Compute">NVIDIA Personal AI Router Distributes AI Tasks across Local Compute</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/linkedin-ai-multi-teacher/" title="How LinkedIn Trains AI Job Search 8x Faster with Multi-Teacher Distillation">How LinkedIn Trains AI Job Search 8x Faster with Multi-Teacher Distillation</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/openai-gpt6-astra/" title="OpenAI Releases GPT-6 Astra for Coding and Computer Use">OpenAI Releases GPT-6 Astra for Coding and Computer Use</a></h5>
</li>
</ul>
</div>
</li>
<li class="devops">
<div class="box__header">
<a class="t_all_footer_more-boxes-header" href="/devops/">DevOps</a>
</div>
<div class="box__content">
<ul class="no-style box__list small">
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/terraform-aws-provider-6-62/" title="Terraform AWS Provider Continues Rapid Expansion as AWS Infrastructure Becomes More Complex">Terraform AWS Provider Continues Rapid Expansion as AWS Infrastructure Becomes More Complex</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/observability-ai-agents/" title="Session Traces and Cost Controls Help Diagnose AI Agent Failures">Session Traces and Cost Controls Help Diagnose AI Agent Failures</a></h5>
</li>
<li>
<h5><a class="t_all_footer_more-boxes-item" href="/news/2026/09/hashicorp-packer-verification/" title="HashiCorp Packer 1.16 Adds Native SLSA Provenance Generation and Verification for Machine Images">HashiCorp Packer 1.16 Adds Native SLSA Provenance Generation and Verification for Machine Images</a></h5>
</li>
</ul>
</div>
</li>
</ul>
</div>
</section>
<section data-nosnippet class="container section section__newsletter">
<div class="columns container__inner">
<div class="newsletter" data-col="1/2"
id="infoq-nl"
>
<h2 class="heading">
<strong>The InfoQ</strong> Newsletter
</h2>
<p class="intro">A round-up of last week&#x2019;s content on InfoQ sent out every Tuesday. Join a community of over 250,000 senior developers.
<a target="_blank" href="https://assets.infoq.com/newsletter/regular/en/newsletter_sample/newsletter_sample.html">View an example</a>
</p>
<ul class="newsletter__features">
<li>Get a quick overview of content published on a variety of innovator and early adopter technologies</li>
<li>Learn what you don&#x2019;t know that you don&#x2019;t know</li>
<li>Stay up to date with the latest information from the topics you are interested in</li>
</ul>
<div class="newsletter__subscribe">
<form class="form gdpr" name="footerNewsletterForm" id="footerNewsletterForm" action="#" onsubmit="footerNewsletter.saveSubscription(); return false;">
<div class="field newsletter__mail input__text input__no-label input__medium email">
<label for="email-newsletter-infoq" class="label field__label">Enter your e-mail address</label>
<input id="email-newsletter-infoq" name="footerNewsletterEmail" placeholder="Enter your e-mail address" class="input field__input" type="email"/>
<input type="text" name="emailH" id="input_email_h" aria-required="false" style="display:none !important" tabindex="-1" autocomplete="off"/>
<input type="hidden" id="fnt" name="fnt" value="3oCTVQJcuE725ucu"/>
<input type="hidden" id="footerNewsletterType" name="footerNewsletterType" value="regular"/>
<input type="hidden" id="cmpi" name="cmpi" value="1"/>
</div>
<div class="hidden">
<span class="input__select field country">
<label for="input-simple-newsletter-country" class="label field__label">Select your country</label>
<select id="input-simple-newsletter-country" class="select field__input">
<option value="" class="select__option">Select a country</option>
</select>
<p class="input__message field__desc"></p>
</span>
<span class="input__checkbox field hidden">
<input type="checkbox" id="gdpr-consent-simple-nl">
<label for="gdpr-consent-simple-nl" class="label"><span>I consent to InfoQ.com handling my data as explained in this <a href="https://www.infoq.com/privacy-notice">Privacy Notice</a>.</span></label>
</span>
</div>
<input type="submit" role="button" value="Subscribe" class="button button__medium button__red" onclick="return footerNewsletter.validateEmail('Invalid email address');"/>
</form>
<p class="meta">
<a href="/privacy-notice/" target="_blank">We protect your privacy.</a>
</p>
<span class="success" style="display:none;" id="footerNewsletterMessage"></span>
</div>
<script type="text/javascript">
var footerNewsletter = new Newsletter('Enter your e-mail address',
'email-newsletter-infoq', 'footerNewsletterType','footerNewsletterMessage', 'fnt', 'input_email_h', 'input-simple-newsletter-country', 'cmpi','footer_except_homepage');
</script>
</div>
<div data-col="1/2" data-bg="certification" class="event__container" style="padding-right: 20px;">
<a href="https://certification.qconferences.com/?utm_source=infoq&amp;utm_medium=referral&amp;utm_campaign=largefooterad_onlinecohorts26" target="_blank" class="qconplus__events-slider" style="">
<div>
<div class="logo-certification"></div>
</div>
<p class="intro" style="margin-bottom: 0;color: white !important;">
<b>InfoQ Online Certification Programs</b>
<br>For Senior Engineers, Architects, and Technical Leaders
</p>
<ul style="margin-bottom: 0;">
<li style="color: white !important;"><b>Architect</b> with Luca Mezzalira | September 14</li>
<li style="color: white !important;"><b>Engineering Leadership</b> with Michelle Brush | September 18</li>
<li style="color: white !important;"><b>AI-Assisted Engineering</b> with Zichuan Xiong & Premanand Chandrasekaran | September 18</li>
<li style="color: white !important;"><b>AI Security &amp; Privacy Engineering</b> with Katharine Jarmul | October 14</li>
</ul>
<br>
<p class="intro" style="margin-bottom: 0;color: white !important;">As your role becomes more senior, the work changes. You're no longer just implementing decisions; you're shaping the systems, trade-offs, and technical direction other teams depend on.<br>These 5-week online programs give you a structured way to work through real decisions from your current role with senior peers from other companies.</p>
<strong class="button button__green cta__button">RESERVE YOUR PLACE</strong>
<small style="color: white;"><br>Live online. 4 hours a week, for 5 weeks.</small>
</a>
</div>
</div>
</section>
<section data-nosnippet class="section container footer__subfooter align__left">
<div class="container__inner columns">
<div data-col="1/4" class="columns footer__links-wrap">
<div class="footer__links">
<a href="/" class="nuxt-link-exact-active active" title="Home">Home</a>
<a href="/reginit.action" class="emphasis" title="Create account">Create account</a>
<button aria-label="Log In" class="login" data-ref="mainLogin">Log In</button>
<a rel="noreferrer noopener" href="http://qconferences.com/" target="_blank" title="QCon Conferences">QCon Conferences</a>
<a rel="noreferrer noopener" href="https://events.infoq.com/" target="_blank">Events</a>
<a href="/write-for-infoq/" title="Write for InfoQ">Write for InfoQ</a>
<a href="/infoq-editors/" title="InfoQ Editors">InfoQ Editors</a>
<a href="/about-infoq/" title="About InfoQ">About InfoQ</a>
<a href="https://c4media.com/" target="_blank" title="About C4Media">About C4Media</a>
<a rel="noreferrer noopener" href="https://get.infoq.com/infoq-mediakit/" title="Media Kit" target="_blank">
Media Kit
</a>
<a href="https://devmarketing.c4media.com/?utm_source=infoq" title="InfoQ Developer Marketing Blog" target="_blank">InfoQ Developer Marketing Blog</a>
<a rel="noreferrer noopener" href="https://c4media.com/diversity" title="Diversity" target="_blank">Diversity</a>
</div>
</div>
<div data-col="1/4" class="events__list">
<h4 class="heading footer__heading">Events</h4>
<ul class="qcons__list no-style">
<li><span class="icon event__type conference"></span>
<div class="qcon__detail">
<h5 class="heading">
<a rel="noreferrer noopener" href="https://certification.qconferences.com/architecture?utm_source=infoq&utm_medium=referral&utm_campaign=footer_onlinecohortarchitecture26" target="_blank">Online InfoQ Architect Certification</a>
</h5>
<span class="meta date">September 14, 2026</span>
</div>
</li>
<li><span class="icon event__type conference"></span>
<div class="qcon__detail">
<h5 class="heading">
<a rel="noreferrer noopener" href="https://certification.qconferences.com/ai-assisted-engineering?utm_source=infoq&utm_medium=referral&utm_campaign=footer_onlinecohortaiassistedengineering26" target="_blank">Online InfoQ AI-Assisted Engineering Certification</a>
</h5>
<span class="meta date">September 18, 2026</span>
</div>
</li>
<li><span class="icon event__type conference"></span>
<div class="qcon__detail">
<h5 class="heading">
<a rel="noreferrer noopener" href="https://certification.qconferences.com/ai-security-privacy?utm_source=infoq&utm_medium=referral&utm_campaign=footer_onlinecohortaisecurity26" target="_blank">Online InfoQ AI Security & Privacy Engineering Program</a>
</h5>
<span class="meta date">October 14, 2026</span>
</div>
</li>
<li><span class="icon event__type conference"></span>
<div class="qcon__detail">
<h5 class="heading">
<a rel="noreferrer noopener" href="https://qconsf.com/?utm_source=infoq&utm_medium=referral&utm_campaign=footer_qsf26" target="_blank">QCon San Francisco</a>
</h5>
<span class="meta date">November 16-20, 2026</span>
</div>
</li>
<li><span class="icon event__type conference"></span>
<div class="qcon__detail">
<h5 class="heading">
<a rel="noreferrer noopener" href="https://newyork.qcon.ai/?utm_source=infoq&utm_medium=referral&utm_campaign=footer_qainy26" target="_blank">QCon AI New York</a>
</h5>
<span class="meta date">December 15-16, 2026</span>
</div>
</li>
<li><span class="icon event__type conference"></span>
<div class="qcon__detail">
<h5 class="heading">
<a rel="noreferrer noopener" href="https://qconlondon.com/?utm_source=infoq&utm_medium=referral&utm_campaign=footer_qlondon27" target="_blank">QCon London 2027</a>
</h5>
<span class="meta date">April 13-16, 2027</span>
</div>
</li>
</ul>
</div>
<div data-col="1/4" class="footer__social-wrap">
<h4 class="heading text-left footer__heading">Follow us on </h4>
<div class="social__links columns social__links__row">
<a href="https://www.youtube.com/infoq"><span class="icon icon__large icon__social icon__youtube"></span><div><span class="social__count">Youtube</span><span class="social__followers">232K Followers</span></div></a>
<a href="http://www.linkedin.com/company/infoq"><span class="icon icon__large icon__social icon__linkedin"></span><div><span class="social__count">Linkedin</span><span class="social__followers">26K Followers</span></div></a>
<a rel="noreferrer noopener" href="https://www.instagram.com/infoqdotcom/" target="_blank"><span class="icon icon__large icon__social icon__instagram"></span><div><span class="social__count">Instagram</span><span class="social__followers">New</span></div></a>
<a href="#" id="footerNewsletterRssLink"><span class="icon icon__large icon__social icon__rss"></span><div><span class="social__count">RSS</span><span class="social__followers">19K Readers</span></div></a>
<a rel="noreferrer noopener" href="http://twitter.com/infoq" target="_blank"><span class="icon icon__large icon__social icon__twitter"></span><div><span class="social__count">X</span><span class="social__followers">57.1k Followers</span></div></a>
<a rel="noreferrer noopener" href="https://www.facebook.com/InfoQ-75911537320" target="_blank"><span class="icon icon__large icon__social icon__fb"></span><div><span class="social__count">Facebook</span><span class="social__followers">21K Likes</span></div></a>
<a rel="noreferrer noopener" href="https://bsky.app/profile/infoq.com" target="_blank"><span class="icon icon__large icon__social icon__bluesky"></span><div><span class="social__count">Bluesky</span><span class="social__followers">New</span></div></a>
</div>
</div>
<div data-col="1/4" class="footer__stayin-wrap">
<h4 class="heading text-left footer__heading">Stay in the know</h4>
<div class="stayIn_panel_container">
<a href="/podcasts/" class="stayIn_panel" target="_blank" rel="noreferrer noopener"><span>The InfoQ Podcast</span><picture><source srcset="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLXBvZGNhc3QuanBnIiwiZWRpdHMiOiB7IndlYnAiOiB7ICJxdWFsaXR5Ijo4MH19fQ==" type="image/webp"><source srcset="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLXBvZGNhc3QuanBnIiwiZWRpdHMiOiB7ImpwZWciOiB7ICJxdWFsaXR5Ijo4MH19fQ==" type="image/webp"><img src="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLXBvZGNhc3QuanBnIiwiZWRpdHMiOiB7ImpwZWciOiB7ICJxdWFsaXR5Ijo4MH19fQ==" loading="lazy" width="65px" height="64px" alt="The InfoQ Podcast Logo - Stay in the know"></picture></a>
<a href="/podcasts/#engineering_culture" class="stayIn_panel" target="_blank" rel="noreferrer noopener"><span>Engineering Culture Podcast</span><picture><source srcset="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLXBvZGNhc3QtZW5naW5lZXJpbmctY3VsdHVyZS5qcGciLCJlZGl0cyI6IHsid2VicCI6IHsgInF1YWxpdHkiOjgwfX19" type="image/webp"><source srcset="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLXBvZGNhc3QtZW5naW5lZXJpbmctY3VsdHVyZS5qcGciLCJlZGl0cyI6IHsianBlZyI6IHsgInF1YWxpdHkiOjgwfX19" type="image/webp"><img src="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLXBvZGNhc3QtZW5naW5lZXJpbmctY3VsdHVyZS5qcGciLCJlZGl0cyI6IHsianBlZyI6IHsgInF1YWxpdHkiOjgwfX19" loading="lazy" width="65px" height="64px" alt="Engineering Culture Podcast Logo - Stay in the knoww"></picture></a>
<a href="/software-architects-newsletter/" class="stayIn_panel" target="_blank" rel="noreferrer noopener"><span>The Software Architects' Newsletter</span><picture><source srcset="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLWFyY2hpdGVjdC1uZXdzbGV0dGVyLmpwZyIsImVkaXRzIjogeyJ3ZWJwIjogeyAicXVhbGl0eSI6ODB9fX0=" type="image/webp"><source srcset="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLWFyY2hpdGVjdC1uZXdzbGV0dGVyLmpwZyIsImVkaXRzIjogeyJqcGVnIjogeyAicXVhbGl0eSI6ODB9fX0=" type="image/webp"><img src="https://imgopt.infoq.com/eyJidWNrZXQiOiAiYXNzZXRzLmluZm9xLmNvbSIsImtleSI6ICJ3ZWIvZm9vdGVyL2luZm9xLWFyY2hpdGVjdC1uZXdzbGV0dGVyLmpwZyIsImVkaXRzIjogeyJqcGVnIjogeyAicXVhbGl0eSI6ODB9fX0=" loading="lazy" width="65px" height="64px" alt="The Software Architects' Newsletter Logo - Stay in the know"></picture></a>
</div>
</div>
</div>
</section>
<section data-nosnippet class="container footer__bottom section white align__left">
<div class="container__inner columns">
<div data-col="2/3" class="column">
<div class="footer__contact contact columns">
<span data-col="1/6">
General Feedback
<a href="mailto:feedback@infoq.com">feedback@infoq.com</a>
</span>
<span data-col="1/6">
Advertising
<a href="mailto:sales@infoq.com">sales@infoq.com</a>
</span>
<span data-col="1/6">
Editorial
<a href="mailto:editors@infoq.com">editors@infoq.com</a>
</span>
<span data-col="1/6">
Marketing
<a href="mailto:marketing@infoq.com">marketing@infoq.com</a>
</span>
</div>
</div>
<div class="column" data-col="1/3">
<p class="footer__more">
InfoQ.com and all content copyright &#169; 2006-2026 C4Media Inc.<br/>
<a href="/privacy-notice" target="_blank">Privacy Notice</a>, <a href="/terms-and-conditions " target="_blank">Terms And Conditions</a>, <a href="/cookie-policy " target="_blank">Cookie Policy</a>
</p>
</div>
</div>
</section>
</footer>
<!-- ####### SITE END ######### -->
</div>
<div class="intbt">
<a href="/int/bt/" title="bt">BT</a>
</div>
<script type="text/javascript">
$.when(humanDetectionAsync()).then(
function(status) {
$.getScript("/scripts/__hd.ifq?hdt=3oCTVQJcuE725ucu&ha=" + status);
}
);
</script>
<script type="text/javascript">
var pageFullyLoaded = false;
// this event is fired by frontend once all the necessary things have been done(mobile display, moving vcr boxes around when needed...)
infoq.event.on('loaded', function(e) {
pageFullyLoaded = true;
});
infoq.event.on('pageWidthChanged', function(e) {
// re-execute tracking vcr impressions when this event happens (it only happens when elements are added/removed from page)
// doTrackVcrImpressions takes into account the data-trk-impr="true" if =false the element was already tracked.
// this is needed when switching from mobile to desktop or when layout on mobile changes and desktop version is displayed. New elements become visible.
Tracker.safeExec(Tracker.doTrackVcrImpressions);
// these 2 need to be called also because we might be on a content page. In case we are not nothing happens
VCR.doTrackingCustomRightbar();
VCR.doTrackingCustomRightbarForPresentations()
});
//check to see if error page
if(window.device !== undefined) {
var intervalImpressions = setInterval(function() {
var shouldTrack = false;
// no vcr widgets on index pages
if(window.isIndexPage) {
if(window.sponsoredPodcastDone === undefined || window.sponsoredPodcastDone) {
shouldTrack = true;
}
} else
//on homepage, bottom widget + 2 native widgets
if(InfoQConstants.pageType == "HOMEPAGE" && window.finishedVcrOptional1 && window.finishedRelatedVcr && (window.finishedVcrOptional2 === undefined || window.finishedVcrOptional2)) {
shouldTrack = true;
} else
// rightbar widgets + native widgets + content vcr widgets
if((InfoQConstants.pageType == "NEWS_PAGE" || InfoQConstants.pageType == "ARTICLE_PAGE")
&& ((window.finishedRightbarVcr || window.finishedRightbarVcr === undefined) && window.contentVcrFinished)) {
shouldTrack = true;
} else if ((InfoQConstants.pageType == "PRESENTATION_PAGE")
&& (window.contentVcrFinished || window.contentVcrFinished === undefined)
&& (window.finishedRightbarVcr || window.finishedRightbarVcr === undefined)
) {
shouldTrack = true;
} else
// native widgets + content widgets
if(window.contentVcrFinished && (window.finishedVcrOptional1 || window.finishedVcrOptional1 === undefined)
&& (window.finishedVcrOptional2 || window.finishedVcrOptional2 === undefined)) {
shouldTrack = true;
}
// we start tracking only after the page is fully loaded, frontend signals that they finished everything related to page display.
if(shouldTrack && pageFullyLoaded) {
clearInterval(intervalImpressions);
Tracker.safeExec(Tracker.doTrackVcrImpressions);
// start tracking viewable impressions also only after everything is ready
function callbackRouter(entries, observer) {
var targets = new Array();
entries.forEach(function (entry) {
var target = entry.target;
if (target.dataset.trkView === 'false') return;
if (entry.intersectionRatio > 0) {
target.dataset.trkView = false;
targets.push(target);
}
});
Tracker.doTrackViewableImpressions(targets);
}
var elementsForTrackingViewableImpressions = document.querySelectorAll('[data-trk-view="true"]')
var observer = new IntersectionObserver(callbackRouter, { threshold: 0.3 });
elementsForTrackingViewableImpressions.forEach(observer.observe.bind(observer));
}
}, 500);
}
$(document).ready(function () {
// desktop notifications widget
Tracker.encodeNotificationLinks($(".f_notificationWidget"));
// mobile notifications widget
Tracker.encodeNotificationLinks($(".h_notifications"));
// desktop notifications page
Tracker.encodeNotificationLinks($(".notification-page"));
// mobile notifications page
Tracker.encodeNotificationLinks($(".notifications_page"));
});
</script>
<script type="text/javascript">
if(window.location.hash){
var hash = window.location.hash.substring(1);
if(hash == 'subscribe'){
$('html,body').animate({scrollTop: $('.ftxt3 > .newsletter').offset().top}, 'slow');
}
}
</script>
<script type="text/javascript">
var newsletterSubscriptionURL ='/newsletter/subscribe.action';
DynamicLinks.updateRssLinks('XnLuCjIf6z9BHCGZ6Nd4R76O32258ENr');
ContentSummary.setSelectedTab('en');
//when user enters the main content area show default topics in the topics bar
$("#content-wrapper").mouseenter(function() {
showDefaultTopics();
});
Bookmarks.contentTitle = "Managing Failure Modes in Microservice Architectures";
Bookmarks.apiUrl = '/widgets/bookmark.action';
Bookmarks.isContentBookmarked = "false";
</script>
<script type="text/javascript">
var $buoop = {vs:{i:6,f:1,o:10.1,s:1}}
$buoop.ol = window.onload;
$(document).ready(function() {
try {if ($buoop.ol) $buoop.ol();}catch (e) {}
var e = document.createElement("script");
e.setAttribute("type", "text/javascript");
e.setAttribute("src", "https://cdn.infoq.com/statics_s1_20260910111632/scripts/lib/browser-update-org/update.js");
document.body.appendChild(e);
});
</script>
<script>
!function(f,b,e,v,n,t,s)
{if(f.fbq)return;n=f.fbq=function(){n.callMethod?
n.callMethod.apply(n,arguments):n.queue.push(arguments)};
if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
n.queue=[];t=b.createElement(e);t.async=!0;
t.src=v;s=b.getElementsByTagName(e)[0];
s.parentNode.insertBefore(t,s)}(window,document,'script',
'https://connect.facebook.net/en_US/fbevents.js');
fbq('init', '842388869148196');
fbq('track', 'PageView');
</script>
<noscript>
<img height="1" width="1" src="https://www.facebook.com/tr?id=842388869148196&ev=PageView&noscript=1"/>
</noscript>
<script type='text/javascript'>
try {
mixpanel.track('page viewed', {
'page name' : document.title,
'url' : window.location.pathname
});
}
catch(err) {
}
</script>
</body>
</html>
<!-- s1 -->