53 lines
89 KiB
HTML
53 lines
89 KiB
HTML
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta http-equiv="x-ua-compatible" content="ie=edge"/><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"/><meta name="generator" content="Gatsby 5.16.0"/><meta data-react-helmet="true" name="description" content="Discover how to design a fault tolerant system that can detect and remediate failures at scale - even when they are partial or intermittent."/><meta data-react-helmet="true" property="og:site_name" content="Ably Realtime"/><meta data-react-helmet="true" property="og:type" content="website"/><meta data-react-helmet="true" property="og:url" content="https://ably.com/blog/engineering-dependability-and-fault-tolerance-in-a-distributed-system"/><meta data-react-helmet="true" property="og:title" content="Engineering a fault tolerant distributed system"/><meta data-react-helmet="true" property="og:description" content="Discover how to design a fault tolerant system that can detect and remediate failures at scale - even when they are partial or intermittent."/><meta data-react-helmet="true" property="og:image" content="https://files.ably.io/ghost/prod/2022/01/engineering-dependability-and-fault-tolerance.png"/><meta data-react-helmet="true" name="twitter:card" content="summary_large_image"/><meta data-react-helmet="true" name="twitter:site" content="@ablyrealtime"/><link href="https://voltaire.ably.com/styles.401201978d276adf9700.css" rel="stylesheet" type="text/css" media="screen"/><style>.gatsby-image-wrapper{position:relative;overflow:hidden}.gatsby-image-wrapper picture.object-fit-polyfill{position:static!important}.gatsby-image-wrapper img{bottom:0;height:100%;left:0;margin:0;max-width:none;padding:0;position:absolute;right:0;top:0;width:100%;object-fit:cover}.gatsby-image-wrapper [data-main-image]{opacity:0;transform:translateZ(0);transition:opacity .25s linear;will-change:opacity}.gatsby-image-wrapper-constrained{display:inline-block;vertical-align:top}</style><noscript><style>.gatsby-image-wrapper noscript [data-main-image]{opacity:1!important}.gatsby-image-wrapper [data-placeholder-image]{opacity:0!important}</style></noscript><script type="module">const e="undefined"!=typeof HTMLImageElement&&"loading"in HTMLImageElement.prototype;e&&document.body.addEventListener("load",(function(e){const t=e.target;if(void 0===t.dataset.mainImage)return;if(void 0===t.dataset.gatsbyImageSsr)return;let a=null,n=t;for(;null===a&&n;)void 0!==n.parentNode.dataset.gatsbyImageWrapper&&(a=n.parentNode),n=n.parentNode;const o=a.querySelector("[data-placeholder-image]"),r=new Image;r.src=t.currentSrc,r.decode().catch((()=>{})).then((()=>{t.style.opacity=1,o&&(o.style.opacity=0,o.style.transition="opacity 500ms linear")}))}),!0);</script><style data-styled="" data-styled-version="6.5.0">.gLNiCf{scroll-margin-top:16px;}/*!sc*/
|
||
data-styled.g13[id="layout__Main-sc-17rhhsn-0"]{content:"gLNiCf,"}/*!sc*/
|
||
.jTvGq{transition:transform 0.3s;position:sticky;top:64px;bottom:1rem;}/*!sc*/
|
||
data-styled.g22[id="sticky-sidebar-mobile__Sidebar-sc-gqtcvm-0"]{content:"jTvGq,"}/*!sc*/
|
||
.dYTnoj{position:fixed;top:0;left:0;right:0;bottom:0;background:rgba(0,0,0,0.5);z-index:1;transform:translateZ(0);pointer-events:none;visibility:hidden;opacity:0;transition:opacity 0.2s;height:100vh;}/*!sc*/
|
||
data-styled.g29[id="overlay__Overlay-sc-j0b6or-0"]{content:"dYTnoj,"}/*!sc*/
|
||
.ILNdO{max-height:calc(100vh - 64px);transition:transform 0.3s ease-in-out;transform:translateY(-100%);z-index:-1;}/*!sc*/
|
||
data-styled.g30[id="table-of-contents-mobile__Container-sc-1g3ebrb-0"]{content:"ILNdO,"}/*!sc*/
|
||
</style><link rel="alternate" type="application/rss+xml" title="The Ably Blog" href="https://voltaire.ably.com/blog/rss.xml"/><script>window.dataLayer = window.dataLayer || [];window.dataLayer.push({"platform":"gatsby","gaPageView":false}); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= 'https://www.googletagmanager.com/gtm.js?id='+i+dl+'';f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer', 'GTM-TZ37KKW');</script><script src="https://cdn-ukwest.onetrust.com/scripttemplates/otSDKStub.js" data-domain-script="b1eb1bbc-c8f1-47ad-87a5-4a2a6f01202c"></script><script>window.OptanonWrapper = function(){};</script><link data-react-helmet="true" rel="canonical" href="https://ably.com/blog/engineering-dependability-and-fault-tolerance-in-a-distributed-system"/><link data-react-helmet="true" rel="icon" href="/favicon.svg" type="image/svg+xml"/><link data-react-helmet="true" rel="icon" href="/favicon-96x96.png" sizes="96x96" type="image/png"/><link data-react-helmet="true" rel="icon" href="/favicon.ico" sizes="48x48 32x32 16x16"/><link data-react-helmet="true" rel="apple-touch-icon" href="/apple-touch-icon.png"/><link data-react-helmet="true" rel="manifest" href="/site.webmanifest"/><link data-react-helmet="true" rel="preconnect" href="https://fonts.googleapis.com"/><link data-react-helmet="true" rel="preconnect" href="https://fonts.gstatic.com" crossorigin=""/><link data-react-helmet="true" href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:ital,wght@0,100..800;1,100..800&family=Manrope:wght@200..800&family=Source+Code+Pro:wght@600&display=swap" rel="stylesheet"/><script data-react-helmet="true" type="application/ld+json">{"@context":"https://schema.org","@graph":[{"@type":"TechArticle","@id":"https://ably.com/blog/engineering-dependability-and-fault-tolerance-in-a-distributed-system#article","headline":"Engineering a fault tolerant distributed system","abstract":"Discover how to design a fault tolerant system that can detect and remediate failures at scale - even when they are partial or intermittent.","mainEntityOfPage":"https://ably.com/blog/engineering-dependability-and-fault-tolerance-in-a-distributed-system","datePublished":"2021-02-15T00:00:00+00:00","dateModified":"2022-10-17T00:00:00+00:00","inLanguage":"en-US","image":{"@type":"ImageObject","url":"https://files.ably.io/ghost/prod/2022/01/engineering-dependability-and-fault-tolerance.png"},"author":{"@type":"Person","name":"Paddy Byers","url":"https://ably.com/blog/author/paddy","worksFor":{"@id":"https://ably.com#organization"}},"publisher":{"@id":"https://ably.com#organization"}},{"@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"item":{"@id":"https://ably.com","name":"Ably"}},{"@type":"ListItem","position":2,"item":{"@id":"https://ably.com/blog","name":"Blog"}},{"@type":"ListItem","position":3,"item":{"@id":"https://ably.com/blog/engineering-dependability-and-fault-tolerance-in-a-distributed-system","name":"Engineering a fault tolerant distributed system"}}]},{"@type":"Organization","@id":"https://ably.com#organization","name":"Ably","url":"https://ably.com","logo":{"@type":"ImageObject","url":"https://ably.com/favicon-512x512.png"},"knowsAbout":["WebSockets","realtime messaging","pub/sub","AI Transport","durable sessions","token streaming","session continuity"],"sameAs":["https://www.wikidata.org/wiki/Q139764920","https://www.linkedin.com/company/ably-realtime/","https://twitter.com/ablyrealtime","https://github.com/ably","https://www.crunchbase.com/organization/ably-realtime","https://www.g2.com/products/ably"]},{"@type":"WebSite","@id":"https://ably.com#website","name":"Ably","url":"https://ably.com","publisher":{"@id":"https://ably.com#organization"}}]}</script><style id="sandpack">--sxs{--sxs:1 sp-k-eyOShd sp-k-iOHdLQ}@media{@keyframes sp-k-eyOShd{0%{opacity:0}100%{opacity:1}}@keyframes sp-k-iOHdLQ{0%{transform:rotateX(-25.5deg) rotateY(45deg)}100%{transform:rotateX(-25.5deg) rotateY(405deg)}}}--sxs{--sxs:2 sp-c-gMfcns sp-c-bxeRRt sp-c-jKPvnt sp-c-fWymNx sp-c-euXojQ sp-c-bpmgvy sp-c-PJLV}@media{.sp-c-gMfcns svg{margin:auto}.sp-c-bxeRRt{-webkit-appearance:none;appearance:none;outline:none;display:flex;align-items:center;font-size:inherit;font-family:inherit;background-color:transparent;transition:color var(--sp-transitions-default), background var(--sp-transitions-default);cursor:pointer;color:var(--sp-colors-clickable);border:0;text-decoration:none}.sp-c-bxeRRt:disabled{color:var(--sp-colors-disabled)}.sp-c-bxeRRt:hover:not(:disabled,[data-active='true']){color:var(--sp-colors-hover)}.sp-c-bxeRRt[data-active="true"]{color:var(--sp-colors-accent)}.sp-c-bxeRRt svg{min-width:var(--sp-space-4);width:var(--sp-space-4);height:var(--sp-space-4)}.sp-c-bxeRRt.sp-c-gMfcns{padding:var(--sp-space-1);height:var(--sp-space-7);display:flex}.sp-c-bxeRRt.sp-c-gMfcns.sp-c-bxeRRt:not(:has(span)){width:var(--sp-space-7)}.sp-c-bxeRRt.sp-c-gMfcns.sp-c-bxeRRt:has(svg + span){padding-right:var(--sp-space-3);padding-left:var(--sp-space-2);gap:var(--sp-space-1)}.sp-c-jKPvnt{padding:0 var(--sp-space-1) 0 var(--sp-space-1);border-radius:var(--sp-border-radius);margin-left:var(--sp-space-1);width:var(--sp-space-5);visibility:hidden;cursor:pointer;position:absolute;right:0px}.sp-c-jKPvnt svg{width:var(--sp-space-3);height:var(--sp-space-3);display:block;position:relative;top:1px}.sp-c-fWymNx{margin:0;display:block;font-family:var(--sp-font-mono);font-size:var(--sp-font-size);color:var(--sp-syntax-color-plain);line-height:var(--sp-font-lineHeight)}.sp-c-euXojQ{display:flex;flex-direction:column;width:100%;position:relative;background-color:var(--sp-colors-surface1);gap:1px}.sp-c-euXojQ:has(.sp-stack){background-color:var(--sp-colors-surface2)}.sp-c-bpmgvy{transform:translate(-4px, 9px) scale(0.13, 0.13)}.sp-c-bpmgvy *{position:absolute;width:96px;height:96px}}--sxs{--sxs:3 sp-c-PJLV-kCOVwI-status-pass sp-c-PJLV-kEzYsr-status-fail sp-c-PJLV-gHAhSA-status-skip sp-c-PJLV-jgnHyR-status-title sp-c-PJLV-iCgxLS-status-run sp-c-PJLV-bnDZSy-status-pass sp-c-PJLV-eYuGwt-status-fail}@media{.sp-c-PJLV-kCOVwI-status-pass{color:var(--test-pass)}.sp-c-PJLV-kEzYsr-status-fail{color:var(--test-fail)}.sp-c-PJLV-gHAhSA-status-skip{color:var(--test-skip)}.sp-c-PJLV-jgnHyR-status-title{color:var(--test-title)}.sp-c-PJLV-iCgxLS-status-run{background:var(--test-run);color:var(--sp-colors-surface1)}.sp-c-PJLV-bnDZSy-status-pass{background:var(--test-pass);color:var(--sp-colors-surface1)}.sp-c-PJLV-eYuGwt-status-fail{background:var(--test-fail);color:var(--sp-colors-surface1)}}</style></head><body><noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-TZ37KKW" height="0" width="0" style="display: none; visibility: hidden" aria-hidden="true"></iframe></noscript><div id="___gatsby"><div style="outline:none" tabindex="-1" id="gatsby-focus-wrapper"><div class="absolute top-0 left-0 right-0 w-full z-50" id="meganav" data-testid="meganav" style="height:64px"><header role="banner" style="height:64px;top:0" class="fixed left-0 top-0 w-full z-50 bg-neutral-000 dark:bg-neutral-1300 border-b border-neutral-300 dark:border-neutral-1000 transition-all duration-300 ease-in-out"><div class="flex items-center h-full max-w-screen-xl mx-auto ui-grid-px"><nav class="flex flex-1 h-full items-center"><a href="/" class="items-center gap-2 justify-center h-full focus-base rounded mr-4 lg:mr-8 flex dark:hidden"><img src="https://voltaire.ably.com/static/ably-logo-200721285a51085f43e8a849a85667bc.svg" width="96px" alt="Ably logo"/></a><a href="/" class="items-center gap-2 justify-center h-full focus-base rounded mr-4 lg:mr-8 hidden dark:flex"><img src="https://voltaire.ably.com/static/ably-logo-white-749ef5fc6551d0e3b4940c2e3e42463f.svg" width="96px" alt="Ably logo"/></a><div class="hidden md:flex flex-1 items-center h-full"><nav aria-label="Main" data-orientation="horizontal" dir="ltr" class="justify-left z-40 flex w-full"><div style="position:relative"><ul data-orientation="horizontal" class="flex list-none center" dir="ltr"><li><button id="radix-:R32p9:-trigger-radix-:R2r2p9:" data-state="closed" aria-expanded="false" aria-controls="radix-:R32p9:-content-radix-:R2r2p9:" class="group outline-none focus:outline-none select-none cursor-pointer relative rounded-md hover:bg-neutral-100 dark:hover:bg-neutral-1200 [&[data-state=open]]:bg-neutral-100 dark:[&[data-state=open]]:bg-neutral-1200 [&[data-state=open]]:text-neutral-1300 dark:[&[data-state=open]]:text-neutral-000 ui-text-label3 font-semibold text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 px-3 py-2 flex items-center justify-between" data-radix-collection-item="">Products</button></li><li><button id="radix-:R32p9:-trigger-radix-:R4r2p9:" data-state="closed" aria-expanded="false" aria-controls="radix-:R32p9:-content-radix-:R4r2p9:" class="group outline-none focus:outline-none select-none cursor-pointer relative rounded-md hover:bg-neutral-100 dark:hover:bg-neutral-1200 [&[data-state=open]]:bg-neutral-100 dark:[&[data-state=open]]:bg-neutral-1200 [&[data-state=open]]:text-neutral-1300 dark:[&[data-state=open]]:text-neutral-000 ui-text-label3 font-semibold text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 px-3 py-2 flex items-center justify-between" data-radix-collection-item="">Solutions</button></li><li><button id="radix-:R32p9:-trigger-radix-:R6r2p9:" data-state="closed" aria-expanded="false" aria-controls="radix-:R32p9:-content-radix-:R6r2p9:" class="group outline-none focus:outline-none select-none cursor-pointer relative rounded-md hover:bg-neutral-100 dark:hover:bg-neutral-1200 [&[data-state=open]]:bg-neutral-100 dark:[&[data-state=open]]:bg-neutral-1200 [&[data-state=open]]:text-neutral-1300 dark:[&[data-state=open]]:text-neutral-000 ui-text-label3 font-semibold text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 px-3 py-2 flex items-center justify-between" data-radix-collection-item="">Company</button></li><a href="/pricing" class="ui-text-label3 font-semibold text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 px-3 py-2 flex items-center justify-between" data-radix-collection-item="">Pricing</a><a href="/docs" class="ui-text-label3 font-semibold text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 px-3 py-2 flex items-center justify-between" data-radix-collection-item="">Docs</a></ul></div><div class="absolute top-full flex justify-left"></div></nav></div></nav><div class="flex md:hidden flex-1 items-center justify-end gap-6 h-full"><button class="cursor-pointer focus-base rounded flex items-center p-0" aria-expanded="false" aria-controls="mobile-menu" aria-label="Toggle menu"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" aria-hidden="true" data-slot="icon" class="text-neutral-1300 dark:text-neutral-000" style="width:1.5rem;height:1.5rem"><path stroke-linecap="round" stroke-linejoin="round" d="M3.75 6.75h16.5M3.75 12h16.5m-16.5 5.25h16.5"></path></svg></button></div><nav class="md:flex-1 md:items-center md:justify-end flex-col md:flex-row border-t-[1px] border-neutral-300 md:border-t-0 md:gap-4 pt-3 pb-4 md:py-0 hidden md:flex flex-1 items-center h-full md:gap-x-6"><a class="font-sans !text-label2 md:!text-label3 font-semibold py-4 text-neutral-1300 dark:text-neutral-000 md:text-neutral-1000 dark:md:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 active:text-neutral-1300 dark:active:text-neutral-000 transition-colors flex items-center gap-1.5 px-4 md:px-0 leading-none" href="/contact">Contact us</a><div class="flex gap-3 pt-3 md:py-0 px-4 md:px-0"><a class="ui-button-secondary flex-1 md:flex-none md:ui-button-secondary-xs hover:text-neutral-1300 dark:hover:text-neutral-000" role="button" tabindex="0" href="/login">Login</a><a class="ui-button-primary flex-1 md:flex-none md:ui-button-primary-xs hover:text-neutral-000 dark:hover:text-neutral-1300" role="button" tabindex="0" href="/sign-up">Start free</a></div></nav></div></header></div><div class="ui-flash" data-id="ui-flashes"></div><main class="layout__Main-sc-17rhhsn-0 gLNiCf vt-blog-editorial font-sans antialiased" id="main" data-testid="layout-main"><div class="vt-blog-editorial-reading-progress" aria-hidden="true"><div class="vt-blog-editorial-reading-progress-bar" style="width:0%"></div></div><aside data-sidebar-sticking="false" class="sticky-sidebar-mobile__Sidebar-sc-gqtcvm-0 jTvGq md:hidden w-full z-20 bg-transparent"><div class="relative"><div class="font-sans ui-grid-px bg-white border-mid-grey border-b flex items-center py-4 relative z-10"><div class="ui-text-overline2 text-dark-grey font-medium">On this page</div><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="currentColor" aria-hidden="true" data-slot="icon" class="ml-auto absolute right-0 mr-6 sm:mr-8" style="width:1.5rem;height:1.5rem"><path fill-rule="evenodd" d="M4.22 6.22a.75.75 0 0 1 1.06 0L8 8.94l2.72-2.72a.75.75 0 1 1 1.06 1.06l-3.25 3.25a.75.75 0 0 1-1.06 0L4.22 7.28a.75.75 0 0 1 0-1.06Z" clip-rule="evenodd"></path></svg></div><div class="table-of-contents-mobile__Container-sc-1g3ebrb-0 ILNdO border-b border-color-mid-grey absolute bg-white overflow-y-auto w-full"></div><div class="overlay__Overlay-sc-j0b6or-0 dYTnoj"></div></div></aside><header class="vt-blog-editorial-header"><div class="vt-blog-editorial-header-back-row"><a href="/blog" class="vt-blog-editorial-header-back"><span aria-hidden="true">←</span> Blog</a></div><span class="vt-blog-editorial-hero-eyebrow"><a href="/blog/ably-engineering" class="vt-blog-editorial-hero-eyebrow-cat no-underline" data-testid="hero-kicker">Ably engineering</a></span><h1 id="article-heading" class="vt-blog-editorial-h1">Engineering a fault tolerant distributed system</h1><p class="vt-blog-editorial-dek">Discover how to design a fault tolerant system that can detect and remediate failures at scale - even when they are partial or intermittent.</p><div class="vt-blog-editorial-byline"><a href="/blog/author/paddy" class="vt-blog-editorial-byline-author"><img src="https://ik.imagekit.io/ably/ghost/prod/2019/05/paddy-byers-83e6fa6a5d47687601b0586d529a4619bbe68178b5ff4d1ba8636e4612ea716f.jpg?tr=w-72,q-50" alt="Paddy Byers" width="72" class="vt-blog-editorial-byline-avatar"/><span class="vt-blog-editorial-byline-name">Paddy Byers</span><span class="vt-blog-editorial-byline-role">Co-founder and CTO at Ably.</span></a><div class="vt-blog-editorial-byline-meta-col"><span class="vt-blog-editorial-byline-meta">Feb 15, 2021</span><span class="vt-blog-editorial-meta-dot">·</span><span class="vt-blog-editorial-byline-meta">20 min read</span></div></div><div class="vt-blog-editorial-header-rule" aria-hidden="true"></div></header><div class="vt-blog-editorial-grid"><article aria-labelledby="article-heading"><figure class="vt-blog-editorial-feature-figure"><img src="https://ik.imagekit.io/ably/ghost/prod/2022/01/engineering-dependability-and-fault-tolerance.png?tr=w-1280,q-50" alt="Engineering a fault tolerant distributed system" width="1280" class="w-full h-auto block"/></figure><div class="ghost-blog-post-body"><p>The core challenge of engineering a fault tolerant system is understanding the nature of failures, especially when partial or intermittent. This article explains how to design a fault tolerant system that can detect and remediate failures at scale.</p><p>Some of the topics that we will cover include:</p><ul><li><a href="#what-is-a-fault-tolerant-system">What is a fault tolerant system?</a></li><li><a href="#fault-tolerant-system-design">Fault tolerant system design for stateful and stateless services</a></li><li><a href="#architectural-approaches-to-achieve-reliability">Architectural approaches to achieve reliability</a></li><li><a href="#engineering-a-fault-tolerant-distributed-system">Engineering a fault tolerant distributed system</a></li></ul><h2 id="the-definition-of-dependability">The definition of dependability</h2><p><strong>Dependability</strong> is a measure of both the availability <em>and</em> reliability of a service.</p><p><strong>Availability</strong> is when a product or service is available for use when required.</p><p><strong>Reliability</strong> is whether a product or service works as expected. </p><p>If availability is the assurance of uptime, reliability preserves quality of that uptime in terms of functionality and user experience.</p><h2 id="what-is-a-fault-tolerant-system">What is a fault tolerant system?</h2><p>A fault tolerant system shows dependability in the presence of component or subsystem failures.</p><p>Fault tolerant systems stay available and reliable because they are engineered to minimize the impact of adverse circumstances and remain dependable. </p><p>Fault tolerant design aims to provide continuity both to business and to the user experience.</p><h2 id="redundancy">Redundancy</h2><p>In most cases, the primary basis for fault tolerant design is <strong>redundancy</strong>: exceeding the capacity required to deliver service.</p><p>In the physical world there is a distinction between a situation where it is acceptable to stop a service and later resume it (such as stopping to put on the spare car wheel) and a case where the service must continue (such as redundancy in engines to keep an airplane in flight).</p><p>The level of continuity needed impacts the way that redundant capacity is provided (a car has a spare wheel in the trunk rather than in constant use).</p><!--kg-card-begin: html--><a name="fault-tolerant-system-design"></a><!--kg-card-end: html--><h2 id="fault-tolerant-system-design">Fault tolerant system design</h2><p>In large-scale systems, the assumption has to be that component failures will happen sooner or later. Any individual failure must be assumed as imminent and, collectively, component failures must be expected to be occurring continuously.</p><p>By contrast with the physical world, failures in digital systems are typically non-binary. The classical measures of component reliability (e.g. Mean Time Between Failures, or MTBF) do not apply; services degrade along a gradient of failure. </p><p>For example, a system component might work intermittently, or produce misleading output. Or you might be dependent on external partners who don’t notify you of a failure until it becomes serious on their end, making your work more difficult. </p><p>Being tolerant to non-binary failures requires a lot of thought, engineering and, sometimes, human intervention. Each potential failure must be identified and classified, and must then be capable of being remediated rapidly, or avoided through extensive testing and robust design decisions. </p><h2 id="stateless-services">Stateless services</h2><p><strong>Stateless components</strong> have no long-lived state. Each invocation of service can be performed independently of any previous invocation. Fault tolerant design for these components is comparatively straightforward: have sufficient resources <strong>available</strong> so that any individual invocation can be handled even if some of the resources have failed.</p><p>The availability of resources directly translates into availability of the layer as a whole. Having access to extra resources whose failures are statistically independent is key to keeping the system going. Wherever possible, layers are designed to be stateless as a key enabler not only of availability, but also of scalability. </p><p>For stateless objects, it suffices to have multiple and independently available components to continue to provide service. Without state, durability of any single component is not a concern.</p><figure class="kg-card kg-image-card"><img src="https://ik.imagekit.io/ably/ghost/prod/2021/02/fault-tolerance-of-a-stateless-component.png" class="kg-image" alt="Fault tolerant design of stateless components: a load balancer receives a request and selects an element to satisfy the request." loading="lazy" srcset="https://ik.imagekit.io/ably/ghost/prod/2021/02/fault-tolerance-of-a-stateless-component.png?tr=w-380,q-50 380w,
|
||
https://ik.imagekit.io/ably/ghost/prod/2021/02/fault-tolerance-of-a-stateless-component.png?tr=w-760,q-50 760w,
|
||
https://ik.imagekit.io/ably/ghost/prod/2021/02/fault-tolerance-of-a-stateless-component.png?tr=w-1520,q-50 1520w" sizes="
|
||
(min-width: 1520px) 1520px,
|
||
100vw
|
||
"></figure><p>However, simply having extra resources is not enough: you also have to use them effectively. You have to have a way of detecting resource availability, and to load-balance among redundant resources. </p><p>As such, the questions that you have to answer are:</p><ul><li>How do you survive different kinds of failure?</li><li>What level of redundancy is possible?</li><li>What is the resource and performance cost of maintaining those levels of redundancy?</li><li>What is the operational cost of managing those levels of redundancy?</li></ul><p>The consequent trade-offs are among the following:</p><ul><li>Customer requirements for achieving high availability</li><li>Business operational cost</li><li>Real world engineering practicality of actually making it possible</li></ul><p>Redundant components, and in turn their dependencies, need to be engineered, configured and operated in <a href="https://www.ntnu.edu/documents/624876/1277590549/chapt04-1.pdf">a way that ensures that any failures are statistically independent</a>. The simple math is: statistically independent failures render your chances of a catastrophic failure exponentially lower as you increase the level of redundancy. If the failures are set up to occur in statistical silos, then there is no cumulative effect, and you decrease the likelihood of complete failure by a whole order of magnitude with each additional redundant resource.</p><p>At Ably, to increase statistical independence of failures, <a href="https://ably.com/network">we place/distribute capacity in multiple availability zones and in multiple regions</a>. Offering service from multiple availability zones within the same region is relatively simple: AWS enables this with very little effort. Availability zones generally have a good track record of failing independently, so this step by itself enables the existence of sufficient redundancy to support very high levels of availability.</p><p>However, this isn’t the whole story. It isn’t sufficient to rely on any specific region for multiple reasons – sometimes multiple availability zones (AZs) do fail at the same time; sometimes there might be local connectivity issues making the region unreachable; and sometimes there might simply be capacity limitations in a region that prevent all services from being supportable there. As a result, we also promote service availability by providing service in multiple regions. This is the ultimate way of ensuring statistical independence of failures.</p><p>Establishing redundancy that spans multiple regions is not as straightforward as supporting multiple AZs. For example, it doesn’t make sense simply to have a load balancer distributing requests among regions; the load balancer itself exists in some region and could become unavailable. </p><p>Instead, we use a <a href="https://faqs.ably.com/routing-around-network-and-dns-issues">combination of measures</a> to ensure that client requests can at all times be routed to a region that is believed to be healthy and have service available. This routing will preferably be to the nearest region, but routing to non-local regions must be possible when the nearest region is unable to provide service.</p><h2 id="stateful-services">Stateful services</h2><p><strong>Stateful components</strong> are dependent on state to provide correct service. Having state implicitly links an invocation of the service to past and future invocations. The essence of fault tolerance for these components is <a href="https://ably.com/four-pillars-of-dependability#reliability">reliability</a>: the business continuity of <em>stateful</em> services. It is a substantially more complicated problem to solve than availability. </p><p>Stateful services have an intrinsic dependency on state that survives each individual invocation of service. Continuity of that state translates into correctness of the service provided by the layer as a whole. That requirement for continuity means that fault tolerance for these services is achieved by thinking of them in classic reliability terms. Redundancy needs to be continuously in use, in order for the state not to become lost in the case of failure. Fault detection and remediation needs to address the possible Byzantine failure modes via consensus formation mechanisms.</p><p>The most simplistic analogy is with airplane safety. An airplane crash is catastrophic because you – and your state – are on a <em>specific</em> airplane; it is <em>that</em> airplane which must provide continuous service. If it fails to do so, state is lost and you are afforded no opportunity to continue by migrating to a different airplane. </p><p>With anything that relies on state, when an alternate resource is selected, the requirement is to be able to carry on with the new resource where the previous resource left off. State is thus a requirement, and in those cases availability alone is insufficient.</p><p>At Ably, we provision enough reserve capacity for stateless resources to support all of our customers’ availability requirements. However, for stateful resources, not only do we need redundant resources, but also explicit mechanisms to make use of that redundancy, in order to support our service guarantees of <em>functional continuity</em>.</p><figure class="kg-card kg-image-card"><img src="https://ik.imagekit.io/ably/ghost/prod/2021/02/fault-tolerance-of-a-stateful-datastore.png" class="kg-image" alt="Fault tolerance of a stateful datastore: a query coordinator handles non-binary (Byzantine) failures via transactional replication of updates on multpile stores." loading="lazy" srcset="https://ik.imagekit.io/ably/ghost/prod/2021/02/fault-tolerance-of-a-stateful-datastore.png?tr=w-380,q-50 380w,
|
||
https://ik.imagekit.io/ably/ghost/prod/2021/02/fault-tolerance-of-a-stateful-datastore.png?tr=w-760,q-50 760w,
|
||
https://ik.imagekit.io/ably/ghost/prod/2021/02/fault-tolerance-of-a-stateful-datastore.png?tr=w-1520,q-50 1520w" sizes="
|
||
(min-width: 1520px) 1520px,
|
||
100vw
|
||
"></figure><p>For example, if processing for a particular channel is taking place on a particular instance within the cluster, and that instance fails (forcing that channel role to move), mechanisms must be in place to ensure that things can continue.</p><p>This operates at several levels. At one level, a mechanism must exist to ensure that that channel’s processing is reassigned to a different, healthy, resource. At another level, there needs to be assurance that the reassigned resource continues at exactly the point that processing was halted in the previous location. Further, each of these mechanisms is itself implemented and operated with a level of redundancy to meet the overall assurance requirements for the service.</p><p>The effectiveness of these continuity mechanisms directly translates into the behavior, and assurance of that behavior, at the service provision boundary. Taking one specific issue in the scenario above: for any given message, you need to know with certainty whether or not processing for that message has been completed. </p><p>When a client submits a message to Ably for publication, and the service accepts the message for publication, it acknowledges that attempt as successful or unsuccessful. At this point, the principal <em>availability</em> question is:</p><blockquote><svg width="24" height="19" viewBox="0 0 24 19" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M8.72445 19H0V12.7188C0 10.1771 0.217335 8.17708 0.652005 6.71875C1.10737 5.23958 1.93532 3.91667 3.13583 2.75C4.33635 1.58333 5.86805 0.666667 7.73092 0L9.43855 3.625C7.69987 4.20833 6.44761 5.02083 5.68176 6.0625C4.93661 7.10417 4.54334 8.48958 4.50194 10.2188H8.72445V19ZM23.2859 19H14.5614V12.7188C14.5614 10.1562 14.7788 8.14583 15.2135 6.6875C15.6688 5.22917 16.4968 3.91667 17.6973 2.75C18.9185 1.58333 20.4502 0.666667 22.2924 0L24 3.625C22.2613 4.20833 21.0091 5.02083 20.2432 6.0625C19.4981 7.10417 19.1048 8.48958 19.0634 10.2188H23.2859V19Z" fill="#03020D"/>
|
||
</svg>What fraction of the time does the service accept (and then process) the message, versus rejecting it?</blockquote><p>The minimum aim in this case is <a href="https://en.wikipedia.org/wiki/High_availability#Percentage_calculation">four, five, or even six 9s</a>.</p><p>If you attempt to publish and we let you know we weren’t able to do that, then that’s merely an <strong>availability shortcoming</strong>. It’s not great, but you end up with an awareness of the situation. </p><p>However, if we respond with success — "yes, we've received your message" — but then we fail to do all the onward processing of it, then that's a different kind of failure. That's a failure to uphold our functional service guarantee. That’s a <strong>reliability shortcoming</strong>, and it’s a far more <a href="https://ably.com/blog/practical-strategies-for-dns-failure">complicated problem to solve in a distributed system</a> — there is significant engineering effort and complexity devoted to meeting this requirement.</p><!--kg-card-begin: html--><a name="architectural-approaches-to-achieve-reliability"></a><!--kg-card-end: html--><h2 id="architectural-approaches-to-achieve-reliability">Architectural approaches to achieve reliability</h2><p>The following are two illustrations of the architectural approaches we adopt at Ably to make optimum use of redundancy within our message processing core.</p><!--kg-card-begin: html--><!--HubSpot Call-to-Action Code --><span class="hs-cta-wrapper" id="hs-cta-wrapper-7d515cef-595f-4305-8d1a-c1e9272b1d85"><span class="hs-cta-node hs-cta-7d515cef-595f-4305-8d1a-c1e9272b1d85" id="hs-cta-7d515cef-595f-4305-8d1a-c1e9272b1d85"><!--[if lte IE 8]><div id="hs-cta-ie-element"></div><![endif]--><a href="https://cta-redirect.hubspot.com/cta/redirect/6939709/7d515cef-595f-4305-8d1a-c1e9272b1d85" ><img class="hs-cta-img" id="hs-cta-img-7d515cef-595f-4305-8d1a-c1e9272b1d85" style="border-width:0px;" height="300" width="962" src="https://no-cache.hubspot.com/cta/default/6939709/7d515cef-595f-4305-8d1a-c1e9272b1d85.png" alt="New call-to-action"></a></span><script charset="utf-8" src="https://js.hscta.net/cta/current.js"></script><script type="text/javascript"> hbspt.cta.load(6939709, '7d515cef-595f-4305-8d1a-c1e9272b1d85', {"useNewLoader":"true","region":"na1"}); </script></span><!-- end HubSpot Call-to-Action Code --><!--kg-card-end: html--><h3 id="stateful-role-placement"><br>Stateful role placement</h3><p>In general, horizontal scalability is achieved by distributing work across a scalable cluster of processing resources. As far as entities that perform <strong>stateless</strong> processing are concerned, they can be distributed across available resources with <strong>few constraints</strong> on their placement: the location of any given operation can be decided based on load balancing, proximity, or other optimization considerations. </p><p>Meanwhile, in the case of <strong>stateful</strong> operations, the placement of processing roles must take their stateful nature into account such that, for example, all concerned entities can agree on the specific location of any given role.</p><p>A specific example is channel message processing: whenever a channel is active it gets assigned a resource that processes it. Being a stateful process, it is possible to achieve greater performance: we know more about the message at the time of processing, so we don’t have to look it up — we can just process it and send it on. </p><p>In order to distribute all the channels across all the available resources as uniformly as possible, we use <a href="https://ably.com/blog/implementing-efficient-consistent-hashing">consistent hashing</a>, with the underlying cluster discovery service providing consensus on both node health and hashring membership.</p><figure class="kg-card kg-image-card kg-card-hascaption"><img src="https://ik.imagekit.io/ably/ghost/prod/2021/02/hash-rings.png" class="kg-image" alt="A consistent placement algorithm is used to decide the placement of any resource in the cluster; all peers must achieve consensus on ring changes. " loading="lazy" srcset="https://ik.imagekit.io/ably/ghost/prod/2021/02/hash-rings.png?tr=w-380,q-50 380w,
|
||
https://ik.imagekit.io/ably/ghost/prod/2021/02/hash-rings.png?tr=w-760,q-50 760w,
|
||
https://ik.imagekit.io/ably/ghost/prod/2021/02/hash-rings.png?tr=w-1520,q-50 1520w" sizes="
|
||
(min-width: 1520px) 1520px,
|
||
100vw
|
||
"><figcaption>Achieving consensus via consistent placement algorithm in a hash ring</figcaption></figure><p>The placement mechanism is required not only to determine the initial placement of a role, but also to relocate the role whenever there is an event, such as node failure, that causes the role to move. Therefore, this dynamic placement mechanism is a core functionality in support of service continuity and reliability.</p><h3 id="detect-hash-resume">Detect, hash, resume</h3><p>The first step in mitigating failure is detecting it. As discussed above, this is classically difficult because of the need to achieve near-simultaneous consensus between distributed entities. Once detected, the updated hashring state implies the new location of that resource, and from that point onward the channel and the state of the failed resource must resume in the new location with continuity.</p><p>Even though the role failed and there was resulting loss of state, there needs to be sufficient state persisted (and with sufficient redundancy) that resumption of the role is possible with continuity. Resumption with continuity is what enables the service to be reliable in the presence of this kind of failure; the state of each in-flight message is preserved across the role relocation. If we were unable to do that, and simply re-established the role without continuity of state, we could ensure service availability — but not <em>reliability</em>.</p><h3 id="channel-persistence-layer">Channel persistence layer</h3><p>When a message is published, we perform some processing, decide on success or failure, then respond to the call. The reliability guarantee means having certainty that once a message is acknowledged, all onward transmission implied by that will in fact take place. This in turn means that we can only acknowledge a message once we know for a fact that it is durably persisted, with sufficient redundancy that it cannot subsequently be lost.</p><p>First, we record the receipt of the message in <strong>at least two different availability zones</strong> (AZs). Then we have the same multiple-AZ redundancy requirement for the onward processing itself. This is the core of the persistence layer at Ably: we write a message to multiple locations, and we also make sure the process of writing it is transactional. You come away knowing the writing of the message was either <em>not successful</em> or <em>unequivocally successful</em>. With assurance of that, subsequent processing can be guaranteed to occur eventually, even if there are failures in the roles responsible for that processing. </p><p>Ensuring that messages are persisted in multiple AZs enables us to assume that failures in those zones are independent, so a single event or cause cannot lead to loss of data. Arranging for this placement requires AZ-aware orchestration, and ensuring that writes to multiple locations are actually transactional requires distributed consensus in the message persistence layer.</p><figure class="kg-card kg-image-card"><img src="https://ik.imagekit.io/ably/ghost/prod/2021/02/ensure-redundant-components-fail-independently.png" class="kg-image" alt="Ensuring independence of failure of redundant components requires placing them into different availability zones such that requests can be made against coordinators in any such zone." loading="lazy" srcset="https://ik.imagekit.io/ably/ghost/prod/2021/02/ensure-redundant-components-fail-independently.png?tr=w-380,q-50 380w,
|
||
https://ik.imagekit.io/ably/ghost/prod/2021/02/ensure-redundant-components-fail-independently.png?tr=w-760,q-50 760w,
|
||
https://ik.imagekit.io/ably/ghost/prod/2021/02/ensure-redundant-components-fail-independently.png?tr=w-1520,q-50 1520w" sizes="
|
||
(min-width: 1520px) 1520px,
|
||
100vw
|
||
"></figure><p>Structuring things this way allows us to start to quantify, probabilistically, the level of assurance. A service failure can only arise if there is a compound failure — that is, a failure in one AZ and, before that failure has been remediated, a failure in a second AZ. </p><p>In our mathematical model, when one node fails, we know how long it takes to detect and achieve consensus on the failure, and how long it takes subsequently to relocate the role. Knowing this, together with the failure rate of each AZ, it is possible to model the probability of an occurrence of a compound failure that results in loss of continuity of state. This is the basis on which we are able to provide our guarantee of eight 9s of reliability.</p><!--kg-card-begin: html--><a name="engineering-a-fault-tolerant-distributed-system"></a><!--kg-card-end: html--><h2 id="engineering-a-fault-tolerant-distributed-system">Engineering a fault tolerant distributed system</h2><p>Once you have a theoretical approach to achieving a particular aspect of fault tolerance, there are numerous practical and systems engineering aspects to consider in the wider context of the system as a whole. We give some examples below. To explore this topic further, read and/or watch our deep-dive on the topic, <a href="https://ably.com/blog/hidden-scaling-issues-of-distributed-systems-real-world">Hidden scaling issues of distributed systems — system design in the real world</a>.</p><h3 id="consensus-formation-in-globally-distributed-systems">Consensus formation in globally-distributed systems</h3><p>The mechanisms described above — such as the role placement algorithm — can only be effective when all of the participating entities are in agreement on the topology of the cluster together with the status and health of each node. </p><p>This is a classical consensus formation problem, where the members of a cluster, who might themselves be subject to failure, must agree on the status of one of their members. Consensus formation protocols such as <a href="https://en.wikipedia.org/wiki/Raft_(algorithm)">Raft</a>/<a href="https://en.wikipedia.org/wiki/Paxos_(computer_science)">Paxos</a> are widely understood and have strong theoretical guarantees, but also have practical limitations in terms of scalability and bandwidth. In particular, they are not effective in networks spanning multiple regions because their efficiency breaks down if the latency becomes too high when communicating among peers.</p><p>Instead, our peers use <a href="https://ably.com/blog/what-is-a-distributed-systems-engineer#gossipprotocolsandconsensusalgorithmsunderpineverything">the Gossip protocol</a>, which is eventually-consistent, fault tolerant, and can work across regions. Gossip is used among regions to share topology information as well as to construct a network map, and this broad cluster state consensus is then used as the basis for sharing various details cluster-wide.</p><h3 id="health-is-not-binary">Health is not binary</h3><p>The classical theory that led to the development of Paxos and Raft originated from the recognition that failed or unhealthy entities oftentimes do not simply crash or stop responding. Instead, failing elements can exhibit partly-working behavior such as delayed responses, higher error rates or, in principle, arbitrary confusing or misleading behavior (see <a href="https://en.wikipedia.org/wiki/Byzantine_fault">Byzantine fault</a>). This issue is pervasive and even extends to the way a client interacts with the service.</p><p>When a client attempts connection to an endpoint in a given region, it is possible that the region is not available. If it’s entirely unavailable, then that’s a simple failure, and we would be able to detect that and redirect the client to greener pastures within the infrastructure. But what happens in practice is that regions can often be in a partially degraded state where they’re working <em>some</em> of the time. This means that the client itself has the problem of handling the failure — knowing where to redirect to obtain service, and knowing when to retry getting service from the default endpoint. </p><p>This is another example of the general fault tolerance problem: with many moving pieces, every single thing introduces additional complexity. If this part breaks, or that thing changes, how do you establish consensus on the <em>existence</em> of the change, the <em>nature</em> of the change, and the consequent <em>plan of action</em>?</p><h3 id="resource-availability-impacts-on-fault-tolerance">Resource availability impacts on fault tolerance</h3><p>At a very simple level, it is possible to provide redundant capacity only if the resources required are available. Occasionally there are situations where resources are simply unavailable at the moment they are demanded in a region, and it is necessary to offload demand to another region.</p><p>However, the resource availability problem also surfaces in a more challenging way when you realize that fault tolerance mechanisms themselves require resources to operate. </p><p>For example, there is a mechanism to manage the relocation of roles when the topology changes. This functionality itself requires resources in order to operate, such as CPU and memory on the affected instances. </p><p>But what if your disruption has come about precisely because your CPU or memory ran out? Now you’re attempting to handle those failures, but to do so you need… <em>CPU and memory</em>. This means that there are multiple dimensions in which you need to ensure that there exists a resource capacity margin, so that fault tolerance measures can be enacted at the time they are needed.</p><h3 id="resource-scalability-impacts-on-fault-tolerance">Resource scalability impacts on fault tolerance</h3><p>Further to the point above, it’s not just about availability of resources, but also about the rate at which demand for them scales. In the steady, healthy state you might have <em>N</em> channels, <em>N</em> connections, and <em>N</em> messages with <em>N</em> capacity to deal with it all. Now imagine there is some failure and ensuing disruption for that cluster of <em>N</em> instances. If the amount of work required to compensate for the disruption is of size N², then maintaining the capacity margin becomes unsustainable, and the only available remediation is to fail over to an undisrupted region or cluster. </p><p>Simplistic fault tolerance mechanisms can exhibit this kind of O(<em>N²</em>) behavior or worse, so approaches need to be analyzed with this in mind. It’s another reminder that while things can fail just because they’re broken in some way, it’s also possible that they can go wrong because they have an unforeseen scale or complexity or some other unsustainable resource implication.</p><h2 id="conclusion">Conclusion</h2><p>Fault tolerance is an approach to building systems able to withstand and mitigate adverse events and operating conditions in order to dependably continue delivering the level of service expected by the users of the system.</p><p>Dependability engineering in the physical world classically makes the distinction between <strong>availability</strong> and <strong>reliability</strong> and there are well-understood formulas for how to achieve each of these through fault tolerance and redundancy. At Ably we make the analogous distinction in the systems world between components that are <strong>stateless</strong> and those that are <strong>stateful</strong>. </p><p>Fault tolerance for <em>stateless</em> components is achieved in the same way as for availability in the physical world: through provision of redundant elements whose failure is statistically independent. Fault tolerance for <em>stateful</em> components can be compared to the physical reliability problem: it is the assurance of service without interruption. <em>Continuity of state</em> is essential in order that failures can be tolerated whilst preserving <em>correctness and continuity of the provided service</em>.</p><p>To be fault tolerant, a system must treat failures not as exceptional events, but as something expectedly routine. Unlike theoretical models of success and failure, threats to a system’s health in the real world are not binary and require complex theoretical and practical approaches to mitigate.</p><p>The Ably service is engineered with multiple layers that each make use of a wide range of fault tolerance mechanisms. In particular, we have confronted the hard engineering problems that arise which include stateful role placement, detection, hashing, and graceful resumption of service, among others. We’ve also articulated, and provide assurance of, the service guarantee at the channel persistence layer, such as assured onward processing once we acknowledge that we’ve received a message. </p><p>Beyond theoretical approaches, designing fault tolerant systems involves numerous <a href="https://ably.com/resources/datasheets/using-ably-at-scale">real-world systems engineering challenges</a>. This includes infrastructure availability and scalability issues, as well as dealing with consensus formation and orchestration of ever-fluctuating topologies of all clusters and nodes in the globally-distributed system, with unpredictable/hard-to-detect health status of any given entity on the network.</p><p>The Ably platform was designed from the ground up with these principles in mind, with the goal of delivering the best-in-class enterprise solution. This is why we can confidently provide our service-level guarantees of both availability and reliability – and thus dependability and fault tolerance.</p><p><em><a href="https://ably.com/contact">Get in touch</a> to learn more about Ably and how we can help you deliver seamless realtime experiences to your customers.</em></p><h2 id="latest-from-ably-engineering"><strong>Latest from Ably Engineering</strong></h2><!--kg-card-begin: html--><!--HubSpot Call-to-Action Code --><span class="hs-cta-wrapper" id="hs-cta-wrapper-7d515cef-595f-4305-8d1a-c1e9272b1d85"><span class="hs-cta-node hs-cta-7d515cef-595f-4305-8d1a-c1e9272b1d85" id="hs-cta-7d515cef-595f-4305-8d1a-c1e9272b1d85"><!--[if lte IE 8]><div id="hs-cta-ie-element"></div><![endif]--><a href="https://cta-redirect.hubspot.com/cta/redirect/6939709/7d515cef-595f-4305-8d1a-c1e9272b1d85" ><img class="hs-cta-img" id="hs-cta-img-7d515cef-595f-4305-8d1a-c1e9272b1d85" style="border-width:0px;" height="300" width="962" src="https://no-cache.hubspot.com/cta/default/6939709/7d515cef-595f-4305-8d1a-c1e9272b1d85.png" alt="New call-to-action"></a></span><script charset="utf-8" src="https://js.hscta.net/cta/current.js"></script><script type="text/javascript"> hbspt.cta.load(6939709, '7d515cef-595f-4305-8d1a-c1e9272b1d85', {"useNewLoader":"true","region":"na1"}); </script></span><!-- end HubSpot Call-to-Action Code --><!--kg-card-end: html--><ul><li><a href="https://ably.com/blog/stretching-a-point-economics-of-elastic-infrastructure">Stretching a point: the economics of elastic infrastructure ?</a></li><li><a href="https://ably.com/blog/best-practices-for-on-call-processes">Save your engineers' sleep: best practices for on-call processes</a></li><li><a href="https://ably.com/blog/how-we-load-tested-control-api">Squid game: how we load-tested Ably’s Control API</a></li><li><a href="https://ably.com/blog/achieving-exactly-once-message-processing-with-ably">Achieving exactly-once delivery with Ably</a></li><li><a href="https://ably.com/blog/limits-aws-network-load-balancers">Balancing act: the current limits of AWS network load balancers</a></li></ul></div><aside class="vt-blog-editorial-tryit" aria-label="Try Ably"><div class="vt-blog-editorial-tryit-rule"></div><div><div class="vt-blog-editorial-tryit-kicker">Try it</div><h3 class="vt-blog-editorial-tryit-heading">Build dependable realtime in minutes.</h3><p class="vt-blog-editorial-tryit-body">Ably runs the same patterns we write about — in production, at scale. Free tier includes 6M messages a month, no card required.</p><div class="vt-blog-editorial-tryit-actions"><a href="https://ably.com/signup/?utm_source=blog&utm_medium=oc&utm_campaign=tryit" class="vt-blog-editorial-tryit-cta">Start building <span aria-hidden="true">→</span></a><a href="https://ably.com/docs" class="vt-blog-editorial-tryit-link">Read the docs</a></div></div></aside><section class="vt-blog-editorial-newsletter" aria-labelledby="newsletter-heading"><div class="vt-blog-editorial-newsletter-text"><div class="vt-blog-editorial-newsletter-kicker"><span class="vt-blog-editorial-orange-rule" aria-hidden="true"></span><span>Subscribe</span></div><h3 id="newsletter-heading" class="vt-blog-editorial-newsletter-heading">New posts from the Ably team, monthly.</h3></div><form class="vt-blog-editorial-newsletter-form"><input type="email" name="EMAIL" required="" placeholder="you@company.com" class="vt-blog-editorial-newsletter-input" aria-label="Email address" value=""/><button type="submit" disabled="" class="vt-blog-editorial-newsletter-cta">Subscribe</button></form></section></article><aside class="vt-blog-editorial-rail" aria-label="Article tools"><nav aria-label="Table of contents"><div class="vt-blog-editorial-rail-label">On this page</div><ol class="vt-blog-editorial-toc-list"><li><a href="#the-definition-of-dependability" class="vt-blog-editorial-toc-link ">The definition of dependability</a></li><li><a href="#what-is-a-fault-tolerant-system" class="vt-blog-editorial-toc-link ">What is a fault tolerant system?</a></li><li><a href="#redundancy" class="vt-blog-editorial-toc-link ">Redundancy</a></li><li><a href="#fault-tolerant-system-design" class="vt-blog-editorial-toc-link ">Fault tolerant system design</a></li><li><a href="#stateless-services" class="vt-blog-editorial-toc-link ">Stateless services</a></li><li><a href="#stateful-services" class="vt-blog-editorial-toc-link ">Stateful services</a></li><li><a href="#architectural-approaches-to-achieve-reliability" class="vt-blog-editorial-toc-link ">Architectural approaches to achieve reliability</a></li><li><a href="#engineering-a-fault-tolerant-distributed-system" class="vt-blog-editorial-toc-link ">Engineering a fault tolerant distributed system</a></li><li><a href="#conclusion" class="vt-blog-editorial-toc-link ">Conclusion</a></li><li><a href="#latest-from-ably-engineering" class="vt-blog-editorial-toc-link ">Latest from Ably Engineering</a></li></ol></nav><div class="vt-blog-editorial-share" role="group" aria-labelledby="share-rail-label"><div id="share-rail-label" class="vt-blog-editorial-rail-label">Share</div><div class="vt-blog-editorial-share-list"><a href="https://twitter.com/intent/tweet?url=https%3A%2F%2Fably.com%2Fblog%2Fengineering-dependability-and-fault-tolerance-in-a-distributed-system&text=Engineering%20a%20fault%20tolerant%20distributed%20system" target="_blank" rel="noopener noreferrer" class="vt-blog-editorial-share-link" aria-label="Share on Twitter / X"><span class="vt-blog-editorial-share-glyph" aria-hidden="true"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="" style="width:0.875rem;height:0.875rem"><path fill="#03020D" d="M17.729 2h3.257l-7.116 8.133L22.24 21.2h-6.554l-5.134-6.712L4.679 21.2h-3.26l7.612-8.699L1 2h6.721l4.64 6.135zm-1.143 17.25h1.804L6.74 3.847H4.804z"></path></svg></span><span class="vt-blog-editorial-share-label">Twitter / X</span></a><a href="https://www.linkedin.com/sharing/share-offsite/?url=https%3A%2F%2Fably.com%2Fblog%2Fengineering-dependability-and-fault-tolerance-in-a-distributed-system" target="_blank" rel="noopener noreferrer" class="vt-blog-editorial-share-link" aria-label="Share on LinkedIn"><span class="vt-blog-editorial-share-glyph" aria-hidden="true"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="" style="width:0.875rem;height:0.875rem"><path fill="#1269BF" d="M24 1.765v20.47A1.764 1.764 0 0 1 22.235 24H1.765A1.765 1.765 0 0 1 0 22.235V1.765A1.765 1.765 0 0 1 1.765 0h20.47A1.765 1.765 0 0 1 24 1.765M7.059 9.176h-3.53v11.295h3.53zm.317-3.882a2.033 2.033 0 0 0-2.018-2.047h-.064a2.047 2.047 0 1 0 0 4.094 2.033 2.033 0 0 0 2.082-1.983zm13.095 8.315c0-3.395-2.16-4.715-4.306-4.715a4.02 4.02 0 0 0-3.572 1.821h-.099V9.176H9.176v11.295h3.53v-6.007a2.344 2.344 0 0 1 2.117-2.527h.135c1.122 0 1.955.705 1.955 2.484v6.05h3.53z"></path></svg></span><span class="vt-blog-editorial-share-label">LinkedIn</span></a><a href="https://news.ycombinator.com/submitlink?u=https%3A%2F%2Fably.com%2Fblog%2Fengineering-dependability-and-fault-tolerance-in-a-distributed-system&t=Engineering%20a%20fault%20tolerant%20distributed%20system" target="_blank" rel="noopener noreferrer" class="vt-blog-editorial-share-link" aria-label="Submit to Hacker News"><span class="vt-blog-editorial-share-glyph" aria-hidden="true"><span class="vt-blog-editorial-share-hn-mark">Y</span></span><span class="vt-blog-editorial-share-label">Hacker News</span></a><button type="button" class="vt-blog-editorial-share-link"><span class="vt-blog-editorial-share-glyph" aria-hidden="true"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" aria-hidden="true" data-slot="icon" class="" style="width:0.875rem;height:0.875rem"><path stroke-linecap="round" stroke-linejoin="round" d="M15.666 3.888A2.25 2.25 0 0 0 13.5 2.25h-3c-1.03 0-1.9.693-2.166 1.638m7.332 0c.055.194.084.4.084.612v0a.75.75 0 0 1-.75.75H9a.75.75 0 0 1-.75-.75v0c0-.212.03-.418.084-.612m7.332 0c.646.049 1.288.11 1.927.184 1.1.128 1.907 1.077 1.907 2.185V19.5a2.25 2.25 0 0 1-2.25 2.25H6.75A2.25 2.25 0 0 1 4.5 19.5V6.257c0-1.108.806-2.057 1.907-2.185a48.208 48.208 0 0 1 1.927-.184"></path></svg></span><span class="vt-blog-editorial-share-label" aria-live="polite">Copy link</span></button></div></div></aside></div><section class="vt-blog-editorial-continue" aria-labelledby="continue-reading-heading"><div class="vt-blog-editorial-continue-inner"><div class="vt-blog-editorial-continue-head"><h2 id="continue-reading-heading" class="vt-blog-editorial-continue-h3">Continue reading</h2><a href="/blog" class="vt-blog-editorial-continue-all">All posts <span aria-hidden="true">→</span></a></div><div class="vt-blog-editorial-continue-grid"><a href="/blog/how-to-connect-to-ably-directly-part-2" class="vt-blog-editorial-tile" data-testid="post-tile-how-to-connect-to-ably-directly-part-2"><div class="vt-blog-editorial-tile-image"><img src="https://ik.imagekit.io/ably/ghost/prod/2021/08/how-to-connect-to-ably-yourself@2x-1.png?tr=w-768,q-50" alt="How to connect to Ably directly (and why you probably shouldn't) – Part 2" width="768" class="w-full h-full object-cover" loading="lazy"/></div><div><div class="vt-blog-editorial-tile-tag"><span class="vt-blog-editorial-badge">Ably engineering</span></div><h3 class="vt-blog-editorial-tile-title">How to connect to Ably directly (and why you probably shouldn't) – Part 2</h3><div class="vt-blog-editorial-tile-meta"><span class="vt-blog-editorial-tile-meta-author">Owen Pearson</span><span class="vt-blog-editorial-meta-dot">·</span><span class="vt-blog-editorial-tile-meta-mono">Sep 7, 2021</span></div></div></a><a href="/blog/aws-vpc-peering-vs-transit-gateway-and-beyond" class="vt-blog-editorial-tile" data-testid="post-tile-aws-vpc-peering-vs-transit-gateway-and-beyond"><div class="vt-blog-editorial-tile-image"><img src="https://ik.imagekit.io/ably/ghost/prod/2022/09/designing-the-next-global-ably-network.png?tr=w-768,q-50" alt="VPC peering vs Transit Gateway and beyond: Key choices in AWS network design" width="768" class="w-full h-full object-cover" loading="lazy"/></div><div><div class="vt-blog-editorial-tile-tag"><span class="vt-blog-editorial-badge">Ably engineering</span></div><h3 class="vt-blog-editorial-tile-title">VPC peering vs Transit Gateway and beyond: Key choices in AWS network design</h3><div class="vt-blog-editorial-tile-meta"><span class="vt-blog-editorial-tile-meta-author">Huw McNamara</span><span class="vt-blog-editorial-meta-dot">·</span><span class="vt-blog-editorial-tile-meta-mono">Sep 13, 2022</span></div></div></a><a href="/blog/crdts-distributed-data-consistency-challenges" class="vt-blog-editorial-tile" data-testid="post-tile-crdts-distributed-data-consistency-challenges"><div class="vt-blog-editorial-tile-image"><img src="https://ik.imagekit.io/ably/ghost/prod/2021/10/blog-crdts-deep-dive.jpg?tr=w-768,q-50" alt="CRDTs solve distributed data consistency challenges" width="768" class="w-full h-full object-cover" loading="lazy"/></div><div><div class="vt-blog-editorial-tile-tag"><span class="vt-blog-editorial-badge">Ably engineering</span></div><h3 class="vt-blog-editorial-tile-title">CRDTs solve distributed data consistency challenges</h3><div class="vt-blog-editorial-tile-meta"><span class="vt-blog-editorial-tile-meta-author">Jo Stichbury</span><span class="vt-blog-editorial-meta-dot">·</span><span class="vt-blog-editorial-tile-meta-mono">Oct 28, 2021</span></div></div></a></div></div></section></main><footer class="w-full bg-neutral-100 dark:bg-neutral-1200 border-t border-neutral-300 dark:border-neutral-1000" data-id="footer"><div class="max-w-screen-xl mx-auto ui-grid-px pt-10 sm:pt-12 md:pt-16 pb-10"><div class="flex flex-col sm:flex-row gap-x-6 gap-y-12 mb-16 justify-between"><div class="flex-1 flex flex-col gap-6"><a href="/" class="items-center gap-2 justify-center focus-base rounded w-[6.375rem] flex dark:hidden"><img src="https://voltaire.ably.com/static/ably-logo-200721285a51085f43e8a849a85667bc.svg" width="96px" alt="Ably logo"/></a><a href="/" class="items-center gap-2 justify-center focus-base rounded w-[6.375rem] hidden dark:flex"><img src="https://voltaire.ably.com/static/ably-logo-white-749ef5fc6551d0e3b4940c2e3e42463f.svg" width="96px" alt="Ably logo"/></a><a href="https://status.ably.com" class="inline-flex group/status items-center gap-2" target="_blank" rel="noreferrer"><span class="inline-flex h-2 aspect-square rounded-full bg-neutral-500 animate-pulse"></span></a><div class="flex gap-x-6"><a href="https://x.com/ablyrealtime" target="_blank" rel="noreferrer noopener" aria-label="Visit Ably on x" class="w-5 h-5 flex group/social-icon"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="text-neutral-1000 dark:text-neutral-300 group-hover/social-icon:hidden" style="width:20px;height:20px"><path fill="currentColor" d="M17.729 2h3.257l-7.116 8.133L22.24 21.2h-6.554l-5.134-6.712L4.679 21.2h-3.26l7.612-8.699L1 2h6.721l4.64 6.135zm-1.143 17.25h1.804L6.74 3.847H4.804z"></path></svg><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="hidden group-hover/social-icon:flex" style="width:20px;height:20px"><path fill="#03020D" d="M17.729 2h3.257l-7.116 8.133L22.24 21.2h-6.554l-5.134-6.712L4.679 21.2h-3.26l7.612-8.699L1 2h6.721l4.64 6.135zm-1.143 17.25h1.804L6.74 3.847H4.804z"></path></svg></a><a href="https://www.linkedin.com/company/ably-realtime" target="_blank" rel="noreferrer noopener" aria-label="Visit Ably on linkedin" class="w-5 h-5 flex group/social-icon"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="text-neutral-1000 dark:text-neutral-300 group-hover/social-icon:hidden" style="width:20px;height:20px"><path fill="currentColor" d="M24 1.765v20.47A1.764 1.764 0 0 1 22.235 24H1.765A1.765 1.765 0 0 1 0 22.235V1.765A1.765 1.765 0 0 1 1.765 0h20.47A1.765 1.765 0 0 1 24 1.765M7.059 9.176h-3.53v11.295h3.53zm.317-3.882a2.033 2.033 0 0 0-2.018-2.047h-.064a2.047 2.047 0 1 0 0 4.094 2.033 2.033 0 0 0 2.082-1.983zm13.095 8.315c0-3.395-2.16-4.715-4.306-4.715a4.02 4.02 0 0 0-3.572 1.821h-.099V9.176H9.176v11.295h3.53v-6.007a2.344 2.344 0 0 1 2.117-2.527h.135c1.122 0 1.955.705 1.955 2.484v6.05h3.53z"></path></svg><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="hidden group-hover/social-icon:flex" style="width:20px;height:20px"><path fill="#1269BF" d="M24 1.765v20.47A1.764 1.764 0 0 1 22.235 24H1.765A1.765 1.765 0 0 1 0 22.235V1.765A1.765 1.765 0 0 1 1.765 0h20.47A1.765 1.765 0 0 1 24 1.765M7.059 9.176h-3.53v11.295h3.53zm.317-3.882a2.033 2.033 0 0 0-2.018-2.047h-.064a2.047 2.047 0 1 0 0 4.094 2.033 2.033 0 0 0 2.082-1.983zm13.095 8.315c0-3.395-2.16-4.715-4.306-4.715a4.02 4.02 0 0 0-3.572 1.821h-.099V9.176H9.176v11.295h3.53v-6.007a2.344 2.344 0 0 1 2.117-2.527h.135c1.122 0 1.955.705 1.955 2.484v6.05h3.53z"></path></svg></a><a href="https://github.com/ably/" target="_blank" rel="noreferrer noopener" aria-label="Visit Ably on github" class="w-5 h-5 flex group/social-icon"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="text-neutral-1000 dark:text-neutral-300 group-hover/social-icon:hidden" style="width:20px;height:20px"><path fill="currentColor" fill-rule="evenodd" d="M12 0C5.4 0 0 5.42 0 12.043c0 5.32 3.4 9.836 8.2 11.441.6.1.8-.3.8-.602v-2.007c-3.3.702-4-1.606-4-1.606-.5-1.405-1.3-1.806-1.3-1.806-1.1-.703.1-.703.1-.703 1.2.1 1.8 1.204 1.8 1.204 1.1 1.807 2.8 1.305 3.5 1.004.1-.803.4-1.305.8-1.606-2.8-.3-5.6-1.304-5.6-5.92 0-1.306.5-2.41 1.2-3.212 0-.402-.5-1.606.2-3.212 0 0 1-.301 3.3 1.204 1-.3 2-.401 3-.401s2 .1 3 .401c2.3-1.505 3.3-1.204 3.3-1.204.7 1.706.2 2.91.1 3.212.8.802 1.2 1.906 1.2 3.211 0 4.617-2.8 5.62-5.5 5.921.4.402.8 1.104.8 2.208v3.312c0 .301.2.703.8.602 4.9-1.606 8.3-6.122 8.3-11.44C24 5.418 18.6 0 12 0" clip-rule="evenodd"></path></svg><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="hidden group-hover/social-icon:flex" style="width:20px;height:20px"><path fill="#191717" fill-rule="evenodd" d="M12 0C5.4 0 0 5.42 0 12.043c0 5.32 3.4 9.836 8.2 11.441.6.1.8-.3.8-.602v-2.007c-3.3.702-4-1.606-4-1.606-.5-1.405-1.3-1.806-1.3-1.806-1.1-.703.1-.703.1-.703 1.2.1 1.8 1.204 1.8 1.204 1.1 1.807 2.8 1.305 3.5 1.004.1-.803.4-1.305.8-1.606-2.8-.3-5.6-1.304-5.6-5.92 0-1.306.5-2.41 1.2-3.212 0-.402-.5-1.606.2-3.212 0 0 1-.301 3.3 1.204 1-.3 2-.401 3-.401s2 .1 3 .401c2.3-1.505 3.3-1.204 3.3-1.204.7 1.706.2 2.91.1 3.212.8.802 1.2 1.906 1.2 3.211 0 4.617-2.8 5.62-5.5 5.921.4.402.8 1.104.8 2.208v3.312c0 .301.2.703.8.602 4.9-1.606 8.3-6.122 8.3-11.44C24 5.418 18.6 0 12 0" clip-rule="evenodd"></path></svg></a><a href="https://discord.gg/g8yqePUVDn" target="_blank" rel="noreferrer noopener" aria-label="Visit Ably on discord" class="w-5 h-5 flex group/social-icon"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="text-neutral-1000 dark:text-neutral-300 group-hover/social-icon:hidden" style="width:20px;height:20px"><path fill="currentColor" d="M20.317 4.607a19.6 19.6 0 0 0-4.885-1.542.074.074 0 0 0-.079.038c-.21.382-.444.88-.608 1.271a18 18 0 0 0-5.487 0c-.163-.4-.406-.89-.617-1.271a.08.08 0 0 0-.079-.038c-1.714.3-3.354.827-4.885 1.542a.07.07 0 0 0-.032.028C.533 9.364-.32 13.976.099 18.532a.08.08 0 0 0 .031.057 19.8 19.8 0 0 0 5.993 3.082.08.08 0 0 0 .084-.028c.462-.642.874-1.318 1.226-2.03a.08.08 0 0 0-.041-.107A13 13 0 0 1 5.52 18.6a.08.08 0 0 1-.008-.13q.19-.145.372-.297a.07.07 0 0 1 .078-.01c3.927 1.824 8.18 1.824 12.061 0a.07.07 0 0 1 .079.01q.18.15.372.297a.08.08 0 0 1-.006.13q-.895.531-1.873.906a.08.08 0 0 0-.041.109c.36.71.772 1.386 1.225 2.028a.075.075 0 0 0 .084.029 19.7 19.7 0 0 0 6.002-3.082.08.08 0 0 0 .032-.056c.5-5.267-.838-9.842-3.549-13.897a.06.06 0 0 0-.031-.03M8.02 15.757c-1.182 0-2.157-1.104-2.157-2.46s.956-2.46 2.157-2.46c1.21 0 2.176 1.113 2.157 2.46 0 1.356-.956 2.46-2.157 2.46m7.975 0c-1.183 0-2.157-1.104-2.157-2.46s.955-2.46 2.157-2.46c1.21 0 2.176 1.113 2.157 2.46 0 1.356-.946 2.46-2.157 2.46"></path></svg><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="hidden group-hover/social-icon:flex" style="width:20px;height:20px"><path fill="#5B64EA" d="M20.317 4.607a19.6 19.6 0 0 0-4.885-1.542.074.074 0 0 0-.079.038c-.21.382-.444.88-.608 1.271a18 18 0 0 0-5.487 0c-.163-.4-.406-.89-.617-1.271a.08.08 0 0 0-.079-.038c-1.714.3-3.354.827-4.885 1.542a.07.07 0 0 0-.032.028C.533 9.364-.32 13.976.099 18.532a.08.08 0 0 0 .031.057 19.8 19.8 0 0 0 5.993 3.082.08.08 0 0 0 .084-.028c.462-.642.874-1.318 1.226-2.03a.08.08 0 0 0-.041-.107A13 13 0 0 1 5.52 18.6a.08.08 0 0 1-.008-.13q.19-.145.372-.297a.07.07 0 0 1 .078-.01c3.927 1.824 8.18 1.824 12.061 0a.07.07 0 0 1 .079.01q.18.15.372.297a.08.08 0 0 1-.006.13q-.895.531-1.873.906a.08.08 0 0 0-.041.109c.36.71.772 1.386 1.225 2.028a.075.075 0 0 0 .084.029 19.7 19.7 0 0 0 6.002-3.082.08.08 0 0 0 .032-.056c.5-5.267-.838-9.842-3.549-13.897a.06.06 0 0 0-.031-.03M8.02 15.757c-1.182 0-2.157-1.104-2.157-2.46s.956-2.46 2.157-2.46c1.21 0 2.176 1.113 2.157 2.46 0 1.356-.956 2.46-2.157 2.46m7.975 0c-1.183 0-2.157-1.104-2.157-2.46s.955-2.46 2.157-2.46c1.21 0 2.176 1.113 2.157 2.46 0 1.356-.946 2.46-2.157 2.46"></path></svg></a><a href="https://www.youtube.com/c/AblyRealtime" target="_blank" rel="noreferrer noopener" aria-label="Visit Ably on youtube" class="w-5 h-5 flex group/social-icon"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="text-neutral-1000 dark:text-neutral-300 group-hover/social-icon:hidden" style="width:20px;height:20px"><path fill="currentColor" d="M23.499 6.631a3.01 3.01 0 0 0-2.122-2.128C19.505 4 12 4 12 4s-7.505 0-9.377.503A3.01 3.01 0 0 0 .502 6.631C0 8.51 0 12.425 0 12.425s0 3.917.502 5.795a3.01 3.01 0 0 0 2.121 2.128c1.872.503 9.377.503 9.377.503s7.505 0 9.377-.503a3.01 3.01 0 0 0 2.122-2.128C24 16.342 24 12.426 24 12.426s0-3.917-.502-5.795"></path><path fill="#fff" d="m9.545 15.982 6.273-3.556-6.273-3.557z"></path></svg><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="hidden group-hover/social-icon:flex" style="width:20px;height:20px"><path fill="#ED1D24" d="M23.499 6.131a3.01 3.01 0 0 0-2.122-2.128C19.505 3.5 12 3.5 12 3.5s-7.505 0-9.377.503A3.01 3.01 0 0 0 .502 6.131C0 8.01 0 11.925 0 11.925s0 3.917.502 5.795a3.01 3.01 0 0 0 2.121 2.128c1.872.503 9.377.503 9.377.503s7.505 0 9.377-.503a3.01 3.01 0 0 0 2.122-2.128C24 15.842 24 11.926 24 11.926s0-3.917-.502-5.795"></path><path fill="#fff" d="m9.545 15.482 6.273-3.556-6.273-3.557z"></path></svg></a></div><div class="flex flex-wrap gap-2 mt-4 max-w-full"><img src="https://voltaire.ably.com/static/g2-best-meets-requirements-spring-2025-7a9232ad72544eec2aa2918866eb3f7e.svg" alt="G2 Best Meets Requirements Spring 2025" width="55" height="63"/><img src="https://voltaire.ably.com/static/g2-best-support-spring-2025-25e3fb42507694fe41661b0ae587227e.svg" alt="G2 Best Support Spring 2025" width="55" height="63"/><img src="https://voltaire.ably.com/static/g2-users-most-likely-to-recommend-spring-2025-ada1333537cf17600c6f72e11b673838.svg" alt="G2 Users Most Likely to Recommend Spring 2025" width="55" height="63"/><img src="https://voltaire.ably.com/static/g2-easiest-to-use-spring-2025-001641898e826e06845da9c5a7fdca0d.svg" alt="G2 Easiest to Use Spring 2025" width="55" height="63"/></div></div><div class="flex-1 md:flex-[2] flex flex-row flex-wrap gap-x-6 gap-y-12"><div class="flex-1 basis-1/3 md:basis-1"><h3 class="ui-text-overline2 text-neutral-700 dark:text-neutral-600 mb-4">Products</h3><ul class="flex flex-col gap-y-3"><li class="flex gap-x-2"><a href="/pubsub" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably Pub/Sub">Ably Pub/Sub</a></li><li class="flex gap-x-2"><a href="/chat" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably Chat">Ably Chat</a></li><li class="flex gap-x-2"><a href="/ai-transport" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably AI Transport">Ably AI Transport</a></li><li class="flex gap-x-2"><a href="/liveobjects" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably LiveObjects">Ably LiveObjects</a></li><li class="flex gap-x-2"><a href="/spaces" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably Spaces">Ably Spaces</a></li><li class="flex gap-x-2"><a href="/livesync" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably LiveSync">Ably LiveSync</a></li><li class="flex gap-x-2"><a href="/compare" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Compare our tech">Compare our tech</a></li></ul></div><div class="flex-1 basis-1/3 md:basis-1"><h3 class="ui-text-overline2 text-neutral-700 dark:text-neutral-600 mb-4">Platform</h3><ul class="flex flex-col gap-y-3"><li class="flex gap-x-2"><a href="/platform" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Architecture">Architecture</a></li><li class="flex gap-x-2"><a href="/docs/platform/integrations" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Integrations">Integrations</a></li><li class="flex gap-x-2"><a href="/docs/sdks" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit SDKs">SDKs</a></li><li class="flex gap-x-2"><a href="https://changelog.ably.com/" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Changelog">Changelog</a></li><li class="flex gap-x-2"><a href="/security-and-compliance" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Security & Compliance">Security & Compliance</a></li></ul></div><div class="flex-1 basis-1/3 md:basis-1"><h3 class="ui-text-overline2 text-neutral-700 dark:text-neutral-600 mb-4">Get started</h3><ul class="flex flex-col gap-y-3"><li class="flex gap-x-2"><a href="/docs" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Documentation">Documentation</a></li><li class="flex gap-x-2"><a href="/examples" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Examples">Examples</a></li><li class="flex gap-x-2"><a href="/pricing" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Pricing">Pricing</a></li><li class="flex gap-x-2"><a href="/topics" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Realtime A-Z">Realtime A-Z</a></li><li class="flex gap-x-2"><a href="/support" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Support">Support</a></li></ul></div><div class="flex-1 basis-1/3 md:basis-1"><h3 class="ui-text-overline2 text-neutral-700 dark:text-neutral-600 mb-4">Company</h3><ul class="flex flex-col gap-y-3"><li class="flex gap-x-2"><a href="/about" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit About Ably">About Ably</a></li><li class="flex gap-x-2"><a href="/blog" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Blog">Blog</a></li><li class="flex gap-x-2"><a href="/careers" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Careers">Careers</a><div class="inline-flex bg-neutral-100 dark:bg-neutral-1200 rounded-2xl gap-1 items-center focus-base transition-colors select-none px-2 py-0 text-[10px] leading-tight text-neutral-900 dark:text-neutral-400 ui-text-p4 font-[10px]"><span class="whitespace-nowrap tracking-[0.04em] leading-[18px]">WE’RE HIRING</span></div></li><li class="flex gap-x-2"><a href="/contact" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Contact us">Contact us</a></li></ul></div></div></div><div class="pt-6 border-t border-neutral-300 dark:border-neutral-1000"><div class="flex gap-6"><a href="/data-protection" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Data protection">Data protection</a><a href="/privacy" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Privacy">Privacy</a><a href="/legals" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Legals">Legals</a><a href="/privacy" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Cookies">Cookies</a></div></div></div></footer><div class="fixed transition-[bottom] left-0 p-8 z-50 bottom-[-100px]" data-testid="layout-scroll-to-top"><div class="relative"><a href="#" class="bg-white w-12 h-12 cursor-pointer overflow-hidden rounded border-2 border-mid-grey hover:border-active-orange transition-all duration-[400ms] block"><div class="w-full h-full transition-all duration-[400ms] md:hover:translate-y-[-100%]"><div class="w-full h-full flex flex-row items-center justify-center"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" aria-hidden="true" data-slot="icon" class="text-cool-black" style="width:1.5rem;height:1.5rem"><path stroke-linecap="round" stroke-linejoin="round" d="M8.25 6.75 12 3m0 0 3.75 3.75M12 3v18"></path></svg></div><div class="w-full h-full flex flex-row items-center justify-center"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" aria-hidden="true" data-slot="icon" class="text-cool-black" style="width:1.5rem;height:1.5rem"><path stroke-linecap="round" stroke-linejoin="round" d="M8.25 6.75 12 3m0 0 3.75 3.75M12 3v18"></path></svg></div></div></a></div></div></div><div id="gatsby-announcer" style="position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0" aria-live="assertive" aria-atomic="true"></div></div><script>
|
||
(function() {
|
||
// Apply initial theme to dark navigation elements
|
||
var darkNav = document.getElementById('meganav-theme-dark');
|
||
if (darkNav) {
|
||
var header = darkNav.querySelector('header');
|
||
if (header && !header.classList.contains('ui-theme-dark')) {
|
||
header.classList.add('ui-theme-dark');
|
||
}
|
||
}
|
||
|
||
var darkMain = document.getElementById('main-theme-dark');
|
||
if (darkMain && !darkMain.classList.contains('ui-theme-dark')) {
|
||
darkMain.classList.add('ui-theme-dark');
|
||
}
|
||
})();
|
||
</script><script id="gatsby-script-loader">/*<![CDATA[*/window.pagePath="/blog/engineering-dependability-and-fault-tolerance-in-a-distributed-system";/*]]>*/</script><!-- slice-start id="_gatsby-scripts-1" -->
|
||
<script
|
||
id="gatsby-chunk-mapping"
|
||
>
|
||
window.___chunkMapping="{\"app\":[\"/app-afd9e708c8f5048ea0db.js\"],\"component---src-pages-404-js\":[\"/component---src-pages-404-js-b6e8f7edd8dea77e0c57.js\"],\"component---src-pages-ai-transport-tsx\":[\"/component---src-pages-ai-transport-tsx-4026b54583210f8a2053.js\"],\"component---src-pages-chat-tsx\":[\"/component---src-pages-chat-tsx-b2bdc3f08ae11253c0e3.js\"],\"component---src-pages-contact-tsx\":[\"/component---src-pages-contact-tsx-dc89e20c2190ed888225.js\"],\"component---src-pages-customer-service-chat-tsx\":[\"/component---src-pages-customer-service-chat-tsx-48ef3c126214086f90be.js\"],\"component---src-pages-cx-tech-tsx\":[\"/component---src-pages-cx-tech-tsx-36c4ce5a367e2a78a600.js\"],\"component---src-pages-ed-tech-tsx\":[\"/component---src-pages-ed-tech-tsx-d37da5b8d8490a573dbd.js\"],\"component---src-pages-fan-engagement-tsx\":[\"/component---src-pages-fan-engagement-tsx-8a1c337ce51c17c42fb7.js\"],\"component---src-pages-fin-tech-tsx\":[\"/component---src-pages-fin-tech-tsx-569a7d8f0503b63d5dc2.js\"],\"component---src-pages-four-pillars-of-dependability-tsx\":[\"/component---src-pages-four-pillars-of-dependability-tsx-d7a6e0386690e0cc591b.js\"],\"component---src-pages-global-round-trip-latencies-tsx\":[\"/component---src-pages-global-round-trip-latencies-tsx-b68878661cea17e95b93.js\"],\"component---src-pages-group-chat-tsx\":[\"/component---src-pages-group-chat-tsx-ebd327d243327ee9f8c8.js\"],\"component---src-pages-health-tech-tsx\":[\"/component---src-pages-health-tech-tsx-35d7fa3532326d5e5c7f.js\"],\"component---src-pages-hipaa-compliant-chat-tsx\":[\"/component---src-pages-hipaa-compliant-chat-tsx-acbc2476763adc3c3443.js\"],\"component---src-pages-index-tsx\":[\"/component---src-pages-index-tsx-322545dbcb61532adb0d.js\"],\"component---src-pages-live-streaming-chat-tsx\":[\"/component---src-pages-live-streaming-chat-tsx-e5e76e4db0722d9207ca.js\"],\"component---src-pages-liveobjects-tsx\":[\"/component---src-pages-liveobjects-tsx-28aa815395acdc2ce82e.js\"],\"component---src-pages-livesync-tsx\":[\"/component---src-pages-livesync-tsx-70885b210a264e5141e5.js\"],\"component---src-pages-periodic-table-of-realtime-js\":[\"/component---src-pages-periodic-table-of-realtime-js-d61b7b51a3fc51b8db26.js\"],\"component---src-pages-pricing-contact-tsx\":[\"/component---src-pages-pricing-contact-tsx-6d358f3f151478534903.js\"],\"component---src-pages-pricing-tsx\":[\"/component---src-pages-pricing-tsx-0354dbd8801f5243ece3.js\"],\"component---src-pages-pubsub-tsx\":[\"/component---src-pages-pubsub-tsx-464d839be2211c173921.js\"],\"component---src-pages-react-tsx\":[\"/component---src-pages-react-tsx-87358a2709374aacf56e.js\"],\"component---src-pages-reference-guide-chat-tsx\":[\"/component---src-pages-reference-guide-chat-tsx-8e3b14932c19bcc1e464.js\"],\"component---src-pages-reference-guide-multiplayer-tsx\":[\"/component---src-pages-reference-guide-multiplayer-tsx-9c53536d0cc84d1f7db8.js\"],\"component---src-pages-spaces-tsx\":[\"/component---src-pages-spaces-tsx-c46f420684aae4fb56f5.js\"],\"component---src-pages-websocket-service-tsx\":[\"/component---src-pages-websocket-service-tsx-29bc4c2263ebd30de9ce.js\"],\"component---src-templates-ably-vs-competitors-tsx\":[\"/component---src-templates-ably-vs-competitors-tsx-052a2027d366b1271875.js\"],\"component---src-templates-alternatives-index-tsx\":[\"/component---src-templates-alternatives-index-tsx-935d9b574657fb06b2f7.js\"],\"component---src-templates-alternatives-tsx\":[\"/component---src-templates-alternatives-tsx-5fc8ac958760659bdf83.js\"],\"component---src-templates-business-user-business-user-landing-tsx\":[\"/component---src-templates-business-user-business-user-landing-tsx-a017f6d2e2f718a2eafb.js\"],\"component---src-templates-compare-ably-js\":[\"/component---src-templates-compare-ably-js-6c0f08a75c6b11b3f317.js\"],\"component---src-templates-compare-index-js\":[\"/component---src-templates-compare-index-js-88ec96d0ed1e663d23ca.js\"],\"component---src-templates-compare-v-1-js\":[\"/component---src-templates-compare-v-1-js-d9b82d0110acbb502264.js\"],\"component---src-templates-compare-v-2-tsx\":[\"/component---src-templates-compare-v-2-tsx-4e1cd6756c146c67224d.js\"],\"component---src-templates-customer-story-tsx\":[\"/component---src-templates-customer-story-tsx-c07c97ff42a331eb75ae.js\"],\"component---src-templates-downloadable-gated-resource-downloadable-gated-resource-tsx\":[\"/component---src-templates-downloadable-gated-resource-downloadable-gated-resource-tsx-91158f3c7c6f4d153cd2.js\"],\"component---src-templates-downloadable-gated-resource-non-gated-resources-tsx\":[\"/component---src-templates-downloadable-gated-resource-non-gated-resources-tsx-7f08268bbae47759ffba.js\"],\"component---src-templates-downloadable-gated-resource-thank-you-page-js\":[\"/component---src-templates-downloadable-gated-resource-thank-you-page-js-0d99d8fac0c6445d301a.js\"],\"component---src-templates-flexible-tsx\":[\"/component---src-templates-flexible-tsx-088d00c89eb5a0ed6bfe.js\"],\"component---src-templates-flexible-v-2-tsx\":[\"/component---src-templates-flexible-v-2-tsx-1a6ff88fd8c866a1c57a.js\"],\"component---src-templates-generic-index-tsx\":[\"/component---src-templates-generic-index-tsx-89128b31a07f898b23dc.js\"],\"component---src-templates-ghost-pages-blog-list-blog-author-category-tsx\":[\"/component---src-templates-ghost-pages-blog-list-blog-author-category-tsx-820582c83ba43ebd3312.js\"],\"component---src-templates-ghost-pages-blog-list-blog-author-tsx\":[\"/component---src-templates-ghost-pages-blog-list-blog-author-tsx-eb3d8eb8dd047249efec.js\"],\"component---src-templates-ghost-pages-blog-list-blog-category-tsx\":[\"/component---src-templates-ghost-pages-blog-list-blog-category-tsx-45a6de80e8d75cb9c029.js\"],\"component---src-templates-ghost-pages-blog-list-blog-index-tsx\":[\"/component---src-templates-ghost-pages-blog-list-blog-index-tsx-0c5b2b87757efc90b336.js\"],\"component---src-templates-ghost-pages-blog-post-tsx\":[\"/component---src-templates-ghost-pages-blog-post-tsx-18ffed0af8673bb36129.js\"],\"component---src-templates-migrate-from-js\":[\"/component---src-templates-migrate-from-js-c3c1db295b667b6a61d9.js\"],\"component---src-templates-page-index-page-index-tsx\":[\"/component---src-templates-page-index-page-index-tsx-98b69b9679ce876767c4.js\"],\"component---src-templates-solution-tsx\":[\"/component---src-templates-solution-tsx-d8b78e013cb19083a3c9.js\"],\"component---src-templates-topic-tsx\":[\"/component---src-templates-topic-tsx-f902ddfff741dba97b74.js\"],\"component---src-templates-topics-category-tsx\":[\"/component---src-templates-topics-category-tsx-1433f8d904055ca038df.js\"],\"component---src-templates-topics-tsx\":[\"/component---src-templates-topics-tsx-436fa51625e6743f0934.js\"]}";
|
||
</script>
|
||
<script>window.___webpackCompilationHash="bd450251d228ea1e62eb";</script><script src="https://voltaire.ably.com/webpack-runtime-835cc4d6ec8a5974e673.js" async></script><script src="https://voltaire.ably.com/framework-5e45d846ab16cc6f99a4.js" async></script><script src="https://voltaire.ably.com/550fdbb7-7d2088aeb95a473eadaf.js" async></script><script src="https://voltaire.ably.com/6c2eb795-43135b94c65e4e3f8c79.js" async></script><script src="https://voltaire.ably.com/67a3bf3c-098d4132adafb49f830f.js" async></script><script src="https://voltaire.ably.com/ce3a710f-71c60be4cf8d98c10377.js" async></script><script src="https://voltaire.ably.com/app-afd9e708c8f5048ea0db.js" async></script><!-- slice-end id="_gatsby-scripts-1" --></body></html> |