41 lines
61 KiB
HTML
41 lines
61 KiB
HTML
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta http-equiv="x-ua-compatible" content="ie=edge"/><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"/><meta name="generator" content="Gatsby 5.16.0"/><meta data-react-helmet="true" name="description" content="After the attack on Dyn we got thinking about how a HTTP request to our API is covered by twenty separate strategies to mitigate single points of failure."/><meta data-react-helmet="true" property="og:site_name" content="Ably Realtime"/><meta data-react-helmet="true" property="og:type" content="website"/><meta data-react-helmet="true" property="og:url" content="https://ably.com/blog/the-18-ghosts-in-your-infrastructure-stack"/><meta data-react-helmet="true" property="og:title" content="The 18 ghosts in your infrastructure stack that can cause failure (and how to avoid them)"/><meta data-react-helmet="true" property="og:description" content="After the attack on Dyn we got thinking about how a HTTP request to our API is covered by twenty separate strategies to mitigate single points of failure."/><meta data-react-helmet="true" property="og:image" content="https://files.ably.io/ghost/prod/2019/05/ghosts.jpg"/><meta data-react-helmet="true" name="twitter:card" content="summary_large_image"/><meta data-react-helmet="true" name="twitter:site" content="@ablyrealtime"/><link href="https://voltaire.ably.com/styles.401201978d276adf9700.css" rel="stylesheet" type="text/css" media="screen"/><style>.gatsby-image-wrapper{position:relative;overflow:hidden}.gatsby-image-wrapper picture.object-fit-polyfill{position:static!important}.gatsby-image-wrapper img{bottom:0;height:100%;left:0;margin:0;max-width:none;padding:0;position:absolute;right:0;top:0;width:100%;object-fit:cover}.gatsby-image-wrapper [data-main-image]{opacity:0;transform:translateZ(0);transition:opacity .25s linear;will-change:opacity}.gatsby-image-wrapper-constrained{display:inline-block;vertical-align:top}</style><noscript><style>.gatsby-image-wrapper noscript [data-main-image]{opacity:1!important}.gatsby-image-wrapper [data-placeholder-image]{opacity:0!important}</style></noscript><script type="module">const e="undefined"!=typeof HTMLImageElement&&"loading"in HTMLImageElement.prototype;e&&document.body.addEventListener("load",(function(e){const t=e.target;if(void 0===t.dataset.mainImage)return;if(void 0===t.dataset.gatsbyImageSsr)return;let a=null,n=t;for(;null===a&&n;)void 0!==n.parentNode.dataset.gatsbyImageWrapper&&(a=n.parentNode),n=n.parentNode;const o=a.querySelector("[data-placeholder-image]"),r=new Image;r.src=t.currentSrc,r.decode().catch((()=>{})).then((()=>{t.style.opacity=1,o&&(o.style.opacity=0,o.style.transition="opacity 500ms linear")}))}),!0);</script><style data-styled="" data-styled-version="6.5.0">.gLNiCf{scroll-margin-top:16px;}/*!sc*/
|
||
data-styled.g13[id="layout__Main-sc-17rhhsn-0"]{content:"gLNiCf,"}/*!sc*/
|
||
.jTvGq{transition:transform 0.3s;position:sticky;top:64px;bottom:1rem;}/*!sc*/
|
||
data-styled.g22[id="sticky-sidebar-mobile__Sidebar-sc-gqtcvm-0"]{content:"jTvGq,"}/*!sc*/
|
||
.dYTnoj{position:fixed;top:0;left:0;right:0;bottom:0;background:rgba(0,0,0,0.5);z-index:1;transform:translateZ(0);pointer-events:none;visibility:hidden;opacity:0;transition:opacity 0.2s;height:100vh;}/*!sc*/
|
||
data-styled.g29[id="overlay__Overlay-sc-j0b6or-0"]{content:"dYTnoj,"}/*!sc*/
|
||
.ILNdO{max-height:calc(100vh - 64px);transition:transform 0.3s ease-in-out;transform:translateY(-100%);z-index:-1;}/*!sc*/
|
||
data-styled.g30[id="table-of-contents-mobile__Container-sc-1g3ebrb-0"]{content:"ILNdO,"}/*!sc*/
|
||
</style><link rel="alternate" type="application/rss+xml" title="The Ably Blog" href="https://voltaire.ably.com/blog/rss.xml"/><script>window.dataLayer = window.dataLayer || [];window.dataLayer.push({"platform":"gatsby","gaPageView":false}); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= 'https://www.googletagmanager.com/gtm.js?id='+i+dl+'';f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer', 'GTM-TZ37KKW');</script><script src="https://cdn-ukwest.onetrust.com/scripttemplates/otSDKStub.js" data-domain-script="b1eb1bbc-c8f1-47ad-87a5-4a2a6f01202c"></script><script>window.OptanonWrapper = function(){};</script><link data-react-helmet="true" rel="canonical" href="https://ably.com/blog/the-18-ghosts-in-your-infrastructure-stack"/><link data-react-helmet="true" rel="icon" href="/favicon.svg" type="image/svg+xml"/><link data-react-helmet="true" rel="icon" href="/favicon-96x96.png" sizes="96x96" type="image/png"/><link data-react-helmet="true" rel="icon" href="/favicon.ico" sizes="48x48 32x32 16x16"/><link data-react-helmet="true" rel="apple-touch-icon" href="/apple-touch-icon.png"/><link data-react-helmet="true" rel="manifest" href="/site.webmanifest"/><link data-react-helmet="true" rel="preconnect" href="https://fonts.googleapis.com"/><link data-react-helmet="true" rel="preconnect" href="https://fonts.gstatic.com" crossorigin=""/><link data-react-helmet="true" href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:ital,wght@0,100..800;1,100..800&family=Manrope:wght@200..800&family=Source+Code+Pro:wght@600&display=swap" rel="stylesheet"/><script data-react-helmet="true" type="application/ld+json">{"@context":"https://schema.org","@graph":[{"@type":"TechArticle","@id":"https://ably.com/blog/the-18-ghosts-in-your-infrastructure-stack#article","headline":"The 18 ghosts in your infrastructure stack that can cause failure (and how to avoid them)","abstract":"After the attack on Dyn we got thinking about how a HTTP request to our API is covered by twenty separate strategies to mitigate single points of failure.","mainEntityOfPage":"https://ably.com/blog/the-18-ghosts-in-your-infrastructure-stack","datePublished":"2016-11-23T00:00:00+00:00","dateModified":"2023-03-23T00:00:00+00:00","inLanguage":"en-US","image":{"@type":"ImageObject","url":"https://files.ably.io/ghost/prod/2019/05/ghosts.jpg"},"author":{"@type":"Person","name":"Matthew O'Riordan","url":"https://ably.com/blog/author/matthew","worksFor":{"@id":"https://ably.com#organization"}},"publisher":{"@id":"https://ably.com#organization"}},{"@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"item":{"@id":"https://ably.com","name":"Ably"}},{"@type":"ListItem","position":2,"item":{"@id":"https://ably.com/blog","name":"Blog"}},{"@type":"ListItem","position":3,"item":{"@id":"https://ably.com/blog/the-18-ghosts-in-your-infrastructure-stack","name":"The 18 ghosts in your infrastructure stack that can cause failure (and how to avoid them)"}}]},{"@type":"Organization","@id":"https://ably.com#organization","name":"Ably","url":"https://ably.com","logo":{"@type":"ImageObject","url":"https://ably.com/favicon-512x512.png"},"knowsAbout":["WebSockets","realtime messaging","pub/sub","AI Transport","durable sessions","token streaming","session continuity"],"sameAs":["https://www.wikidata.org/wiki/Q139764920","https://www.linkedin.com/company/ably-realtime/","https://twitter.com/ablyrealtime","https://github.com/ably","https://www.crunchbase.com/organization/ably-realtime","https://www.g2.com/products/ably"]},{"@type":"WebSite","@id":"https://ably.com#website","name":"Ably","url":"https://ably.com","publisher":{"@id":"https://ably.com#organization"}}]}</script><style id="sandpack">--sxs{--sxs:1 sp-k-eyOShd sp-k-iOHdLQ}@media{@keyframes sp-k-eyOShd{0%{opacity:0}100%{opacity:1}}@keyframes sp-k-iOHdLQ{0%{transform:rotateX(-25.5deg) rotateY(45deg)}100%{transform:rotateX(-25.5deg) rotateY(405deg)}}}--sxs{--sxs:2 sp-c-gMfcns sp-c-bxeRRt sp-c-jKPvnt sp-c-fWymNx sp-c-euXojQ sp-c-bpmgvy sp-c-PJLV}@media{.sp-c-gMfcns svg{margin:auto}.sp-c-bxeRRt{-webkit-appearance:none;appearance:none;outline:none;display:flex;align-items:center;font-size:inherit;font-family:inherit;background-color:transparent;transition:color var(--sp-transitions-default), background var(--sp-transitions-default);cursor:pointer;color:var(--sp-colors-clickable);border:0;text-decoration:none}.sp-c-bxeRRt:disabled{color:var(--sp-colors-disabled)}.sp-c-bxeRRt:hover:not(:disabled,[data-active='true']){color:var(--sp-colors-hover)}.sp-c-bxeRRt[data-active="true"]{color:var(--sp-colors-accent)}.sp-c-bxeRRt svg{min-width:var(--sp-space-4);width:var(--sp-space-4);height:var(--sp-space-4)}.sp-c-bxeRRt.sp-c-gMfcns{padding:var(--sp-space-1);height:var(--sp-space-7);display:flex}.sp-c-bxeRRt.sp-c-gMfcns.sp-c-bxeRRt:not(:has(span)){width:var(--sp-space-7)}.sp-c-bxeRRt.sp-c-gMfcns.sp-c-bxeRRt:has(svg + span){padding-right:var(--sp-space-3);padding-left:var(--sp-space-2);gap:var(--sp-space-1)}.sp-c-jKPvnt{padding:0 var(--sp-space-1) 0 var(--sp-space-1);border-radius:var(--sp-border-radius);margin-left:var(--sp-space-1);width:var(--sp-space-5);visibility:hidden;cursor:pointer;position:absolute;right:0px}.sp-c-jKPvnt svg{width:var(--sp-space-3);height:var(--sp-space-3);display:block;position:relative;top:1px}.sp-c-fWymNx{margin:0;display:block;font-family:var(--sp-font-mono);font-size:var(--sp-font-size);color:var(--sp-syntax-color-plain);line-height:var(--sp-font-lineHeight)}.sp-c-euXojQ{display:flex;flex-direction:column;width:100%;position:relative;background-color:var(--sp-colors-surface1);gap:1px}.sp-c-euXojQ:has(.sp-stack){background-color:var(--sp-colors-surface2)}.sp-c-bpmgvy{transform:translate(-4px, 9px) scale(0.13, 0.13)}.sp-c-bpmgvy *{position:absolute;width:96px;height:96px}}--sxs{--sxs:3 sp-c-PJLV-kCOVwI-status-pass sp-c-PJLV-kEzYsr-status-fail sp-c-PJLV-gHAhSA-status-skip sp-c-PJLV-jgnHyR-status-title sp-c-PJLV-iCgxLS-status-run sp-c-PJLV-bnDZSy-status-pass sp-c-PJLV-eYuGwt-status-fail}@media{.sp-c-PJLV-kCOVwI-status-pass{color:var(--test-pass)}.sp-c-PJLV-kEzYsr-status-fail{color:var(--test-fail)}.sp-c-PJLV-gHAhSA-status-skip{color:var(--test-skip)}.sp-c-PJLV-jgnHyR-status-title{color:var(--test-title)}.sp-c-PJLV-iCgxLS-status-run{background:var(--test-run);color:var(--sp-colors-surface1)}.sp-c-PJLV-bnDZSy-status-pass{background:var(--test-pass);color:var(--sp-colors-surface1)}.sp-c-PJLV-eYuGwt-status-fail{background:var(--test-fail);color:var(--sp-colors-surface1)}}</style></head><body><noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-TZ37KKW" height="0" width="0" style="display: none; visibility: hidden" aria-hidden="true"></iframe></noscript><div id="___gatsby"><div style="outline:none" tabindex="-1" id="gatsby-focus-wrapper"><div class="absolute top-0 left-0 right-0 w-full z-50" id="meganav" data-testid="meganav" style="height:64px"><header role="banner" style="height:64px;top:0" class="fixed left-0 top-0 w-full z-50 bg-neutral-000 dark:bg-neutral-1300 border-b border-neutral-300 dark:border-neutral-1000 transition-all duration-300 ease-in-out"><div class="flex items-center h-full max-w-screen-xl mx-auto ui-grid-px"><nav class="flex flex-1 h-full items-center"><a href="/" class="items-center gap-2 justify-center h-full focus-base rounded mr-4 lg:mr-8 flex dark:hidden"><img src="https://voltaire.ably.com/static/ably-logo-200721285a51085f43e8a849a85667bc.svg" width="96px" alt="Ably logo"/></a><a href="/" class="items-center gap-2 justify-center h-full focus-base rounded mr-4 lg:mr-8 hidden dark:flex"><img src="https://voltaire.ably.com/static/ably-logo-white-749ef5fc6551d0e3b4940c2e3e42463f.svg" width="96px" alt="Ably logo"/></a><div class="hidden md:flex flex-1 items-center h-full"><nav aria-label="Main" data-orientation="horizontal" dir="ltr" class="justify-left z-40 flex w-full"><div style="position:relative"><ul data-orientation="horizontal" class="flex list-none center" dir="ltr"><li><button id="radix-:R32p9:-trigger-radix-:R2r2p9:" data-state="closed" aria-expanded="false" aria-controls="radix-:R32p9:-content-radix-:R2r2p9:" class="group outline-none focus:outline-none select-none cursor-pointer relative rounded-md hover:bg-neutral-100 dark:hover:bg-neutral-1200 [&[data-state=open]]:bg-neutral-100 dark:[&[data-state=open]]:bg-neutral-1200 [&[data-state=open]]:text-neutral-1300 dark:[&[data-state=open]]:text-neutral-000 ui-text-label3 font-semibold text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 px-3 py-2 flex items-center justify-between" data-radix-collection-item="">Products</button></li><li><button id="radix-:R32p9:-trigger-radix-:R4r2p9:" data-state="closed" aria-expanded="false" aria-controls="radix-:R32p9:-content-radix-:R4r2p9:" class="group outline-none focus:outline-none select-none cursor-pointer relative rounded-md hover:bg-neutral-100 dark:hover:bg-neutral-1200 [&[data-state=open]]:bg-neutral-100 dark:[&[data-state=open]]:bg-neutral-1200 [&[data-state=open]]:text-neutral-1300 dark:[&[data-state=open]]:text-neutral-000 ui-text-label3 font-semibold text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 px-3 py-2 flex items-center justify-between" data-radix-collection-item="">Solutions</button></li><li><button id="radix-:R32p9:-trigger-radix-:R6r2p9:" data-state="closed" aria-expanded="false" aria-controls="radix-:R32p9:-content-radix-:R6r2p9:" class="group outline-none focus:outline-none select-none cursor-pointer relative rounded-md hover:bg-neutral-100 dark:hover:bg-neutral-1200 [&[data-state=open]]:bg-neutral-100 dark:[&[data-state=open]]:bg-neutral-1200 [&[data-state=open]]:text-neutral-1300 dark:[&[data-state=open]]:text-neutral-000 ui-text-label3 font-semibold text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 px-3 py-2 flex items-center justify-between" data-radix-collection-item="">Company</button></li><a href="/pricing" class="ui-text-label3 font-semibold text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 px-3 py-2 flex items-center justify-between" data-radix-collection-item="">Pricing</a><a href="/docs" class="ui-text-label3 font-semibold text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 px-3 py-2 flex items-center justify-between" data-radix-collection-item="">Docs</a></ul></div><div class="absolute top-full flex justify-left"></div></nav></div></nav><div class="flex md:hidden flex-1 items-center justify-end gap-6 h-full"><button class="cursor-pointer focus-base rounded flex items-center p-0" aria-expanded="false" aria-controls="mobile-menu" aria-label="Toggle menu"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" aria-hidden="true" data-slot="icon" class="text-neutral-1300 dark:text-neutral-000" style="width:1.5rem;height:1.5rem"><path stroke-linecap="round" stroke-linejoin="round" d="M3.75 6.75h16.5M3.75 12h16.5m-16.5 5.25h16.5"></path></svg></button></div><nav class="md:flex-1 md:items-center md:justify-end flex-col md:flex-row border-t-[1px] border-neutral-300 md:border-t-0 md:gap-4 pt-3 pb-4 md:py-0 hidden md:flex flex-1 items-center h-full md:gap-x-6"><a class="font-sans !text-label2 md:!text-label3 font-semibold py-4 text-neutral-1300 dark:text-neutral-000 md:text-neutral-1000 dark:md:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 active:text-neutral-1300 dark:active:text-neutral-000 transition-colors flex items-center gap-1.5 px-4 md:px-0 leading-none" href="/contact">Contact us</a><div class="flex gap-3 pt-3 md:py-0 px-4 md:px-0"><a class="ui-button-secondary flex-1 md:flex-none md:ui-button-secondary-xs hover:text-neutral-1300 dark:hover:text-neutral-000" role="button" tabindex="0" href="/login">Login</a><a class="ui-button-primary flex-1 md:flex-none md:ui-button-primary-xs hover:text-neutral-000 dark:hover:text-neutral-1300" role="button" tabindex="0" href="/sign-up">Start free</a></div></nav></div></header></div><div class="ui-flash" data-id="ui-flashes"></div><main class="layout__Main-sc-17rhhsn-0 gLNiCf vt-blog-editorial font-sans antialiased" id="main" data-testid="layout-main"><div class="vt-blog-editorial-reading-progress" aria-hidden="true"><div class="vt-blog-editorial-reading-progress-bar" style="width:0%"></div></div><aside data-sidebar-sticking="false" class="sticky-sidebar-mobile__Sidebar-sc-gqtcvm-0 jTvGq md:hidden w-full z-20 bg-transparent"><div class="relative"><div class="font-sans ui-grid-px bg-white border-mid-grey border-b flex items-center py-4 relative z-10"><div class="ui-text-overline2 text-dark-grey font-medium">On this page</div><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="currentColor" aria-hidden="true" data-slot="icon" class="ml-auto absolute right-0 mr-6 sm:mr-8" style="width:1.5rem;height:1.5rem"><path fill-rule="evenodd" d="M4.22 6.22a.75.75 0 0 1 1.06 0L8 8.94l2.72-2.72a.75.75 0 1 1 1.06 1.06l-3.25 3.25a.75.75 0 0 1-1.06 0L4.22 7.28a.75.75 0 0 1 0-1.06Z" clip-rule="evenodd"></path></svg></div><div class="table-of-contents-mobile__Container-sc-1g3ebrb-0 ILNdO border-b border-color-mid-grey absolute bg-white overflow-y-auto w-full"></div><div class="overlay__Overlay-sc-j0b6or-0 dYTnoj"></div></div></aside><header class="vt-blog-editorial-header"><div class="vt-blog-editorial-header-back-row"><a href="/blog" class="vt-blog-editorial-header-back"><span aria-hidden="true">←</span> Blog</a></div><span class="vt-blog-editorial-hero-eyebrow"><a href="/blog/ably-engineering" class="vt-blog-editorial-hero-eyebrow-cat no-underline" data-testid="hero-kicker">Ably engineering</a></span><h1 id="article-heading" class="vt-blog-editorial-h1">The 18 ghosts in your infrastructure stack that can cause failure (and how to avoid them)</h1><p class="vt-blog-editorial-dek">After the attack on Dyn we got thinking about how a HTTP request to our API is covered by twenty separate strategies to mitigate single points of failure.</p><div class="vt-blog-editorial-byline"><a href="/blog/author/matthew" class="vt-blog-editorial-byline-author"><img src="//www.gravatar.com/avatar/eed5c6e9e342b45e190e8de35cb9a132?s=250&d=mm&r=x" alt="Matthew O'Riordan" width="72" class="vt-blog-editorial-byline-avatar"/><span class="vt-blog-editorial-byline-name">Matthew O'Riordan</span><span class="vt-blog-editorial-byline-role">CEO and co-founder of Ably.</span></a><div class="vt-blog-editorial-byline-meta-col"><span class="vt-blog-editorial-byline-meta">Nov 23, 2016</span><span class="vt-blog-editorial-meta-dot">·</span><span class="vt-blog-editorial-byline-meta">4 min read</span></div></div><div class="vt-blog-editorial-header-rule" aria-hidden="true"></div></header><div class="vt-blog-editorial-grid"><article aria-labelledby="article-heading"><figure class="vt-blog-editorial-feature-figure"><img src="https://ik.imagekit.io/ably/ghost/prod/2019/05/ghosts.jpg?tr=w-1280,q-50" alt="The 18 ghosts in your infrastructure stack that can cause failure (and how to avoid them)" width="1280" class="w-full h-auto block"/></figure><div class="ghost-blog-post-body"><p>Following the <a href="https://blog.ably.io/routing-around-single-point-of-failure-dns-issues-7c20a8757903#.706361619https://blog.ably.io/routing-around-single-point-of-failure-dns-issues-7c20a8757903#.706361619" rel="noopener">recent attack on Dyn</a> that caused so much disruption because so many companies had a crucial single point of failure, I started thinking about how a trivial HTTP request to our API, that’s typically completed in under 100ms, is covered by almost twenty separate strategies we employ to address single points of failure. The diagram below is far from exhaustive, but I hope helps people to think about how every point of failure needs to be addressed if you want to offer <a href="https://support.ably.io/solution/articles/3000029531-100-uptime-guarantee" rel="noopener">guaranteed service continuity</a>.</p><figure class="kg-card kg-image-card kg-card-hascaption"><img src="https://ik.imagekit.io/ably/ghost/medium/max/1600/1*F783irEqCPArpT7JMc-SjA.png" class="kg-image" alt="" loading="lazy" srcset="https://ik.imagekit.io/ably/ghost/medium/max/1600/1*F783irEqCPArpT7JMc-SjA.png?tr=w-380,q-50 380w,
|
||
https://ik.imagekit.io/ably/ghost/medium/max/1600/1*F783irEqCPArpT7JMc-SjA.png?tr=w-760,q-50 760w,
|
||
https://ik.imagekit.io/ably/ghost/medium/max/1600/1*F783irEqCPArpT7JMc-SjA.png?tr=w-1520,q-50 1520w" sizes="
|
||
(min-width: 1520px) 1520px,
|
||
100vw
|
||
"><figcaption><a href="https://files.ably.io/blog/ably-pacman-in-the-infrastructure.png" data-href="https://files.ably.io/blog/ably-pacman-in-the-infrastructure.png" class="markup--anchor markup--figure-anchor" rel="noopener" target="_blank">Download this diagram</a></figcaption></figure><h3 id="notes">Notes</h3><ul><li>TLD domains do go down, see <a href="https://news.ycombinator.com/item?id=5838670" rel="noopener">when the .io domain was unavailable</a>. Our TLD fallback ensured the service remained up.</li><li>Big reliable DNS providers go down too, <a href="http://arstechnica.co.uk/security/2016/10/dos-attack-on-major-dns-provider-brings-internet-to-morning-crawl/" rel="noopener">Dyn went down and took a large chunk of Internet services with it</a>.</li><li>DDoS prevention, especially Layer 3/4 is generally handled better by an external edge barrier provider such as CloudFlare or Akamai. However, that’s not going to protect you against simple expensive attacks against your servers. You need to think about how you can very cheaply reject requests. For example, if you have to make a database lookup for every auth request and then have to do an expensive bit of cryptography to validate the auth details, it is trivial to generate significant load with a few expensive request that most DDoS providers will be unable to help you with.</li><li>Certificates can go down as well, don’t assume that’s not a single point of failure. <a href="https://www.globalsign.com/en/customer-revocation-error/" rel="noopener">GlobalSign recently mistakenly issued an incorrect revocation list</a> which resulted in a lot of their certificates being revoked and thus no longer usable. Think about fallback strategies.</li><li>Load balancers are a single point of failure if they don’t exist in multiple distinct datacentres and don’t autoscale to meet demand. If you are using Amazon, use ELB with multiple availability zones. Be aware however, whilst ELB is elastic and will scale, <a href="http://blog.mattheworiordan.com/post/19257368025/how-elastic-are-amazon-elastic-load-balancers" rel="noopener">it scales up within limits</a></li><li>If you deem it your responsibility to handle unexpected failures as opposed to your customers, then you need to handle in-flight HTTP requests and retry them at least once against another node that can satisfy the request. To do this, you’ll probably need your own routing layer that understands your business logic so that you don’t simply retry every failing request when the system is simply overloaded thus increasing the load further.</li><li>Use rate limiting to prevent abuse. You’ll need to figure out what you can rate limit against i.e. it could be an account, a user, or even an IP (although that’s rarely a good idea). Without it, one user, even unintentionally, can adversely affect other users.</li><li>If your application or data layer exists only in one region, you’ll struggle to offer guaranteed continuity. Network partitions occur, and entire regions go down (<a href="http://www.itnews.com.au/news/aws-sydney-outage-downs-big-name-web-companies-420478" rel="noopener">it’s happened with AWS</a>). If you’re only in one datacenter (availability zone), then you’re very susceptible to failures, try and spread your app and database across at least two datacenters with each being able to operate autonomously.</li></ul><p>We’ve been fortunate enough at <a href="https://www.ably.io" rel="noopener">Ably</a> to have decided early on that addressing all single points of failure is key to our platform offering and what we believe will set us apart. As a result, we’ve allocated considerable resource and time to solving these problems, mostly “just in case”. If you can make the time, and at least have a strategy to start removing single points of failure in your own platform, when the next big Internet outage occurs or you have unexpected failures in your system, I hope like us, you’ll be thankful you invested the time upfront.</p><figure class="kg-card kg-image-card"><img src="https://ik.imagekit.io/ably/ghost/prod/2019/10/image-14.png" class="kg-image" alt="" loading="lazy" srcset="https://ik.imagekit.io/ably/ghost/prod/2019/10/image-14.png?tr=w-380,q-50 380w,
|
||
https://ik.imagekit.io/ably/ghost/prod/2019/10/image-14.png?tr=w-760,q-50 760w,
|
||
https://ik.imagekit.io/ably/ghost/prod/2019/10/image-14.png?tr=w-1520,q-50 1520w" sizes="
|
||
(min-width: 1520px) 1520px,
|
||
100vw
|
||
"></figure></div><aside class="vt-blog-editorial-tryit" aria-label="Try Ably"><div class="vt-blog-editorial-tryit-rule"></div><div><div class="vt-blog-editorial-tryit-kicker">Try it</div><h3 class="vt-blog-editorial-tryit-heading">Build dependable realtime in minutes.</h3><p class="vt-blog-editorial-tryit-body">Ably runs the same patterns we write about — in production, at scale. Free tier includes 6M messages a month, no card required.</p><div class="vt-blog-editorial-tryit-actions"><a href="https://ably.com/signup/?utm_source=blog&utm_medium=oc&utm_campaign=tryit" class="vt-blog-editorial-tryit-cta">Start building <span aria-hidden="true">→</span></a><a href="https://ably.com/docs" class="vt-blog-editorial-tryit-link">Read the docs</a></div></div></aside><section class="vt-blog-editorial-newsletter" aria-labelledby="newsletter-heading"><div class="vt-blog-editorial-newsletter-text"><div class="vt-blog-editorial-newsletter-kicker"><span class="vt-blog-editorial-orange-rule" aria-hidden="true"></span><span>Subscribe</span></div><h3 id="newsletter-heading" class="vt-blog-editorial-newsletter-heading">New posts from the Ably team, monthly.</h3></div><form class="vt-blog-editorial-newsletter-form"><input type="email" name="EMAIL" required="" placeholder="you@company.com" class="vt-blog-editorial-newsletter-input" aria-label="Email address" value=""/><button type="submit" disabled="" class="vt-blog-editorial-newsletter-cta">Subscribe</button></form></section></article><aside class="vt-blog-editorial-rail" aria-label="Article tools"><nav aria-label="Table of contents"><div class="vt-blog-editorial-rail-label">On this page</div><ol class="vt-blog-editorial-toc-list"><li><a href="#notes" class="vt-blog-editorial-toc-link ">Notes</a></li></ol></nav><div class="vt-blog-editorial-share" role="group" aria-labelledby="share-rail-label"><div id="share-rail-label" class="vt-blog-editorial-rail-label">Share</div><div class="vt-blog-editorial-share-list"><a href="https://twitter.com/intent/tweet?url=https%3A%2F%2Fably.com%2Fblog%2Fthe-18-ghosts-in-your-infrastructure-stack&text=The%2018%20ghosts%20in%20your%20infrastructure%20stack%20that%20can%20cause%20failure%20(and%20how%20to%20avoid%C2%A0them)" target="_blank" rel="noopener noreferrer" class="vt-blog-editorial-share-link" aria-label="Share on Twitter / X"><span class="vt-blog-editorial-share-glyph" aria-hidden="true"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="" style="width:0.875rem;height:0.875rem"><path fill="#03020D" d="M17.729 2h3.257l-7.116 8.133L22.24 21.2h-6.554l-5.134-6.712L4.679 21.2h-3.26l7.612-8.699L1 2h6.721l4.64 6.135zm-1.143 17.25h1.804L6.74 3.847H4.804z"></path></svg></span><span class="vt-blog-editorial-share-label">Twitter / X</span></a><a href="https://www.linkedin.com/sharing/share-offsite/?url=https%3A%2F%2Fably.com%2Fblog%2Fthe-18-ghosts-in-your-infrastructure-stack" target="_blank" rel="noopener noreferrer" class="vt-blog-editorial-share-link" aria-label="Share on LinkedIn"><span class="vt-blog-editorial-share-glyph" aria-hidden="true"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="" style="width:0.875rem;height:0.875rem"><path fill="#1269BF" d="M24 1.765v20.47A1.764 1.764 0 0 1 22.235 24H1.765A1.765 1.765 0 0 1 0 22.235V1.765A1.765 1.765 0 0 1 1.765 0h20.47A1.765 1.765 0 0 1 24 1.765M7.059 9.176h-3.53v11.295h3.53zm.317-3.882a2.033 2.033 0 0 0-2.018-2.047h-.064a2.047 2.047 0 1 0 0 4.094 2.033 2.033 0 0 0 2.082-1.983zm13.095 8.315c0-3.395-2.16-4.715-4.306-4.715a4.02 4.02 0 0 0-3.572 1.821h-.099V9.176H9.176v11.295h3.53v-6.007a2.344 2.344 0 0 1 2.117-2.527h.135c1.122 0 1.955.705 1.955 2.484v6.05h3.53z"></path></svg></span><span class="vt-blog-editorial-share-label">LinkedIn</span></a><a href="https://news.ycombinator.com/submitlink?u=https%3A%2F%2Fably.com%2Fblog%2Fthe-18-ghosts-in-your-infrastructure-stack&t=The%2018%20ghosts%20in%20your%20infrastructure%20stack%20that%20can%20cause%20failure%20(and%20how%20to%20avoid%C2%A0them)" target="_blank" rel="noopener noreferrer" class="vt-blog-editorial-share-link" aria-label="Submit to Hacker News"><span class="vt-blog-editorial-share-glyph" aria-hidden="true"><span class="vt-blog-editorial-share-hn-mark">Y</span></span><span class="vt-blog-editorial-share-label">Hacker News</span></a><button type="button" class="vt-blog-editorial-share-link"><span class="vt-blog-editorial-share-glyph" aria-hidden="true"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" aria-hidden="true" data-slot="icon" class="" style="width:0.875rem;height:0.875rem"><path stroke-linecap="round" stroke-linejoin="round" d="M15.666 3.888A2.25 2.25 0 0 0 13.5 2.25h-3c-1.03 0-1.9.693-2.166 1.638m7.332 0c.055.194.084.4.084.612v0a.75.75 0 0 1-.75.75H9a.75.75 0 0 1-.75-.75v0c0-.212.03-.418.084-.612m7.332 0c.646.049 1.288.11 1.927.184 1.1.128 1.907 1.077 1.907 2.185V19.5a2.25 2.25 0 0 1-2.25 2.25H6.75A2.25 2.25 0 0 1 4.5 19.5V6.257c0-1.108.806-2.057 1.907-2.185a48.208 48.208 0 0 1 1.927-.184"></path></svg></span><span class="vt-blog-editorial-share-label" aria-live="polite">Copy link</span></button></div></div></aside></div><section class="vt-blog-editorial-continue" aria-labelledby="continue-reading-heading"><div class="vt-blog-editorial-continue-inner"><div class="vt-blog-editorial-continue-head"><h2 id="continue-reading-heading" class="vt-blog-editorial-continue-h3">Continue reading</h2><a href="/blog" class="vt-blog-editorial-continue-all">All posts <span aria-hidden="true">→</span></a></div><div class="vt-blog-editorial-continue-grid"><a href="/blog/how-to-connect-to-ably-directly-part-2" class="vt-blog-editorial-tile" data-testid="post-tile-how-to-connect-to-ably-directly-part-2"><div class="vt-blog-editorial-tile-image"><img src="https://ik.imagekit.io/ably/ghost/prod/2021/08/how-to-connect-to-ably-yourself@2x-1.png?tr=w-768,q-50" alt="How to connect to Ably directly (and why you probably shouldn't) – Part 2" width="768" class="w-full h-full object-cover" loading="lazy"/></div><div><div class="vt-blog-editorial-tile-tag"><span class="vt-blog-editorial-badge">Ably engineering</span></div><h3 class="vt-blog-editorial-tile-title">How to connect to Ably directly (and why you probably shouldn't) – Part 2</h3><div class="vt-blog-editorial-tile-meta"><span class="vt-blog-editorial-tile-meta-author">Owen Pearson</span><span class="vt-blog-editorial-meta-dot">·</span><span class="vt-blog-editorial-tile-meta-mono">Sep 7, 2021</span></div></div></a><a href="/blog/aws-vpc-peering-vs-transit-gateway-and-beyond" class="vt-blog-editorial-tile" data-testid="post-tile-aws-vpc-peering-vs-transit-gateway-and-beyond"><div class="vt-blog-editorial-tile-image"><img src="https://ik.imagekit.io/ably/ghost/prod/2022/09/designing-the-next-global-ably-network.png?tr=w-768,q-50" alt="VPC peering vs Transit Gateway and beyond: Key choices in AWS network design" width="768" class="w-full h-full object-cover" loading="lazy"/></div><div><div class="vt-blog-editorial-tile-tag"><span class="vt-blog-editorial-badge">Ably engineering</span></div><h3 class="vt-blog-editorial-tile-title">VPC peering vs Transit Gateway and beyond: Key choices in AWS network design</h3><div class="vt-blog-editorial-tile-meta"><span class="vt-blog-editorial-tile-meta-author">Huw McNamara</span><span class="vt-blog-editorial-meta-dot">·</span><span class="vt-blog-editorial-tile-meta-mono">Sep 13, 2022</span></div></div></a><a href="/blog/crdts-distributed-data-consistency-challenges" class="vt-blog-editorial-tile" data-testid="post-tile-crdts-distributed-data-consistency-challenges"><div class="vt-blog-editorial-tile-image"><img src="https://ik.imagekit.io/ably/ghost/prod/2021/10/blog-crdts-deep-dive.jpg?tr=w-768,q-50" alt="CRDTs solve distributed data consistency challenges" width="768" class="w-full h-full object-cover" loading="lazy"/></div><div><div class="vt-blog-editorial-tile-tag"><span class="vt-blog-editorial-badge">Ably engineering</span></div><h3 class="vt-blog-editorial-tile-title">CRDTs solve distributed data consistency challenges</h3><div class="vt-blog-editorial-tile-meta"><span class="vt-blog-editorial-tile-meta-author">Jo Stichbury</span><span class="vt-blog-editorial-meta-dot">·</span><span class="vt-blog-editorial-tile-meta-mono">Oct 28, 2021</span></div></div></a></div></div></section></main><footer class="w-full bg-neutral-100 dark:bg-neutral-1200 border-t border-neutral-300 dark:border-neutral-1000" data-id="footer"><div class="max-w-screen-xl mx-auto ui-grid-px pt-10 sm:pt-12 md:pt-16 pb-10"><div class="flex flex-col sm:flex-row gap-x-6 gap-y-12 mb-16 justify-between"><div class="flex-1 flex flex-col gap-6"><a href="/" class="items-center gap-2 justify-center focus-base rounded w-[6.375rem] flex dark:hidden"><img src="https://voltaire.ably.com/static/ably-logo-200721285a51085f43e8a849a85667bc.svg" width="96px" alt="Ably logo"/></a><a href="/" class="items-center gap-2 justify-center focus-base rounded w-[6.375rem] hidden dark:flex"><img src="https://voltaire.ably.com/static/ably-logo-white-749ef5fc6551d0e3b4940c2e3e42463f.svg" width="96px" alt="Ably logo"/></a><a href="https://status.ably.com" class="inline-flex group/status items-center gap-2" target="_blank" rel="noreferrer"><span class="inline-flex h-2 aspect-square rounded-full bg-neutral-500 animate-pulse"></span></a><div class="flex gap-x-6"><a href="https://x.com/ablyrealtime" target="_blank" rel="noreferrer noopener" aria-label="Visit Ably on x" class="w-5 h-5 flex group/social-icon"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="text-neutral-1000 dark:text-neutral-300 group-hover/social-icon:hidden" style="width:20px;height:20px"><path fill="currentColor" d="M17.729 2h3.257l-7.116 8.133L22.24 21.2h-6.554l-5.134-6.712L4.679 21.2h-3.26l7.612-8.699L1 2h6.721l4.64 6.135zm-1.143 17.25h1.804L6.74 3.847H4.804z"></path></svg><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="hidden group-hover/social-icon:flex" style="width:20px;height:20px"><path fill="#03020D" d="M17.729 2h3.257l-7.116 8.133L22.24 21.2h-6.554l-5.134-6.712L4.679 21.2h-3.26l7.612-8.699L1 2h6.721l4.64 6.135zm-1.143 17.25h1.804L6.74 3.847H4.804z"></path></svg></a><a href="https://www.linkedin.com/company/ably-realtime" target="_blank" rel="noreferrer noopener" aria-label="Visit Ably on linkedin" class="w-5 h-5 flex group/social-icon"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="text-neutral-1000 dark:text-neutral-300 group-hover/social-icon:hidden" style="width:20px;height:20px"><path fill="currentColor" d="M24 1.765v20.47A1.764 1.764 0 0 1 22.235 24H1.765A1.765 1.765 0 0 1 0 22.235V1.765A1.765 1.765 0 0 1 1.765 0h20.47A1.765 1.765 0 0 1 24 1.765M7.059 9.176h-3.53v11.295h3.53zm.317-3.882a2.033 2.033 0 0 0-2.018-2.047h-.064a2.047 2.047 0 1 0 0 4.094 2.033 2.033 0 0 0 2.082-1.983zm13.095 8.315c0-3.395-2.16-4.715-4.306-4.715a4.02 4.02 0 0 0-3.572 1.821h-.099V9.176H9.176v11.295h3.53v-6.007a2.344 2.344 0 0 1 2.117-2.527h.135c1.122 0 1.955.705 1.955 2.484v6.05h3.53z"></path></svg><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="hidden group-hover/social-icon:flex" style="width:20px;height:20px"><path fill="#1269BF" d="M24 1.765v20.47A1.764 1.764 0 0 1 22.235 24H1.765A1.765 1.765 0 0 1 0 22.235V1.765A1.765 1.765 0 0 1 1.765 0h20.47A1.765 1.765 0 0 1 24 1.765M7.059 9.176h-3.53v11.295h3.53zm.317-3.882a2.033 2.033 0 0 0-2.018-2.047h-.064a2.047 2.047 0 1 0 0 4.094 2.033 2.033 0 0 0 2.082-1.983zm13.095 8.315c0-3.395-2.16-4.715-4.306-4.715a4.02 4.02 0 0 0-3.572 1.821h-.099V9.176H9.176v11.295h3.53v-6.007a2.344 2.344 0 0 1 2.117-2.527h.135c1.122 0 1.955.705 1.955 2.484v6.05h3.53z"></path></svg></a><a href="https://github.com/ably/" target="_blank" rel="noreferrer noopener" aria-label="Visit Ably on github" class="w-5 h-5 flex group/social-icon"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="text-neutral-1000 dark:text-neutral-300 group-hover/social-icon:hidden" style="width:20px;height:20px"><path fill="currentColor" fill-rule="evenodd" d="M12 0C5.4 0 0 5.42 0 12.043c0 5.32 3.4 9.836 8.2 11.441.6.1.8-.3.8-.602v-2.007c-3.3.702-4-1.606-4-1.606-.5-1.405-1.3-1.806-1.3-1.806-1.1-.703.1-.703.1-.703 1.2.1 1.8 1.204 1.8 1.204 1.1 1.807 2.8 1.305 3.5 1.004.1-.803.4-1.305.8-1.606-2.8-.3-5.6-1.304-5.6-5.92 0-1.306.5-2.41 1.2-3.212 0-.402-.5-1.606.2-3.212 0 0 1-.301 3.3 1.204 1-.3 2-.401 3-.401s2 .1 3 .401c2.3-1.505 3.3-1.204 3.3-1.204.7 1.706.2 2.91.1 3.212.8.802 1.2 1.906 1.2 3.211 0 4.617-2.8 5.62-5.5 5.921.4.402.8 1.104.8 2.208v3.312c0 .301.2.703.8.602 4.9-1.606 8.3-6.122 8.3-11.44C24 5.418 18.6 0 12 0" clip-rule="evenodd"></path></svg><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="hidden group-hover/social-icon:flex" style="width:20px;height:20px"><path fill="#191717" fill-rule="evenodd" d="M12 0C5.4 0 0 5.42 0 12.043c0 5.32 3.4 9.836 8.2 11.441.6.1.8-.3.8-.602v-2.007c-3.3.702-4-1.606-4-1.606-.5-1.405-1.3-1.806-1.3-1.806-1.1-.703.1-.703.1-.703 1.2.1 1.8 1.204 1.8 1.204 1.1 1.807 2.8 1.305 3.5 1.004.1-.803.4-1.305.8-1.606-2.8-.3-5.6-1.304-5.6-5.92 0-1.306.5-2.41 1.2-3.212 0-.402-.5-1.606.2-3.212 0 0 1-.301 3.3 1.204 1-.3 2-.401 3-.401s2 .1 3 .401c2.3-1.505 3.3-1.204 3.3-1.204.7 1.706.2 2.91.1 3.212.8.802 1.2 1.906 1.2 3.211 0 4.617-2.8 5.62-5.5 5.921.4.402.8 1.104.8 2.208v3.312c0 .301.2.703.8.602 4.9-1.606 8.3-6.122 8.3-11.44C24 5.418 18.6 0 12 0" clip-rule="evenodd"></path></svg></a><a href="https://discord.gg/g8yqePUVDn" target="_blank" rel="noreferrer noopener" aria-label="Visit Ably on discord" class="w-5 h-5 flex group/social-icon"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="text-neutral-1000 dark:text-neutral-300 group-hover/social-icon:hidden" style="width:20px;height:20px"><path fill="currentColor" d="M20.317 4.607a19.6 19.6 0 0 0-4.885-1.542.074.074 0 0 0-.079.038c-.21.382-.444.88-.608 1.271a18 18 0 0 0-5.487 0c-.163-.4-.406-.89-.617-1.271a.08.08 0 0 0-.079-.038c-1.714.3-3.354.827-4.885 1.542a.07.07 0 0 0-.032.028C.533 9.364-.32 13.976.099 18.532a.08.08 0 0 0 .031.057 19.8 19.8 0 0 0 5.993 3.082.08.08 0 0 0 .084-.028c.462-.642.874-1.318 1.226-2.03a.08.08 0 0 0-.041-.107A13 13 0 0 1 5.52 18.6a.08.08 0 0 1-.008-.13q.19-.145.372-.297a.07.07 0 0 1 .078-.01c3.927 1.824 8.18 1.824 12.061 0a.07.07 0 0 1 .079.01q.18.15.372.297a.08.08 0 0 1-.006.13q-.895.531-1.873.906a.08.08 0 0 0-.041.109c.36.71.772 1.386 1.225 2.028a.075.075 0 0 0 .084.029 19.7 19.7 0 0 0 6.002-3.082.08.08 0 0 0 .032-.056c.5-5.267-.838-9.842-3.549-13.897a.06.06 0 0 0-.031-.03M8.02 15.757c-1.182 0-2.157-1.104-2.157-2.46s.956-2.46 2.157-2.46c1.21 0 2.176 1.113 2.157 2.46 0 1.356-.956 2.46-2.157 2.46m7.975 0c-1.183 0-2.157-1.104-2.157-2.46s.955-2.46 2.157-2.46c1.21 0 2.176 1.113 2.157 2.46 0 1.356-.946 2.46-2.157 2.46"></path></svg><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="hidden group-hover/social-icon:flex" style="width:20px;height:20px"><path fill="#5B64EA" d="M20.317 4.607a19.6 19.6 0 0 0-4.885-1.542.074.074 0 0 0-.079.038c-.21.382-.444.88-.608 1.271a18 18 0 0 0-5.487 0c-.163-.4-.406-.89-.617-1.271a.08.08 0 0 0-.079-.038c-1.714.3-3.354.827-4.885 1.542a.07.07 0 0 0-.032.028C.533 9.364-.32 13.976.099 18.532a.08.08 0 0 0 .031.057 19.8 19.8 0 0 0 5.993 3.082.08.08 0 0 0 .084-.028c.462-.642.874-1.318 1.226-2.03a.08.08 0 0 0-.041-.107A13 13 0 0 1 5.52 18.6a.08.08 0 0 1-.008-.13q.19-.145.372-.297a.07.07 0 0 1 .078-.01c3.927 1.824 8.18 1.824 12.061 0a.07.07 0 0 1 .079.01q.18.15.372.297a.08.08 0 0 1-.006.13q-.895.531-1.873.906a.08.08 0 0 0-.041.109c.36.71.772 1.386 1.225 2.028a.075.075 0 0 0 .084.029 19.7 19.7 0 0 0 6.002-3.082.08.08 0 0 0 .032-.056c.5-5.267-.838-9.842-3.549-13.897a.06.06 0 0 0-.031-.03M8.02 15.757c-1.182 0-2.157-1.104-2.157-2.46s.956-2.46 2.157-2.46c1.21 0 2.176 1.113 2.157 2.46 0 1.356-.956 2.46-2.157 2.46m7.975 0c-1.183 0-2.157-1.104-2.157-2.46s.955-2.46 2.157-2.46c1.21 0 2.176 1.113 2.157 2.46 0 1.356-.946 2.46-2.157 2.46"></path></svg></a><a href="https://www.youtube.com/c/AblyRealtime" target="_blank" rel="noreferrer noopener" aria-label="Visit Ably on youtube" class="w-5 h-5 flex group/social-icon"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="text-neutral-1000 dark:text-neutral-300 group-hover/social-icon:hidden" style="width:20px;height:20px"><path fill="currentColor" d="M23.499 6.631a3.01 3.01 0 0 0-2.122-2.128C19.505 4 12 4 12 4s-7.505 0-9.377.503A3.01 3.01 0 0 0 .502 6.631C0 8.51 0 12.425 0 12.425s0 3.917.502 5.795a3.01 3.01 0 0 0 2.121 2.128c1.872.503 9.377.503 9.377.503s7.505 0 9.377-.503a3.01 3.01 0 0 0 2.122-2.128C24 16.342 24 12.426 24 12.426s0-3.917-.502-5.795"></path><path fill="#fff" d="m9.545 15.982 6.273-3.556-6.273-3.557z"></path></svg><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="hidden group-hover/social-icon:flex" style="width:20px;height:20px"><path fill="#ED1D24" d="M23.499 6.131a3.01 3.01 0 0 0-2.122-2.128C19.505 3.5 12 3.5 12 3.5s-7.505 0-9.377.503A3.01 3.01 0 0 0 .502 6.131C0 8.01 0 11.925 0 11.925s0 3.917.502 5.795a3.01 3.01 0 0 0 2.121 2.128c1.872.503 9.377.503 9.377.503s7.505 0 9.377-.503a3.01 3.01 0 0 0 2.122-2.128C24 15.842 24 11.926 24 11.926s0-3.917-.502-5.795"></path><path fill="#fff" d="m9.545 15.482 6.273-3.556-6.273-3.557z"></path></svg></a></div><div class="flex flex-wrap gap-2 mt-4 max-w-full"><img src="https://voltaire.ably.com/static/g2-best-meets-requirements-spring-2025-7a9232ad72544eec2aa2918866eb3f7e.svg" alt="G2 Best Meets Requirements Spring 2025" width="55" height="63"/><img src="https://voltaire.ably.com/static/g2-best-support-spring-2025-25e3fb42507694fe41661b0ae587227e.svg" alt="G2 Best Support Spring 2025" width="55" height="63"/><img src="https://voltaire.ably.com/static/g2-users-most-likely-to-recommend-spring-2025-ada1333537cf17600c6f72e11b673838.svg" alt="G2 Users Most Likely to Recommend Spring 2025" width="55" height="63"/><img src="https://voltaire.ably.com/static/g2-easiest-to-use-spring-2025-001641898e826e06845da9c5a7fdca0d.svg" alt="G2 Easiest to Use Spring 2025" width="55" height="63"/></div></div><div class="flex-1 md:flex-[2] flex flex-row flex-wrap gap-x-6 gap-y-12"><div class="flex-1 basis-1/3 md:basis-1"><h3 class="ui-text-overline2 text-neutral-700 dark:text-neutral-600 mb-4">Products</h3><ul class="flex flex-col gap-y-3"><li class="flex gap-x-2"><a href="/pubsub" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably Pub/Sub">Ably Pub/Sub</a></li><li class="flex gap-x-2"><a href="/chat" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably Chat">Ably Chat</a></li><li class="flex gap-x-2"><a href="/ai-transport" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably AI Transport">Ably AI Transport</a></li><li class="flex gap-x-2"><a href="/liveobjects" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably LiveObjects">Ably LiveObjects</a></li><li class="flex gap-x-2"><a href="/spaces" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably Spaces">Ably Spaces</a></li><li class="flex gap-x-2"><a href="/livesync" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably LiveSync">Ably LiveSync</a></li><li class="flex gap-x-2"><a href="/compare" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Compare our tech">Compare our tech</a></li></ul></div><div class="flex-1 basis-1/3 md:basis-1"><h3 class="ui-text-overline2 text-neutral-700 dark:text-neutral-600 mb-4">Platform</h3><ul class="flex flex-col gap-y-3"><li class="flex gap-x-2"><a href="/platform" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Architecture">Architecture</a></li><li class="flex gap-x-2"><a href="/docs/platform/integrations" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Integrations">Integrations</a></li><li class="flex gap-x-2"><a href="/docs/sdks" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit SDKs">SDKs</a></li><li class="flex gap-x-2"><a href="https://changelog.ably.com/" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Changelog">Changelog</a></li><li class="flex gap-x-2"><a href="/security-and-compliance" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Security & Compliance">Security & Compliance</a></li></ul></div><div class="flex-1 basis-1/3 md:basis-1"><h3 class="ui-text-overline2 text-neutral-700 dark:text-neutral-600 mb-4">Get started</h3><ul class="flex flex-col gap-y-3"><li class="flex gap-x-2"><a href="/docs" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Documentation">Documentation</a></li><li class="flex gap-x-2"><a href="/examples" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Examples">Examples</a></li><li class="flex gap-x-2"><a href="/pricing" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Pricing">Pricing</a></li><li class="flex gap-x-2"><a href="/topics" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Realtime A-Z">Realtime A-Z</a></li><li class="flex gap-x-2"><a href="/support" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Support">Support</a></li></ul></div><div class="flex-1 basis-1/3 md:basis-1"><h3 class="ui-text-overline2 text-neutral-700 dark:text-neutral-600 mb-4">Company</h3><ul class="flex flex-col gap-y-3"><li class="flex gap-x-2"><a href="/about" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit About Ably">About Ably</a></li><li class="flex gap-x-2"><a href="/blog" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Blog">Blog</a></li><li class="flex gap-x-2"><a href="/careers" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Careers">Careers</a><div class="inline-flex bg-neutral-100 dark:bg-neutral-1200 rounded-2xl gap-1 items-center focus-base transition-colors select-none px-2 py-0 text-[10px] leading-tight text-neutral-900 dark:text-neutral-400 ui-text-p4 font-[10px]"><span class="whitespace-nowrap tracking-[0.04em] leading-[18px]">WE’RE HIRING</span></div></li><li class="flex gap-x-2"><a href="/contact" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Contact us">Contact us</a></li></ul></div></div></div><div class="pt-6 border-t border-neutral-300 dark:border-neutral-1000"><div class="flex gap-6"><a href="/data-protection" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Data protection">Data protection</a><a href="/privacy" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Privacy">Privacy</a><a href="/legals" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Legals">Legals</a><a href="/privacy" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Cookies">Cookies</a></div></div></div></footer><div class="fixed transition-[bottom] left-0 p-8 z-50 bottom-[-100px]" data-testid="layout-scroll-to-top"><div class="relative"><a href="#" class="bg-white w-12 h-12 cursor-pointer overflow-hidden rounded border-2 border-mid-grey hover:border-active-orange transition-all duration-[400ms] block"><div class="w-full h-full transition-all duration-[400ms] md:hover:translate-y-[-100%]"><div class="w-full h-full flex flex-row items-center justify-center"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" aria-hidden="true" data-slot="icon" class="text-cool-black" style="width:1.5rem;height:1.5rem"><path stroke-linecap="round" stroke-linejoin="round" d="M8.25 6.75 12 3m0 0 3.75 3.75M12 3v18"></path></svg></div><div class="w-full h-full flex flex-row items-center justify-center"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" aria-hidden="true" data-slot="icon" class="text-cool-black" style="width:1.5rem;height:1.5rem"><path stroke-linecap="round" stroke-linejoin="round" d="M8.25 6.75 12 3m0 0 3.75 3.75M12 3v18"></path></svg></div></div></a></div></div></div><div id="gatsby-announcer" style="position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0" aria-live="assertive" aria-atomic="true"></div></div><script>
|
||
(function() {
|
||
// Apply initial theme to dark navigation elements
|
||
var darkNav = document.getElementById('meganav-theme-dark');
|
||
if (darkNav) {
|
||
var header = darkNav.querySelector('header');
|
||
if (header && !header.classList.contains('ui-theme-dark')) {
|
||
header.classList.add('ui-theme-dark');
|
||
}
|
||
}
|
||
|
||
var darkMain = document.getElementById('main-theme-dark');
|
||
if (darkMain && !darkMain.classList.contains('ui-theme-dark')) {
|
||
darkMain.classList.add('ui-theme-dark');
|
||
}
|
||
})();
|
||
</script><script id="gatsby-script-loader">/*<![CDATA[*/window.pagePath="/blog/the-18-ghosts-in-your-infrastructure-stack";/*]]>*/</script><!-- slice-start id="_gatsby-scripts-1" -->
|
||
<script
|
||
id="gatsby-chunk-mapping"
|
||
>
|
||
window.___chunkMapping="{\"app\":[\"/app-afd9e708c8f5048ea0db.js\"],\"component---src-pages-404-js\":[\"/component---src-pages-404-js-b6e8f7edd8dea77e0c57.js\"],\"component---src-pages-ai-transport-tsx\":[\"/component---src-pages-ai-transport-tsx-4026b54583210f8a2053.js\"],\"component---src-pages-chat-tsx\":[\"/component---src-pages-chat-tsx-b2bdc3f08ae11253c0e3.js\"],\"component---src-pages-contact-tsx\":[\"/component---src-pages-contact-tsx-dc89e20c2190ed888225.js\"],\"component---src-pages-customer-service-chat-tsx\":[\"/component---src-pages-customer-service-chat-tsx-48ef3c126214086f90be.js\"],\"component---src-pages-cx-tech-tsx\":[\"/component---src-pages-cx-tech-tsx-36c4ce5a367e2a78a600.js\"],\"component---src-pages-ed-tech-tsx\":[\"/component---src-pages-ed-tech-tsx-d37da5b8d8490a573dbd.js\"],\"component---src-pages-fan-engagement-tsx\":[\"/component---src-pages-fan-engagement-tsx-8a1c337ce51c17c42fb7.js\"],\"component---src-pages-fin-tech-tsx\":[\"/component---src-pages-fin-tech-tsx-569a7d8f0503b63d5dc2.js\"],\"component---src-pages-four-pillars-of-dependability-tsx\":[\"/component---src-pages-four-pillars-of-dependability-tsx-d7a6e0386690e0cc591b.js\"],\"component---src-pages-global-round-trip-latencies-tsx\":[\"/component---src-pages-global-round-trip-latencies-tsx-b68878661cea17e95b93.js\"],\"component---src-pages-group-chat-tsx\":[\"/component---src-pages-group-chat-tsx-ebd327d243327ee9f8c8.js\"],\"component---src-pages-health-tech-tsx\":[\"/component---src-pages-health-tech-tsx-35d7fa3532326d5e5c7f.js\"],\"component---src-pages-hipaa-compliant-chat-tsx\":[\"/component---src-pages-hipaa-compliant-chat-tsx-acbc2476763adc3c3443.js\"],\"component---src-pages-index-tsx\":[\"/component---src-pages-index-tsx-322545dbcb61532adb0d.js\"],\"component---src-pages-live-streaming-chat-tsx\":[\"/component---src-pages-live-streaming-chat-tsx-e5e76e4db0722d9207ca.js\"],\"component---src-pages-liveobjects-tsx\":[\"/component---src-pages-liveobjects-tsx-28aa815395acdc2ce82e.js\"],\"component---src-pages-livesync-tsx\":[\"/component---src-pages-livesync-tsx-70885b210a264e5141e5.js\"],\"component---src-pages-periodic-table-of-realtime-js\":[\"/component---src-pages-periodic-table-of-realtime-js-d61b7b51a3fc51b8db26.js\"],\"component---src-pages-pricing-contact-tsx\":[\"/component---src-pages-pricing-contact-tsx-6d358f3f151478534903.js\"],\"component---src-pages-pricing-tsx\":[\"/component---src-pages-pricing-tsx-0354dbd8801f5243ece3.js\"],\"component---src-pages-pubsub-tsx\":[\"/component---src-pages-pubsub-tsx-464d839be2211c173921.js\"],\"component---src-pages-react-tsx\":[\"/component---src-pages-react-tsx-87358a2709374aacf56e.js\"],\"component---src-pages-reference-guide-chat-tsx\":[\"/component---src-pages-reference-guide-chat-tsx-8e3b14932c19bcc1e464.js\"],\"component---src-pages-reference-guide-multiplayer-tsx\":[\"/component---src-pages-reference-guide-multiplayer-tsx-9c53536d0cc84d1f7db8.js\"],\"component---src-pages-spaces-tsx\":[\"/component---src-pages-spaces-tsx-c46f420684aae4fb56f5.js\"],\"component---src-pages-websocket-service-tsx\":[\"/component---src-pages-websocket-service-tsx-29bc4c2263ebd30de9ce.js\"],\"component---src-templates-ably-vs-competitors-tsx\":[\"/component---src-templates-ably-vs-competitors-tsx-052a2027d366b1271875.js\"],\"component---src-templates-alternatives-index-tsx\":[\"/component---src-templates-alternatives-index-tsx-935d9b574657fb06b2f7.js\"],\"component---src-templates-alternatives-tsx\":[\"/component---src-templates-alternatives-tsx-5fc8ac958760659bdf83.js\"],\"component---src-templates-business-user-business-user-landing-tsx\":[\"/component---src-templates-business-user-business-user-landing-tsx-a017f6d2e2f718a2eafb.js\"],\"component---src-templates-compare-ably-js\":[\"/component---src-templates-compare-ably-js-6c0f08a75c6b11b3f317.js\"],\"component---src-templates-compare-index-js\":[\"/component---src-templates-compare-index-js-88ec96d0ed1e663d23ca.js\"],\"component---src-templates-compare-v-1-js\":[\"/component---src-templates-compare-v-1-js-d9b82d0110acbb502264.js\"],\"component---src-templates-compare-v-2-tsx\":[\"/component---src-templates-compare-v-2-tsx-4e1cd6756c146c67224d.js\"],\"component---src-templates-customer-story-tsx\":[\"/component---src-templates-customer-story-tsx-c07c97ff42a331eb75ae.js\"],\"component---src-templates-downloadable-gated-resource-downloadable-gated-resource-tsx\":[\"/component---src-templates-downloadable-gated-resource-downloadable-gated-resource-tsx-91158f3c7c6f4d153cd2.js\"],\"component---src-templates-downloadable-gated-resource-non-gated-resources-tsx\":[\"/component---src-templates-downloadable-gated-resource-non-gated-resources-tsx-7f08268bbae47759ffba.js\"],\"component---src-templates-downloadable-gated-resource-thank-you-page-js\":[\"/component---src-templates-downloadable-gated-resource-thank-you-page-js-0d99d8fac0c6445d301a.js\"],\"component---src-templates-flexible-tsx\":[\"/component---src-templates-flexible-tsx-088d00c89eb5a0ed6bfe.js\"],\"component---src-templates-flexible-v-2-tsx\":[\"/component---src-templates-flexible-v-2-tsx-1a6ff88fd8c866a1c57a.js\"],\"component---src-templates-generic-index-tsx\":[\"/component---src-templates-generic-index-tsx-89128b31a07f898b23dc.js\"],\"component---src-templates-ghost-pages-blog-list-blog-author-category-tsx\":[\"/component---src-templates-ghost-pages-blog-list-blog-author-category-tsx-820582c83ba43ebd3312.js\"],\"component---src-templates-ghost-pages-blog-list-blog-author-tsx\":[\"/component---src-templates-ghost-pages-blog-list-blog-author-tsx-eb3d8eb8dd047249efec.js\"],\"component---src-templates-ghost-pages-blog-list-blog-category-tsx\":[\"/component---src-templates-ghost-pages-blog-list-blog-category-tsx-45a6de80e8d75cb9c029.js\"],\"component---src-templates-ghost-pages-blog-list-blog-index-tsx\":[\"/component---src-templates-ghost-pages-blog-list-blog-index-tsx-0c5b2b87757efc90b336.js\"],\"component---src-templates-ghost-pages-blog-post-tsx\":[\"/component---src-templates-ghost-pages-blog-post-tsx-18ffed0af8673bb36129.js\"],\"component---src-templates-migrate-from-js\":[\"/component---src-templates-migrate-from-js-c3c1db295b667b6a61d9.js\"],\"component---src-templates-page-index-page-index-tsx\":[\"/component---src-templates-page-index-page-index-tsx-98b69b9679ce876767c4.js\"],\"component---src-templates-solution-tsx\":[\"/component---src-templates-solution-tsx-d8b78e013cb19083a3c9.js\"],\"component---src-templates-topic-tsx\":[\"/component---src-templates-topic-tsx-f902ddfff741dba97b74.js\"],\"component---src-templates-topics-category-tsx\":[\"/component---src-templates-topics-category-tsx-1433f8d904055ca038df.js\"],\"component---src-templates-topics-tsx\":[\"/component---src-templates-topics-tsx-436fa51625e6743f0934.js\"]}";
|
||
</script>
|
||
<script>window.___webpackCompilationHash="bd450251d228ea1e62eb";</script><script src="https://voltaire.ably.com/webpack-runtime-835cc4d6ec8a5974e673.js" async></script><script src="https://voltaire.ably.com/framework-5e45d846ab16cc6f99a4.js" async></script><script src="https://voltaire.ably.com/550fdbb7-7d2088aeb95a473eadaf.js" async></script><script src="https://voltaire.ably.com/6c2eb795-43135b94c65e4e3f8c79.js" async></script><script src="https://voltaire.ably.com/67a3bf3c-098d4132adafb49f830f.js" async></script><script src="https://voltaire.ably.com/ce3a710f-71c60be4cf8d98c10377.js" async></script><script src="https://voltaire.ably.com/app-afd9e708c8f5048ea0db.js" async></script><!-- slice-end id="_gatsby-scripts-1" --></body></html> |