Files
nexus/sreweekly/articles/439/06-optimizing-global-message-transit-latency-a-journey-through-tcp-config.html
2026-09-12 17:23:01 +08:00

51 lines
74 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta http-equiv="x-ua-compatible" content="ie=edge"/><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"/><meta name="generator" content="Gatsby 5.16.0"/><meta data-react-helmet="true" name="description" content="Our team tackled high latency in realtime messaging by optimizing TCP/IP settings. Learn about our process, findings, and TCP config solutions."/><meta data-react-helmet="true" property="og:site_name" content="Ably Realtime"/><meta data-react-helmet="true" property="og:type" content="website"/><meta data-react-helmet="true" property="og:url" content="https://ably.com/blog/optimizing-global-message-transit-latency-a-journey-through-tcp-configuration"/><meta data-react-helmet="true" property="og:title" content="Optimizing global message transit latency: a journey through TCP configuration"/><meta data-react-helmet="true" property="og:description" content="Our team tackled high latency in realtime messaging by optimizing TCP/IP settings. Learn about our process, findings, and TCP config solutions."/><meta data-react-helmet="true" property="og:image" content="https://files.ably.io/ghost/prod/2024/07/image.png"/><meta data-react-helmet="true" name="twitter:card" content="summary_large_image"/><meta data-react-helmet="true" name="twitter:site" content="@ablyrealtime"/><link href="https://voltaire.ably.com/styles.401201978d276adf9700.css" rel="stylesheet" type="text/css" media="screen"/><style>.gatsby-image-wrapper{position:relative;overflow:hidden}.gatsby-image-wrapper picture.object-fit-polyfill{position:static!important}.gatsby-image-wrapper img{bottom:0;height:100%;left:0;margin:0;max-width:none;padding:0;position:absolute;right:0;top:0;width:100%;object-fit:cover}.gatsby-image-wrapper [data-main-image]{opacity:0;transform:translateZ(0);transition:opacity .25s linear;will-change:opacity}.gatsby-image-wrapper-constrained{display:inline-block;vertical-align:top}</style><noscript><style>.gatsby-image-wrapper noscript [data-main-image]{opacity:1!important}.gatsby-image-wrapper [data-placeholder-image]{opacity:0!important}</style></noscript><script type="module">const e="undefined"!=typeof HTMLImageElement&&"loading"in HTMLImageElement.prototype;e&&document.body.addEventListener("load",(function(e){const t=e.target;if(void 0===t.dataset.mainImage)return;if(void 0===t.dataset.gatsbyImageSsr)return;let a=null,n=t;for(;null===a&&n;)void 0!==n.parentNode.dataset.gatsbyImageWrapper&&(a=n.parentNode),n=n.parentNode;const o=a.querySelector("[data-placeholder-image]"),r=new Image;r.src=t.currentSrc,r.decode().catch((()=>{})).then((()=>{t.style.opacity=1,o&&(o.style.opacity=0,o.style.transition="opacity 500ms linear")}))}),!0);</script><style data-styled="" data-styled-version="6.5.0">.gLNiCf{scroll-margin-top:16px;}/*!sc*/
data-styled.g13[id="layout__Main-sc-17rhhsn-0"]{content:"gLNiCf,"}/*!sc*/
.jTvGq{transition:transform 0.3s;position:sticky;top:64px;bottom:1rem;}/*!sc*/
data-styled.g22[id="sticky-sidebar-mobile__Sidebar-sc-gqtcvm-0"]{content:"jTvGq,"}/*!sc*/
.dYTnoj{position:fixed;top:0;left:0;right:0;bottom:0;background:rgba(0,0,0,0.5);z-index:1;transform:translateZ(0);pointer-events:none;visibility:hidden;opacity:0;transition:opacity 0.2s;height:100vh;}/*!sc*/
data-styled.g29[id="overlay__Overlay-sc-j0b6or-0"]{content:"dYTnoj,"}/*!sc*/
.ILNdO{max-height:calc(100vh - 64px);transition:transform 0.3s ease-in-out;transform:translateY(-100%);z-index:-1;}/*!sc*/
data-styled.g30[id="table-of-contents-mobile__Container-sc-1g3ebrb-0"]{content:"ILNdO,"}/*!sc*/
</style><link rel="alternate" type="application/rss+xml" title="The Ably Blog" href="https://voltaire.ably.com/blog/rss.xml"/><script>window.dataLayer = window.dataLayer || [];window.dataLayer.push({"platform":"gatsby","gaPageView":false}); (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= 'https://www.googletagmanager.com/gtm.js?id='+i+dl+'';f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer', 'GTM-TZ37KKW');</script><script src="https://cdn-ukwest.onetrust.com/scripttemplates/otSDKStub.js" data-domain-script="b1eb1bbc-c8f1-47ad-87a5-4a2a6f01202c"></script><script>window.OptanonWrapper = function(){};</script><link data-react-helmet="true" rel="canonical" href="https://ably.com/blog/optimizing-global-message-transit-latency-a-journey-through-tcp-configuration"/><link data-react-helmet="true" rel="icon" href="/favicon.svg" type="image/svg+xml"/><link data-react-helmet="true" rel="icon" href="/favicon-96x96.png" sizes="96x96" type="image/png"/><link data-react-helmet="true" rel="icon" href="/favicon.ico" sizes="48x48 32x32 16x16"/><link data-react-helmet="true" rel="apple-touch-icon" href="/apple-touch-icon.png"/><link data-react-helmet="true" rel="manifest" href="/site.webmanifest"/><link data-react-helmet="true" rel="preconnect" href="https://fonts.googleapis.com"/><link data-react-helmet="true" rel="preconnect" href="https://fonts.gstatic.com" crossorigin=""/><link data-react-helmet="true" href="https://fonts.googleapis.com/css2?family=JetBrains+Mono:ital,wght@0,100..800;1,100..800&amp;family=Manrope:wght@200..800&amp;family=Source+Code+Pro:wght@600&amp;display=swap" rel="stylesheet"/><script data-react-helmet="true" type="application/ld+json">{"@context":"https://schema.org","@graph":[{"@type":"TechArticle","@id":"https://ably.com/blog/optimizing-global-message-transit-latency-a-journey-through-tcp-configuration#article","headline":"Optimizing global message transit latency: a journey through TCP configuration","abstract":"Our team tackled high latency in realtime messaging by optimizing TCP/IP settings. Learn about our process, findings, and TCP config solutions.","mainEntityOfPage":"https://ably.com/blog/optimizing-global-message-transit-latency-a-journey-through-tcp-configuration","datePublished":"2024-07-29T00:00:00+00:00","dateModified":"2024-07-29T00:00:00+00:00","inLanguage":"en-US","image":{"@type":"ImageObject","url":"https://files.ably.io/ghost/prod/2024/07/image.png"},"author":{"@type":"Person","name":"Amnon Cohen","url":"https://ably.com/blog/author/amnon","worksFor":{"@id":"https://ably.com#organization"}},"publisher":{"@id":"https://ably.com#organization"}},{"@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"item":{"@id":"https://ably.com","name":"Ably"}},{"@type":"ListItem","position":2,"item":{"@id":"https://ably.com/blog","name":"Blog"}},{"@type":"ListItem","position":3,"item":{"@id":"https://ably.com/blog/optimizing-global-message-transit-latency-a-journey-through-tcp-configuration","name":"Optimizing global message transit latency: a journey through TCP configuration"}}]},{"@type":"Organization","@id":"https://ably.com#organization","name":"Ably","url":"https://ably.com","logo":{"@type":"ImageObject","url":"https://ably.com/favicon-512x512.png"},"knowsAbout":["WebSockets","realtime messaging","pub/sub","AI Transport","durable sessions","token streaming","session continuity"],"sameAs":["https://www.wikidata.org/wiki/Q139764920","https://www.linkedin.com/company/ably-realtime/","https://twitter.com/ablyrealtime","https://github.com/ably","https://www.crunchbase.com/organization/ably-realtime","https://www.g2.com/products/ably"]},{"@type":"WebSite","@id":"https://ably.com#website","name":"Ably","url":"https://ably.com","publisher":{"@id":"https://ably.com#organization"}}]}</script><style id="sandpack">--sxs{--sxs:1 sp-k-eyOShd sp-k-iOHdLQ}@media{@keyframes sp-k-eyOShd{0%{opacity:0}100%{opacity:1}}@keyframes sp-k-iOHdLQ{0%{transform:rotateX(-25.5deg) rotateY(45deg)}100%{transform:rotateX(-25.5deg) rotateY(405deg)}}}--sxs{--sxs:2 sp-c-gMfcns sp-c-bxeRRt sp-c-jKPvnt sp-c-fWymNx sp-c-euXojQ sp-c-bpmgvy sp-c-PJLV}@media{.sp-c-gMfcns svg{margin:auto}.sp-c-bxeRRt{-webkit-appearance:none;appearance:none;outline:none;display:flex;align-items:center;font-size:inherit;font-family:inherit;background-color:transparent;transition:color var(--sp-transitions-default), background var(--sp-transitions-default);cursor:pointer;color:var(--sp-colors-clickable);border:0;text-decoration:none}.sp-c-bxeRRt:disabled{color:var(--sp-colors-disabled)}.sp-c-bxeRRt:hover:not(:disabled,[data-active='true']){color:var(--sp-colors-hover)}.sp-c-bxeRRt[data-active="true"]{color:var(--sp-colors-accent)}.sp-c-bxeRRt svg{min-width:var(--sp-space-4);width:var(--sp-space-4);height:var(--sp-space-4)}.sp-c-bxeRRt.sp-c-gMfcns{padding:var(--sp-space-1);height:var(--sp-space-7);display:flex}.sp-c-bxeRRt.sp-c-gMfcns.sp-c-bxeRRt:not(:has(span)){width:var(--sp-space-7)}.sp-c-bxeRRt.sp-c-gMfcns.sp-c-bxeRRt:has(svg + span){padding-right:var(--sp-space-3);padding-left:var(--sp-space-2);gap:var(--sp-space-1)}.sp-c-jKPvnt{padding:0 var(--sp-space-1) 0 var(--sp-space-1);border-radius:var(--sp-border-radius);margin-left:var(--sp-space-1);width:var(--sp-space-5);visibility:hidden;cursor:pointer;position:absolute;right:0px}.sp-c-jKPvnt svg{width:var(--sp-space-3);height:var(--sp-space-3);display:block;position:relative;top:1px}.sp-c-fWymNx{margin:0;display:block;font-family:var(--sp-font-mono);font-size:var(--sp-font-size);color:var(--sp-syntax-color-plain);line-height:var(--sp-font-lineHeight)}.sp-c-euXojQ{display:flex;flex-direction:column;width:100%;position:relative;background-color:var(--sp-colors-surface1);gap:1px}.sp-c-euXojQ:has(.sp-stack){background-color:var(--sp-colors-surface2)}.sp-c-bpmgvy{transform:translate(-4px, 9px) scale(0.13, 0.13)}.sp-c-bpmgvy *{position:absolute;width:96px;height:96px}}--sxs{--sxs:3 sp-c-PJLV-kCOVwI-status-pass sp-c-PJLV-kEzYsr-status-fail sp-c-PJLV-gHAhSA-status-skip sp-c-PJLV-jgnHyR-status-title sp-c-PJLV-iCgxLS-status-run sp-c-PJLV-bnDZSy-status-pass sp-c-PJLV-eYuGwt-status-fail}@media{.sp-c-PJLV-kCOVwI-status-pass{color:var(--test-pass)}.sp-c-PJLV-kEzYsr-status-fail{color:var(--test-fail)}.sp-c-PJLV-gHAhSA-status-skip{color:var(--test-skip)}.sp-c-PJLV-jgnHyR-status-title{color:var(--test-title)}.sp-c-PJLV-iCgxLS-status-run{background:var(--test-run);color:var(--sp-colors-surface1)}.sp-c-PJLV-bnDZSy-status-pass{background:var(--test-pass);color:var(--sp-colors-surface1)}.sp-c-PJLV-eYuGwt-status-fail{background:var(--test-fail);color:var(--sp-colors-surface1)}}</style></head><body><noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-TZ37KKW" height="0" width="0" style="display: none; visibility: hidden" aria-hidden="true"></iframe></noscript><div id="___gatsby"><div style="outline:none" tabindex="-1" id="gatsby-focus-wrapper"><div class="absolute top-0 left-0 right-0 w-full z-50" id="meganav" data-testid="meganav" style="height:64px"><header role="banner" style="height:64px;top:0" class="fixed left-0 top-0 w-full z-50 bg-neutral-000 dark:bg-neutral-1300 border-b border-neutral-300 dark:border-neutral-1000 transition-all duration-300 ease-in-out"><div class="flex items-center h-full max-w-screen-xl mx-auto ui-grid-px"><nav class="flex flex-1 h-full items-center"><a href="/" class="items-center gap-2 justify-center h-full focus-base rounded mr-4 lg:mr-8 flex dark:hidden"><img src="https://voltaire.ably.com/static/ably-logo-200721285a51085f43e8a849a85667bc.svg" width="96px" alt="Ably logo"/></a><a href="/" class="items-center gap-2 justify-center h-full focus-base rounded mr-4 lg:mr-8 hidden dark:flex"><img src="https://voltaire.ably.com/static/ably-logo-white-749ef5fc6551d0e3b4940c2e3e42463f.svg" width="96px" alt="Ably logo"/></a><div class="hidden md:flex flex-1 items-center h-full"><nav aria-label="Main" data-orientation="horizontal" dir="ltr" class="justify-left z-40 flex w-full"><div style="position:relative"><ul data-orientation="horizontal" class="flex list-none center" dir="ltr"><li><button id="radix-:R32p9:-trigger-radix-:R2r2p9:" data-state="closed" aria-expanded="false" aria-controls="radix-:R32p9:-content-radix-:R2r2p9:" class="group outline-none focus:outline-none select-none cursor-pointer relative rounded-md hover:bg-neutral-100 dark:hover:bg-neutral-1200 [&amp;[data-state=open]]:bg-neutral-100 dark:[&amp;[data-state=open]]:bg-neutral-1200 [&amp;[data-state=open]]:text-neutral-1300 dark:[&amp;[data-state=open]]:text-neutral-000 ui-text-label3 font-semibold text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 px-3 py-2 flex items-center justify-between" data-radix-collection-item="">Products</button></li><li><button id="radix-:R32p9:-trigger-radix-:R4r2p9:" data-state="closed" aria-expanded="false" aria-controls="radix-:R32p9:-content-radix-:R4r2p9:" class="group outline-none focus:outline-none select-none cursor-pointer relative rounded-md hover:bg-neutral-100 dark:hover:bg-neutral-1200 [&amp;[data-state=open]]:bg-neutral-100 dark:[&amp;[data-state=open]]:bg-neutral-1200 [&amp;[data-state=open]]:text-neutral-1300 dark:[&amp;[data-state=open]]:text-neutral-000 ui-text-label3 font-semibold text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 px-3 py-2 flex items-center justify-between" data-radix-collection-item="">Solutions</button></li><li><button id="radix-:R32p9:-trigger-radix-:R6r2p9:" data-state="closed" aria-expanded="false" aria-controls="radix-:R32p9:-content-radix-:R6r2p9:" class="group outline-none focus:outline-none select-none cursor-pointer relative rounded-md hover:bg-neutral-100 dark:hover:bg-neutral-1200 [&amp;[data-state=open]]:bg-neutral-100 dark:[&amp;[data-state=open]]:bg-neutral-1200 [&amp;[data-state=open]]:text-neutral-1300 dark:[&amp;[data-state=open]]:text-neutral-000 ui-text-label3 font-semibold text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 px-3 py-2 flex items-center justify-between" data-radix-collection-item="">Company</button></li><a href="/pricing" class="ui-text-label3 font-semibold text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 px-3 py-2 flex items-center justify-between" data-radix-collection-item="">Pricing</a><a href="/docs" class="ui-text-label3 font-semibold text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 px-3 py-2 flex items-center justify-between" data-radix-collection-item="">Docs</a></ul></div><div class="absolute top-full flex justify-left"></div></nav></div></nav><div class="flex md:hidden flex-1 items-center justify-end gap-6 h-full"><button class="cursor-pointer focus-base rounded flex items-center p-0" aria-expanded="false" aria-controls="mobile-menu" aria-label="Toggle menu"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" aria-hidden="true" data-slot="icon" class="text-neutral-1300 dark:text-neutral-000" style="width:1.5rem;height:1.5rem"><path stroke-linecap="round" stroke-linejoin="round" d="M3.75 6.75h16.5M3.75 12h16.5m-16.5 5.25h16.5"></path></svg></button></div><nav class="md:flex-1 md:items-center md:justify-end flex-col md:flex-row border-t-[1px] border-neutral-300 md:border-t-0 md:gap-4 pt-3 pb-4 md:py-0 hidden md:flex flex-1 items-center h-full md:gap-x-6"><a class="font-sans !text-label2 md:!text-label3 font-semibold py-4 text-neutral-1300 dark:text-neutral-000 md:text-neutral-1000 dark:md:text-neutral-300 hover:text-neutral-1300 dark:hover:text-neutral-000 active:text-neutral-1300 dark:active:text-neutral-000 transition-colors flex items-center gap-1.5 px-4 md:px-0 leading-none" href="/contact">Contact us</a><div class="flex gap-3 pt-3 md:py-0 px-4 md:px-0"><a class="ui-button-secondary flex-1 md:flex-none md:ui-button-secondary-xs hover:text-neutral-1300 dark:hover:text-neutral-000" role="button" tabindex="0" href="/login">Login</a><a class="ui-button-primary flex-1 md:flex-none md:ui-button-primary-xs hover:text-neutral-000 dark:hover:text-neutral-1300" role="button" tabindex="0" href="/sign-up">Start free</a></div></nav></div></header></div><div class="ui-flash" data-id="ui-flashes"></div><main class="layout__Main-sc-17rhhsn-0 gLNiCf vt-blog-editorial font-sans antialiased" id="main" data-testid="layout-main"><div class="vt-blog-editorial-reading-progress" aria-hidden="true"><div class="vt-blog-editorial-reading-progress-bar" style="width:0%"></div></div><aside data-sidebar-sticking="false" class="sticky-sidebar-mobile__Sidebar-sc-gqtcvm-0 jTvGq md:hidden w-full z-20 bg-transparent"><div class="relative"><div class="font-sans ui-grid-px bg-white border-mid-grey border-b flex items-center py-4 relative z-10"><div class="ui-text-overline2 text-dark-grey font-medium">On this page</div><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="currentColor" aria-hidden="true" data-slot="icon" class="ml-auto absolute right-0 mr-6 sm:mr-8" style="width:1.5rem;height:1.5rem"><path fill-rule="evenodd" d="M4.22 6.22a.75.75 0 0 1 1.06 0L8 8.94l2.72-2.72a.75.75 0 1 1 1.06 1.06l-3.25 3.25a.75.75 0 0 1-1.06 0L4.22 7.28a.75.75 0 0 1 0-1.06Z" clip-rule="evenodd"></path></svg></div><div class="table-of-contents-mobile__Container-sc-1g3ebrb-0 ILNdO border-b border-color-mid-grey absolute bg-white overflow-y-auto w-full"></div><div class="overlay__Overlay-sc-j0b6or-0 dYTnoj"></div></div></aside><header class="vt-blog-editorial-header"><div class="vt-blog-editorial-header-back-row"><a href="/blog" class="vt-blog-editorial-header-back"><span aria-hidden="true">←</span> Blog</a></div><span class="vt-blog-editorial-hero-eyebrow"><a href="/blog/ably-engineering" class="vt-blog-editorial-hero-eyebrow-cat no-underline" data-testid="hero-kicker">Ably engineering</a></span><h1 id="article-heading" class="vt-blog-editorial-h1">Optimizing global message transit latency: a journey through TCP configuration</h1><p class="vt-blog-editorial-dek">Discover how our engineering team tackled high latency variance in realtime messaging by optimizing TCP/IP settings. Learn about our investigative process, findings, and solutions around TCP/IP config to ensure low and consistent latency.</p><div class="vt-blog-editorial-byline"><a href="/blog/author/amnon" class="vt-blog-editorial-byline-author"><img src="https://www.gravatar.com/avatar/a07294f23c03f8ea8c6c7bdefe0b717a?s=250&amp;r=x&amp;d=mp" alt="Amnon Cohen" width="72" class="vt-blog-editorial-byline-avatar"/><span class="vt-blog-editorial-byline-name">Amnon Cohen</span></a><div class="vt-blog-editorial-byline-meta-col"><span class="vt-blog-editorial-byline-meta">Jul 29, 2024</span><span class="vt-blog-editorial-meta-dot">·</span><span class="vt-blog-editorial-byline-meta">12 min read</span></div></div><div class="vt-blog-editorial-header-rule" aria-hidden="true"></div></header><div class="vt-blog-editorial-grid"><article aria-labelledby="article-heading"><figure class="vt-blog-editorial-feature-figure"><img src="https://ik.imagekit.io/ably/ghost/prod/2024/07/image.png?tr=w-1280,q-50" alt="Optimizing global message transit latency: a journey through TCP configuration" width="1280" class="w-full h-auto block"/></figure><div class="ghost-blog-post-body"><p>At Ably, we provide a realtime messaging service that aims to deliver messages between globally-distributed participants with the lowest latency possible. When powering apps that enable realtime collaboration, or require time-critical updates, low and consistent latency is essential.<br></p><p>We recently deployed dedicated infrastructure for a new customer whose workload involved sporadic but highly time-critical updates between locations in Europe, South America and South East Asia. What we observed with this traffic was unusually high variance in the latency of messages passing through Ably between these locations. Taking the example of London (<code>AWS region eu-west-2</code>) to Singapore (<code>AWS region ap-southeast-1</code>), although the minimum transit time was close to the best possible at around 90ms, the slowest 5% of messages took over 400ms; this was much greater than our target for this particular workload of below 150ms. We set out to understand and fix whatever issues were causing that higher than expected variance.</p><h2 id="initial-measurements">Initial measurements</h2><p>The round trip time between two machines can be measured using a utility called <code>ping</code>, which bounces packets between two machines. The <code>ping</code> time between our machines in London and Singapore was around 180ms, which meant that a packet <em>could</em> travel one way from Europe to SE Asia in 90ms. This implied that we could reliably deliver messages between Europe and SE Asia in 90ms, plus a small number of milliseconds for message processing at intermediate nodes.</p><p>When we measured our <em>actual</em> latency in the customer’s case, we found that 5% of messages took over 400ms to arrive instead of the expected 90ms. We refer to this figure as the p95 latency or the latency of the 95th percentile. An initial examination of our own code stack did not show a source of the delay, but we noticed that many of the messages were delayed by multiples of the 180ms round trip time (RTT), with peaks around 270ms and 450ms.</p><h3 id="initial-investigation">Initial investigation</h3><p>This particular workload was sporadic, with periods of seconds when no messages were sent, followed by bursts of messages. We found that if we send a high rate of <a href="https://www.geeksforgeeks.org/what-are-heartbeat-messages/">“heartbeats”</a> (periodic messages), at the rate of 50 messages per second, the latency tails would disappear.</p><figure class="kg-card kg-image-card kg-card-hascaption"><img src="https://ik.imagekit.io/ably/ghost/prod/2024/07/Screenshot-2024-07-26-at-12.21.05.png" class="kg-image" alt="" loading="lazy" width="1356" height="622" srcset="https://ik.imagekit.io/ably/ghost/prod/2024/07/Screenshot-2024-07-26-at-12.21.05.png?tr=w-380,q-50 380w,
https://ik.imagekit.io/ably/ghost/prod/2024/07/Screenshot-2024-07-26-at-12.21.05.png?tr=w-760,q-50 760w,
https://ik.imagekit.io/ably/ghost/prod/2024/07/Screenshot-2024-07-26-at-12.21.05.png?tr=w-1520,q-50 1520w" sizes="
(min-width: 1520px) 1520px,
100vw
"><figcaption>Message latency from London &gt; Singapore: High frequency heartbeats were turned on at 20:44</figcaption></figure><p>This led us to suspect that the source of the delay was message propagation delays on the actual TCP/IP connection, rather than processing delays arising from our own code. Normally, propagation delays might be due to congestion on a connection, but the fact that a higher message rate could eliminate the latency variance strongly suggested that the problem was not due to congestion.</p><p>In order to investigate further, we wrote a <a href="https://github.com/amnonbc/tcpping/blob/main/tcpping.go">simple program</a> which bounced TCP/IP messages between servers located in London and Singapore. Since we could reproduce the problem with our program, we were confident that the problem wasn't in our realtime messaging stack. This raised the question of whether our TCP stacks were mis-configured in some way to produce this behavior.</p><h2 id="investigating-tcp-ip-settings">Investigating TCP/IP settings</h2><h3 id="a-brief-overview-of-tcp-ip">A brief overview of TCP/IP</h3><p>TCP is a reliable protocol which runs over an unreliable IP layer. The IP layer can drop or reorder packets, and the TCP layer provides guaranteed delivery; the TCP receiver acknowledges receipt of packets, and the TCP sender retransmits packets if this acknowledgement is not received within a timeout window, called Receive Time Out (RTO). This RTO is set to double the RTT of the connection.<br></p><p>If the TCP sender had to wait for an acknowledgement of each packet before sending the next one, it would transmit data at a very slow rate, especially on connections with a large round trip time. To avoid this, TCP has a congestion control window, which specifies how many unacknowledged packets can be in flight at any given time. The size of this window is dynamically adjusted by the sender side to maintain an optimal size. If the window size is too small, then the bandwidth of the connection will be too small. But if it is too large, intermediate routers on the connection path will suffer congestion, and either drop the packets (leading to the need to retransmit), or delay the packets by queuing them in large buffers. <br></p><p>TCP can be sent across a wide variety of physical links, covering every combination of bandwidth, latency, and reliability. For this reason, TCP connections start off with a small window size and send a small number of packets. This beginning phase of starting with a small window size and ramping up is called the <em>slow start phase</em> of a TCP connection. When these initial packets are successfully acknowledged, they increase the window size until packet loss occurs or latency increases. This behavior discovers the optimal transmit rate for a given link. The rate of incoming acknowledgements effectively clocks the transmission of packets to an appropriate rate. This rate can change during the lifetime of a connection, for instance due to changes of routing of a link, or due to changes in congestion of a segment of the link.</p><h3 id="investigating-tcp-slow-start-after-idle">Investigating <code>tcp_slow_start_after_idle</code><br></h3><p>The Linux TCP stack has many knobs, which can be examined and controlled by files in <code>/proc/sys/net/ipv4/</code>. This directory contains 130 files. One which seemed relevant to the investigation was <code>tcp_slow_start_after_idle</code>, which is enabled by default.</p><p>The <code>tcp_slow_start_after_idle</code> setting in the TCP stack is <a href="https://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt">defined </a>as follows:</p><pre><code>tcp_slow_start_after_idle - BOOLEAN
If set, provide RFC2861 behavior and time out the congestion
window after an idle period. An idle period is defined at
the current RTO. If unset, the congestion window will not
be timed out after an idle period. Default: 1</code></pre><p>So after an idle period defined as two round trip times, the TCP congestion window will reset to slow start mode, in which only two packets are allowed to be in flight.</p><h2 id="the-solution-disabling-tcp-slow-start-after-idle">The solution: disabling <code>tcp_slow_start_after_idle</code></h2><p>In our case, the RTT from Europe to SE-Asia was 180ms, so the Receive Timeout (RTO), defined as twice the RTT, would be 360ms. Our customer did have bursty traffic with entire seconds when no data was transmitted. These silent periods were not interminably long in the scale of human history, but they were long enough for <code>tcp_slow_start_after_idle</code> to kick in. When the customer did transmit the next message, if this message was larger than the initial congestion window size, then the TCP stack would send the first two packets (about 3kb) and then wait for an ack to be received. This explained the delay of 180ms seen on some messages.<br></p><p>Adding a high frequency heartbeat to our connection - which sent a message every 20ms - meant the receiver would return regular ACK packets, which un-blocked the congestion window and allowed queued packets to be sent. The constant flow of packets meant that the connection was never idle for long enough for <code>tcp_slow_start_after_idle</code> to kick in.<br></p><p>To validate this, we turned off <code>tcp_slow_start_after_idle</code> on our sending machine:</p><pre><code>sysctl -w net.ipv4.tcp_slow_start_after_idle=0</code></pre><p>This led to an immediate decrease in our p99 times, from 358ms to 200ms, on our test TCPping program.<br></p><figure class="kg-card kg-image-card kg-card-hascaption"><img src="https://ik.imagekit.io/ably/ghost/prod/2024/07/Screenshot-2024-07-26-at-12.28.06.png" class="kg-image" alt="" loading="lazy" width="1376" height="718" srcset="https://ik.imagekit.io/ably/ghost/prod/2024/07/Screenshot-2024-07-26-at-12.28.06.png?tr=w-380,q-50 380w,
https://ik.imagekit.io/ably/ghost/prod/2024/07/Screenshot-2024-07-26-at-12.28.06.png?tr=w-760,q-50 760w,
https://ik.imagekit.io/ably/ghost/prod/2024/07/Screenshot-2024-07-26-at-12.28.06.png?tr=w-1520,q-50 1520w" sizes="
(min-width: 1520px) 1520px,
100vw
"><figcaption>TCPping message latency - tcp_slow_start_after_idle was turned off at 13:34</figcaption></figure><h3 id="disabling-tcp-slow-start-after-idle-in-the-container">Disabling <code>tcp_slow_start_after_idle</code> in the container</h3><p>However, our problems did not end there. To our surprise, disabling <code>tcp_slow_start_after_idle</code> on our machines did not remove the tail latencies in our production system, even though it worked for our TCPping benchmark. After some digging, we realized that the documentation for <code>tcp_slow_start_after_idle</code> setting in the Linux kernel was last updated in 2006 - two years before namespaces were added to the kernel. Docker containers are built on the functionality provided by these Linux namespaces, of which there are eight different types. One of these is the network namespace, which includes port assignments, routing tables, and its own private copy of network settings, such as <code>tcp_slow_start_after_idle</code>.</p><p><br>To disable <code>tcp_slow_start_after_idle</code> for a program running inside a container, you must disable it within the container itself, not on the host machine (the documentation does not mention this). If you are running your application inside a Docker container, you need to disable <code>tcp_slow_start_after_idle</code> within the container that runs your application. This can be done by writing <code>0</code> into the <code>/proc/sys/net/ipv4/tcp_slow_start_after_idle</code> file inside the container (if you have the appropriate permissions), or by using the <code>--sysctl</code> option when launching Docker. Once we launched our Docker containers with the appropriate option, the latency tails disappeared from our production system.</p><h3 id="a-note-on-observing-window-sizes-in-tcp-ip">A note on observing window sizes in TCP/IP</h3><p>We can observe the behavior in the previous sections directly by focusing on the congestion window.</p><p><code>tcpdump</code> is the standard workhorse tool for observing network behavior, but it displays the receive window advertised by the receiver. This window indicates how much buffer space the receiver has for incoming data, which differs from the congestion window we are interested in. The amount of data the sender can transmit is limited by the smaller of the receive window and the congestion window.</p><p>Our focus is on the congestion window, which is managed by the sender's TCP stack and not transmitted over the network, making it invisible to <code>tcpdump</code>. However, we can examine it using a relatively obscure <a href="https://man7.org/linux/man-pages/man8/ss.8.html">ss tool</a>:</p><pre><div class="vt-ghost-blog-code-block ui-theme-dark"><div class="hljs" data-id="code"><pre lang=bash><code class="language-bash highlight-code">ss -i</code></pre></div></div></pre><p>which produces output like:</p><pre><div class="vt-ghost-blog-code-block ui-theme-dark"><div class="hljs" data-id="code"><pre lang=bash><code class="language-bash highlight-code">tcp ESTAB 0 0 [::ffff:127.0.0.1]:9302 [::ffff:127.0.0.1]:59210
cubic wscale:7,7 rto:204 rtt:0.023/0.015 ato:40 mss:32768 pmtu:65535 rcvmss:536 advmss:65483 cwnd:10 bytes_sent:72071646 bytes_acked:72071646 bytes_received:1320690 segs_out:21421 segs_in:27884 data_segs_out:20930 data_segs_in:6951 send 113975.7Mbps lastsnd:4764 lastrcv:4800 lastack:4764 pacing_rate 225500.2Mbps delivery_rate 65536.0Mbps delivered:20931 app_limited busy:584ms rcv_rtt:343162 rcv_space:65740 rcv_ssthresh:1178436 minrtt:0.005</code></pre></div></div></pre><p>The congestion window size is calculated with the formula <code>cwnd * 2^wscale</code> . In the output above, notice that <code>wscale</code> equals <code>7</code> , and <code>cwnd</code> equals <code>10</code>. This comes out to <code>10 x 2^7</code> according to the formula, equal to 1280 bytes. </p><p>With <code>tcp_slow_start_after_idle enabled</code>, <code>cwnd</code> collapsed to 1280 bytes - one packet, when the connection went idle. After we disabled <code>tcp_slow_start_after_idle</code>, <code>cwnd</code> increased to a larger number when the connection was established, and stayed there.</p><h2 id="is-the-default-setting-for-tcp-slow-start-after-idle-wrong">Is the default setting for <code>tcp_slow_start_after_idle</code> wrong?</h2><figure class="kg-card kg-image-card kg-card-hascaption"><img src="https://ik.imagekit.io/ably/ghost/prod/2024/07/Screenshot-2024-07-26-at-12.46.40.png" class="kg-image" alt="" loading="lazy" width="388" height="648" srcset="https://ik.imagekit.io/ably/ghost/prod/2024/07/Screenshot-2024-07-26-at-12.46.40.png?tr=w-380,q-50 380w,
https://ik.imagekit.io/ably/ghost/prod/2024/07/Screenshot-2024-07-26-at-12.46.40.png?tr=w-760,q-50 760w,
https://ik.imagekit.io/ably/ghost/prod/2024/07/Screenshot-2024-07-26-at-12.46.40.png?tr=w-1520,q-50 1520w" sizes="
(min-width: 1520px) 1520px,
100vw
"><figcaption>A riff on the original <a href="https://xkcd.com/619/">xkcd</a> comic :)</figcaption></figure><p>There was a certain degree of surprise from even some of our most experienced developers that the kernel's TCP stack delays packets after the connection becomes idle, and that the idle period timeout was so short.</p><p><code>tcp_slow_start_after_idle</code> was introduced by Internet pioneer <a href="https://www.wired.com/2012/05/van-jacobson/">Van Jacobson</a> in 1986 in response to severe congestion on the ARPAnet. During this period of dramatic growth, the network experienced a number of "congestion collapse" incidents, where large numbers of dropped packets led to TCP retransmits dominating internet traffic, effectively halting data transmission. In his 1988 paper <a href="https://ee.lbl.gov/papers/congavoid.pdf">Congestion Avoidance and Control</a> Van Jacobson describes seven algorithms they added to the BSD TCP/IP stack (on which most other TCP stacks are based), to avoid these meltdowns. One of these was the slow-start phase of congestion control, which was used to clock the rate of packet transmission. When a connection became idle for more than two round trip times (RTTs), all ACK packets would have been received, and the congestion window would be emptied. If the sender resumed transmission, it could potentially send a full window of packets simultaneously, potentially overwhelming intermediate links in the connection. To prevent this, the TCP stack was given the default behavior to revert to slow start whenever the connection was idle for two RTTs, which ensures that a large number of packets are not simultaneously dumped into the connection in a way which could overwhelm intermediate routers.</p><p>This and other algorithms successfully prevented "congestion collapse," but they also introduced increased latency for bursty applications and unnecessary delays in situations where the network had sufficient capacity. Consequently, system administrators were given the option to disable <code>tcp_slow_start_after_idle</code> when appropriate.</p><p>In our case where a customer's message transmission was bursty, and the network capacity was sufficient to handle these message bursts at the sender's line rate, then the default behavior did cause unacceptable delays for a realtime application.</p><h2 id="conclusion">Conclusion</h2><p>In 2024, the Internet has evolved into an essential part of daily life. We expect it to “just work”, and often take for granted that transmitted packets will appear at the other end without issue. However, when low latency is critical, as in our case with realtime messaging, it's sometimes necessary to delve into the intricacies of TCP/IP configurations.</p><p><br>Solving customer problems sometimes leads us to interesting journeys into the history of Internet infrastructure and the Linux networking stack on which much of it is based. In this particular case, our investigation revealed that the default TCP setting, tcp_slow_start_after_idle, while historically effective in preventing network congestion, was causing significant delays for our customer's bursty traffic. By disabling this setting within Docker containers, we successfully reduced latency variance and improved message delivery times.</p><p>This experience underscores the importance of understanding and optimizing network stack configurations tailored to specific use cases. Our journey highlighted that sometimes, the solution lies beyond our immediate software stack, requiring a deep dive into the foundational technologies that support our applications.</p><p>As software engineers, we naturally tend to look for solutions to problems in the software we control and are most familiar with. But sometimes we need to look further afield and be ready to explore and adjust the underlying settings that can impact our systems in less obvious ways.</p><p>TCP/IP has generally stood the test of time well. It works well on links with bandwidth ranging from 1200 bps packet radio links, to modern 400 GB network backbones, and with latencies spanning microseconds on LANs, to hundreds of milliseconds for satellite links. But designing for such a wide range of environments does inevitably mean compromises and trade-offs which are not always suitable for our particular use case, especially when latency is of prime importance.</p></div><aside class="vt-blog-editorial-tryit" aria-label="Try Ably"><div class="vt-blog-editorial-tryit-rule"></div><div><div class="vt-blog-editorial-tryit-kicker">Try it</div><h3 class="vt-blog-editorial-tryit-heading">Build dependable realtime in minutes.</h3><p class="vt-blog-editorial-tryit-body">Ably runs the same patterns we write about — in production, at scale. Free tier includes 6M messages a month, no card required.</p><div class="vt-blog-editorial-tryit-actions"><a href="https://ably.com/signup/?utm_source=blog&amp;utm_medium=oc&amp;utm_campaign=tryit" class="vt-blog-editorial-tryit-cta">Start building <span aria-hidden="true">→</span></a><a href="https://ably.com/docs" class="vt-blog-editorial-tryit-link">Read the docs</a></div></div></aside><section class="vt-blog-editorial-newsletter" aria-labelledby="newsletter-heading"><div class="vt-blog-editorial-newsletter-text"><div class="vt-blog-editorial-newsletter-kicker"><span class="vt-blog-editorial-orange-rule" aria-hidden="true"></span><span>Subscribe</span></div><h3 id="newsletter-heading" class="vt-blog-editorial-newsletter-heading">New posts from the Ably team, monthly.</h3></div><form class="vt-blog-editorial-newsletter-form"><input type="email" name="EMAIL" required="" placeholder="you@company.com" class="vt-blog-editorial-newsletter-input" aria-label="Email address" value=""/><button type="submit" disabled="" class="vt-blog-editorial-newsletter-cta">Subscribe</button></form></section></article><aside class="vt-blog-editorial-rail" aria-label="Article tools"><nav aria-label="Table of contents"><div class="vt-blog-editorial-rail-label">On this page</div><ol class="vt-blog-editorial-toc-list"><li><a href="#initial-measurements" class="vt-blog-editorial-toc-link ">Initial measurements</a></li><li><a href="#investigating-tcp-ip-settings" class="vt-blog-editorial-toc-link ">Investigating TCP/IP settings</a></li><li><a href="#the-solution-disabling-tcp-slow-start-after-idle" class="vt-blog-editorial-toc-link ">The solution: disabling tcp_slow_start_after_idle</a></li><li><a href="#is-the-default-setting-for-tcp-slow-start-after-idle-wrong" class="vt-blog-editorial-toc-link ">Is the default setting for tcp_slow_start_after_idle wrong?</a></li><li><a href="#conclusion" class="vt-blog-editorial-toc-link ">Conclusion</a></li></ol></nav><div class="vt-blog-editorial-share" role="group" aria-labelledby="share-rail-label"><div id="share-rail-label" class="vt-blog-editorial-rail-label">Share</div><div class="vt-blog-editorial-share-list"><a href="https://twitter.com/intent/tweet?url=https%3A%2F%2Fably.com%2Fblog%2Foptimizing-global-message-transit-latency-a-journey-through-tcp-configuration&amp;text=Optimizing%20global%20message%20transit%20latency%3A%20a%20journey%20through%20TCP%20configuration" target="_blank" rel="noopener noreferrer" class="vt-blog-editorial-share-link" aria-label="Share on Twitter / X"><span class="vt-blog-editorial-share-glyph" aria-hidden="true"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="" style="width:0.875rem;height:0.875rem"><path fill="#03020D" d="M17.729 2h3.257l-7.116 8.133L22.24 21.2h-6.554l-5.134-6.712L4.679 21.2h-3.26l7.612-8.699L1 2h6.721l4.64 6.135zm-1.143 17.25h1.804L6.74 3.847H4.804z"></path></svg></span><span class="vt-blog-editorial-share-label">Twitter / X</span></a><a href="https://www.linkedin.com/sharing/share-offsite/?url=https%3A%2F%2Fably.com%2Fblog%2Foptimizing-global-message-transit-latency-a-journey-through-tcp-configuration" target="_blank" rel="noopener noreferrer" class="vt-blog-editorial-share-link" aria-label="Share on LinkedIn"><span class="vt-blog-editorial-share-glyph" aria-hidden="true"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="" style="width:0.875rem;height:0.875rem"><path fill="#1269BF" d="M24 1.765v20.47A1.764 1.764 0 0 1 22.235 24H1.765A1.765 1.765 0 0 1 0 22.235V1.765A1.765 1.765 0 0 1 1.765 0h20.47A1.765 1.765 0 0 1 24 1.765M7.059 9.176h-3.53v11.295h3.53zm.317-3.882a2.033 2.033 0 0 0-2.018-2.047h-.064a2.047 2.047 0 1 0 0 4.094 2.033 2.033 0 0 0 2.082-1.983zm13.095 8.315c0-3.395-2.16-4.715-4.306-4.715a4.02 4.02 0 0 0-3.572 1.821h-.099V9.176H9.176v11.295h3.53v-6.007a2.344 2.344 0 0 1 2.117-2.527h.135c1.122 0 1.955.705 1.955 2.484v6.05h3.53z"></path></svg></span><span class="vt-blog-editorial-share-label">LinkedIn</span></a><a href="https://news.ycombinator.com/submitlink?u=https%3A%2F%2Fably.com%2Fblog%2Foptimizing-global-message-transit-latency-a-journey-through-tcp-configuration&amp;t=Optimizing%20global%20message%20transit%20latency%3A%20a%20journey%20through%20TCP%20configuration" target="_blank" rel="noopener noreferrer" class="vt-blog-editorial-share-link" aria-label="Submit to Hacker News"><span class="vt-blog-editorial-share-glyph" aria-hidden="true"><span class="vt-blog-editorial-share-hn-mark">Y</span></span><span class="vt-blog-editorial-share-label">Hacker News</span></a><button type="button" class="vt-blog-editorial-share-link"><span class="vt-blog-editorial-share-glyph" aria-hidden="true"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" aria-hidden="true" data-slot="icon" class="" style="width:0.875rem;height:0.875rem"><path stroke-linecap="round" stroke-linejoin="round" d="M15.666 3.888A2.25 2.25 0 0 0 13.5 2.25h-3c-1.03 0-1.9.693-2.166 1.638m7.332 0c.055.194.084.4.084.612v0a.75.75 0 0 1-.75.75H9a.75.75 0 0 1-.75-.75v0c0-.212.03-.418.084-.612m7.332 0c.646.049 1.288.11 1.927.184 1.1.128 1.907 1.077 1.907 2.185V19.5a2.25 2.25 0 0 1-2.25 2.25H6.75A2.25 2.25 0 0 1 4.5 19.5V6.257c0-1.108.806-2.057 1.907-2.185a48.208 48.208 0 0 1 1.927-.184"></path></svg></span><span class="vt-blog-editorial-share-label" aria-live="polite">Copy link</span></button></div></div></aside></div><section class="vt-blog-editorial-continue" aria-labelledby="continue-reading-heading"><div class="vt-blog-editorial-continue-inner"><div class="vt-blog-editorial-continue-head"><h2 id="continue-reading-heading" class="vt-blog-editorial-continue-h3">Continue reading</h2><a href="/blog" class="vt-blog-editorial-continue-all">All posts <span aria-hidden="true">→</span></a></div><div class="vt-blog-editorial-continue-grid"><a href="/blog/how-to-connect-to-ably-directly-part-2" class="vt-blog-editorial-tile" data-testid="post-tile-how-to-connect-to-ably-directly-part-2"><div class="vt-blog-editorial-tile-image"><img src="https://ik.imagekit.io/ably/ghost/prod/2021/08/how-to-connect-to-ably-yourself@2x-1.png?tr=w-768,q-50" alt="How to connect to Ably directly (and why you probably shouldn&#x27;t) – Part 2" width="768" class="w-full h-full object-cover" loading="lazy"/></div><div><div class="vt-blog-editorial-tile-tag"><span class="vt-blog-editorial-badge">Ably engineering</span></div><h3 class="vt-blog-editorial-tile-title">How to connect to Ably directly (and why you probably shouldn&#x27;t) – Part 2</h3><div class="vt-blog-editorial-tile-meta"><span class="vt-blog-editorial-tile-meta-author">Owen Pearson</span><span class="vt-blog-editorial-meta-dot">·</span><span class="vt-blog-editorial-tile-meta-mono">Sep 7, 2021</span></div></div></a><a href="/blog/aws-vpc-peering-vs-transit-gateway-and-beyond" class="vt-blog-editorial-tile" data-testid="post-tile-aws-vpc-peering-vs-transit-gateway-and-beyond"><div class="vt-blog-editorial-tile-image"><img src="https://ik.imagekit.io/ably/ghost/prod/2022/09/designing-the-next-global-ably-network.png?tr=w-768,q-50" alt="VPC peering vs Transit Gateway and beyond: Key choices in AWS network design" width="768" class="w-full h-full object-cover" loading="lazy"/></div><div><div class="vt-blog-editorial-tile-tag"><span class="vt-blog-editorial-badge">Ably engineering</span></div><h3 class="vt-blog-editorial-tile-title">VPC peering vs Transit Gateway and beyond: Key choices in AWS network design</h3><div class="vt-blog-editorial-tile-meta"><span class="vt-blog-editorial-tile-meta-author">Huw McNamara</span><span class="vt-blog-editorial-meta-dot">·</span><span class="vt-blog-editorial-tile-meta-mono">Sep 13, 2022</span></div></div></a><a href="/blog/crdts-distributed-data-consistency-challenges" class="vt-blog-editorial-tile" data-testid="post-tile-crdts-distributed-data-consistency-challenges"><div class="vt-blog-editorial-tile-image"><img src="https://ik.imagekit.io/ably/ghost/prod/2021/10/blog-crdts-deep-dive.jpg?tr=w-768,q-50" alt="CRDTs solve distributed data consistency challenges" width="768" class="w-full h-full object-cover" loading="lazy"/></div><div><div class="vt-blog-editorial-tile-tag"><span class="vt-blog-editorial-badge">Ably engineering</span></div><h3 class="vt-blog-editorial-tile-title">CRDTs solve distributed data consistency challenges</h3><div class="vt-blog-editorial-tile-meta"><span class="vt-blog-editorial-tile-meta-author">Jo Stichbury</span><span class="vt-blog-editorial-meta-dot">·</span><span class="vt-blog-editorial-tile-meta-mono">Oct 28, 2021</span></div></div></a></div></div></section></main><footer class="w-full bg-neutral-100 dark:bg-neutral-1200 border-t border-neutral-300 dark:border-neutral-1000" data-id="footer"><div class="max-w-screen-xl mx-auto ui-grid-px pt-10 sm:pt-12 md:pt-16 pb-10"><div class="flex flex-col sm:flex-row gap-x-6 gap-y-12 mb-16 justify-between"><div class="flex-1 flex flex-col gap-6"><a href="/" class="items-center gap-2 justify-center focus-base rounded w-[6.375rem] flex dark:hidden"><img src="https://voltaire.ably.com/static/ably-logo-200721285a51085f43e8a849a85667bc.svg" width="96px" alt="Ably logo"/></a><a href="/" class="items-center gap-2 justify-center focus-base rounded w-[6.375rem] hidden dark:flex"><img src="https://voltaire.ably.com/static/ably-logo-white-749ef5fc6551d0e3b4940c2e3e42463f.svg" width="96px" alt="Ably logo"/></a><a href="https://status.ably.com" class="inline-flex group/status items-center gap-2" target="_blank" rel="noreferrer"><span class="inline-flex h-2 aspect-square rounded-full bg-neutral-500 animate-pulse"></span></a><div class="flex gap-x-6"><a href="https://x.com/ablyrealtime" target="_blank" rel="noreferrer noopener" aria-label="Visit Ably on x" class="w-5 h-5 flex group/social-icon"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="text-neutral-1000 dark:text-neutral-300 group-hover/social-icon:hidden" style="width:20px;height:20px"><path fill="currentColor" d="M17.729 2h3.257l-7.116 8.133L22.24 21.2h-6.554l-5.134-6.712L4.679 21.2h-3.26l7.612-8.699L1 2h6.721l4.64 6.135zm-1.143 17.25h1.804L6.74 3.847H4.804z"></path></svg><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="hidden group-hover/social-icon:flex" style="width:20px;height:20px"><path fill="#03020D" d="M17.729 2h3.257l-7.116 8.133L22.24 21.2h-6.554l-5.134-6.712L4.679 21.2h-3.26l7.612-8.699L1 2h6.721l4.64 6.135zm-1.143 17.25h1.804L6.74 3.847H4.804z"></path></svg></a><a href="https://www.linkedin.com/company/ably-realtime" target="_blank" rel="noreferrer noopener" aria-label="Visit Ably on linkedin" class="w-5 h-5 flex group/social-icon"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="text-neutral-1000 dark:text-neutral-300 group-hover/social-icon:hidden" style="width:20px;height:20px"><path fill="currentColor" d="M24 1.765v20.47A1.764 1.764 0 0 1 22.235 24H1.765A1.765 1.765 0 0 1 0 22.235V1.765A1.765 1.765 0 0 1 1.765 0h20.47A1.765 1.765 0 0 1 24 1.765M7.059 9.176h-3.53v11.295h3.53zm.317-3.882a2.033 2.033 0 0 0-2.018-2.047h-.064a2.047 2.047 0 1 0 0 4.094 2.033 2.033 0 0 0 2.082-1.983zm13.095 8.315c0-3.395-2.16-4.715-4.306-4.715a4.02 4.02 0 0 0-3.572 1.821h-.099V9.176H9.176v11.295h3.53v-6.007a2.344 2.344 0 0 1 2.117-2.527h.135c1.122 0 1.955.705 1.955 2.484v6.05h3.53z"></path></svg><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="hidden group-hover/social-icon:flex" style="width:20px;height:20px"><path fill="#1269BF" d="M24 1.765v20.47A1.764 1.764 0 0 1 22.235 24H1.765A1.765 1.765 0 0 1 0 22.235V1.765A1.765 1.765 0 0 1 1.765 0h20.47A1.765 1.765 0 0 1 24 1.765M7.059 9.176h-3.53v11.295h3.53zm.317-3.882a2.033 2.033 0 0 0-2.018-2.047h-.064a2.047 2.047 0 1 0 0 4.094 2.033 2.033 0 0 0 2.082-1.983zm13.095 8.315c0-3.395-2.16-4.715-4.306-4.715a4.02 4.02 0 0 0-3.572 1.821h-.099V9.176H9.176v11.295h3.53v-6.007a2.344 2.344 0 0 1 2.117-2.527h.135c1.122 0 1.955.705 1.955 2.484v6.05h3.53z"></path></svg></a><a href="https://github.com/ably/" target="_blank" rel="noreferrer noopener" aria-label="Visit Ably on github" class="w-5 h-5 flex group/social-icon"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="text-neutral-1000 dark:text-neutral-300 group-hover/social-icon:hidden" style="width:20px;height:20px"><path fill="currentColor" fill-rule="evenodd" d="M12 0C5.4 0 0 5.42 0 12.043c0 5.32 3.4 9.836 8.2 11.441.6.1.8-.3.8-.602v-2.007c-3.3.702-4-1.606-4-1.606-.5-1.405-1.3-1.806-1.3-1.806-1.1-.703.1-.703.1-.703 1.2.1 1.8 1.204 1.8 1.204 1.1 1.807 2.8 1.305 3.5 1.004.1-.803.4-1.305.8-1.606-2.8-.3-5.6-1.304-5.6-5.92 0-1.306.5-2.41 1.2-3.212 0-.402-.5-1.606.2-3.212 0 0 1-.301 3.3 1.204 1-.3 2-.401 3-.401s2 .1 3 .401c2.3-1.505 3.3-1.204 3.3-1.204.7 1.706.2 2.91.1 3.212.8.802 1.2 1.906 1.2 3.211 0 4.617-2.8 5.62-5.5 5.921.4.402.8 1.104.8 2.208v3.312c0 .301.2.703.8.602 4.9-1.606 8.3-6.122 8.3-11.44C24 5.418 18.6 0 12 0" clip-rule="evenodd"></path></svg><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="hidden group-hover/social-icon:flex" style="width:20px;height:20px"><path fill="#191717" fill-rule="evenodd" d="M12 0C5.4 0 0 5.42 0 12.043c0 5.32 3.4 9.836 8.2 11.441.6.1.8-.3.8-.602v-2.007c-3.3.702-4-1.606-4-1.606-.5-1.405-1.3-1.806-1.3-1.806-1.1-.703.1-.703.1-.703 1.2.1 1.8 1.204 1.8 1.204 1.1 1.807 2.8 1.305 3.5 1.004.1-.803.4-1.305.8-1.606-2.8-.3-5.6-1.304-5.6-5.92 0-1.306.5-2.41 1.2-3.212 0-.402-.5-1.606.2-3.212 0 0 1-.301 3.3 1.204 1-.3 2-.401 3-.401s2 .1 3 .401c2.3-1.505 3.3-1.204 3.3-1.204.7 1.706.2 2.91.1 3.212.8.802 1.2 1.906 1.2 3.211 0 4.617-2.8 5.62-5.5 5.921.4.402.8 1.104.8 2.208v3.312c0 .301.2.703.8.602 4.9-1.606 8.3-6.122 8.3-11.44C24 5.418 18.6 0 12 0" clip-rule="evenodd"></path></svg></a><a href="https://discord.gg/g8yqePUVDn" target="_blank" rel="noreferrer noopener" aria-label="Visit Ably on discord" class="w-5 h-5 flex group/social-icon"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="text-neutral-1000 dark:text-neutral-300 group-hover/social-icon:hidden" style="width:20px;height:20px"><path fill="currentColor" d="M20.317 4.607a19.6 19.6 0 0 0-4.885-1.542.074.074 0 0 0-.079.038c-.21.382-.444.88-.608 1.271a18 18 0 0 0-5.487 0c-.163-.4-.406-.89-.617-1.271a.08.08 0 0 0-.079-.038c-1.714.3-3.354.827-4.885 1.542a.07.07 0 0 0-.032.028C.533 9.364-.32 13.976.099 18.532a.08.08 0 0 0 .031.057 19.8 19.8 0 0 0 5.993 3.082.08.08 0 0 0 .084-.028c.462-.642.874-1.318 1.226-2.03a.08.08 0 0 0-.041-.107A13 13 0 0 1 5.52 18.6a.08.08 0 0 1-.008-.13q.19-.145.372-.297a.07.07 0 0 1 .078-.01c3.927 1.824 8.18 1.824 12.061 0a.07.07 0 0 1 .079.01q.18.15.372.297a.08.08 0 0 1-.006.13q-.895.531-1.873.906a.08.08 0 0 0-.041.109c.36.71.772 1.386 1.225 2.028a.075.075 0 0 0 .084.029 19.7 19.7 0 0 0 6.002-3.082.08.08 0 0 0 .032-.056c.5-5.267-.838-9.842-3.549-13.897a.06.06 0 0 0-.031-.03M8.02 15.757c-1.182 0-2.157-1.104-2.157-2.46s.956-2.46 2.157-2.46c1.21 0 2.176 1.113 2.157 2.46 0 1.356-.956 2.46-2.157 2.46m7.975 0c-1.183 0-2.157-1.104-2.157-2.46s.955-2.46 2.157-2.46c1.21 0 2.176 1.113 2.157 2.46 0 1.356-.946 2.46-2.157 2.46"></path></svg><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="hidden group-hover/social-icon:flex" style="width:20px;height:20px"><path fill="#5B64EA" d="M20.317 4.607a19.6 19.6 0 0 0-4.885-1.542.074.074 0 0 0-.079.038c-.21.382-.444.88-.608 1.271a18 18 0 0 0-5.487 0c-.163-.4-.406-.89-.617-1.271a.08.08 0 0 0-.079-.038c-1.714.3-3.354.827-4.885 1.542a.07.07 0 0 0-.032.028C.533 9.364-.32 13.976.099 18.532a.08.08 0 0 0 .031.057 19.8 19.8 0 0 0 5.993 3.082.08.08 0 0 0 .084-.028c.462-.642.874-1.318 1.226-2.03a.08.08 0 0 0-.041-.107A13 13 0 0 1 5.52 18.6a.08.08 0 0 1-.008-.13q.19-.145.372-.297a.07.07 0 0 1 .078-.01c3.927 1.824 8.18 1.824 12.061 0a.07.07 0 0 1 .079.01q.18.15.372.297a.08.08 0 0 1-.006.13q-.895.531-1.873.906a.08.08 0 0 0-.041.109c.36.71.772 1.386 1.225 2.028a.075.075 0 0 0 .084.029 19.7 19.7 0 0 0 6.002-3.082.08.08 0 0 0 .032-.056c.5-5.267-.838-9.842-3.549-13.897a.06.06 0 0 0-.031-.03M8.02 15.757c-1.182 0-2.157-1.104-2.157-2.46s.956-2.46 2.157-2.46c1.21 0 2.176 1.113 2.157 2.46 0 1.356-.956 2.46-2.157 2.46m7.975 0c-1.183 0-2.157-1.104-2.157-2.46s.955-2.46 2.157-2.46c1.21 0 2.176 1.113 2.157 2.46 0 1.356-.946 2.46-2.157 2.46"></path></svg></a><a href="https://www.youtube.com/c/AblyRealtime" target="_blank" rel="noreferrer noopener" aria-label="Visit Ably on youtube" class="w-5 h-5 flex group/social-icon"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="text-neutral-1000 dark:text-neutral-300 group-hover/social-icon:hidden" style="width:20px;height:20px"><path fill="currentColor" d="M23.499 6.631a3.01 3.01 0 0 0-2.122-2.128C19.505 4 12 4 12 4s-7.505 0-9.377.503A3.01 3.01 0 0 0 .502 6.631C0 8.51 0 12.425 0 12.425s0 3.917.502 5.795a3.01 3.01 0 0 0 2.121 2.128c1.872.503 9.377.503 9.377.503s7.505 0 9.377-.503a3.01 3.01 0 0 0 2.122-2.128C24 16.342 24 12.426 24 12.426s0-3.917-.502-5.795"></path><path fill="#fff" d="m9.545 15.982 6.273-3.556-6.273-3.557z"></path></svg><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="hidden group-hover/social-icon:flex" style="width:20px;height:20px"><path fill="#ED1D24" d="M23.499 6.131a3.01 3.01 0 0 0-2.122-2.128C19.505 3.5 12 3.5 12 3.5s-7.505 0-9.377.503A3.01 3.01 0 0 0 .502 6.131C0 8.01 0 11.925 0 11.925s0 3.917.502 5.795a3.01 3.01 0 0 0 2.121 2.128c1.872.503 9.377.503 9.377.503s7.505 0 9.377-.503a3.01 3.01 0 0 0 2.122-2.128C24 15.842 24 11.926 24 11.926s0-3.917-.502-5.795"></path><path fill="#fff" d="m9.545 15.482 6.273-3.556-6.273-3.557z"></path></svg></a></div><div class="flex flex-wrap gap-2 mt-4 max-w-full"><img src="https://voltaire.ably.com/static/g2-best-meets-requirements-spring-2025-7a9232ad72544eec2aa2918866eb3f7e.svg" alt="G2 Best Meets Requirements Spring 2025" width="55" height="63"/><img src="https://voltaire.ably.com/static/g2-best-support-spring-2025-25e3fb42507694fe41661b0ae587227e.svg" alt="G2 Best Support Spring 2025" width="55" height="63"/><img src="https://voltaire.ably.com/static/g2-users-most-likely-to-recommend-spring-2025-ada1333537cf17600c6f72e11b673838.svg" alt="G2 Users Most Likely to Recommend Spring 2025" width="55" height="63"/><img src="https://voltaire.ably.com/static/g2-easiest-to-use-spring-2025-001641898e826e06845da9c5a7fdca0d.svg" alt="G2 Easiest to Use Spring 2025" width="55" height="63"/></div></div><div class="flex-1 md:flex-[2] flex flex-row flex-wrap gap-x-6 gap-y-12"><div class="flex-1 basis-1/3 md:basis-1"><h3 class="ui-text-overline2 text-neutral-700 dark:text-neutral-600 mb-4">Products</h3><ul class="flex flex-col gap-y-3"><li class="flex gap-x-2"><a href="/pubsub" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably Pub/Sub">Ably Pub/Sub</a></li><li class="flex gap-x-2"><a href="/chat" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably Chat">Ably Chat</a></li><li class="flex gap-x-2"><a href="/ai-transport" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably AI Transport">Ably AI Transport</a></li><li class="flex gap-x-2"><a href="/liveobjects" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably LiveObjects">Ably LiveObjects</a></li><li class="flex gap-x-2"><a href="/spaces" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably Spaces">Ably Spaces</a></li><li class="flex gap-x-2"><a href="/livesync" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Ably LiveSync">Ably LiveSync</a></li><li class="flex gap-x-2"><a href="/compare" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Compare our tech">Compare our tech</a></li></ul></div><div class="flex-1 basis-1/3 md:basis-1"><h3 class="ui-text-overline2 text-neutral-700 dark:text-neutral-600 mb-4">Platform</h3><ul class="flex flex-col gap-y-3"><li class="flex gap-x-2"><a href="/platform" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Architecture">Architecture</a></li><li class="flex gap-x-2"><a href="/docs/platform/integrations" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Integrations">Integrations</a></li><li class="flex gap-x-2"><a href="/docs/sdks" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit SDKs">SDKs</a></li><li class="flex gap-x-2"><a href="https://changelog.ably.com/" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Changelog">Changelog</a></li><li class="flex gap-x-2"><a href="/security-and-compliance" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Security &amp; Compliance">Security &amp; Compliance</a></li></ul></div><div class="flex-1 basis-1/3 md:basis-1"><h3 class="ui-text-overline2 text-neutral-700 dark:text-neutral-600 mb-4">Get started</h3><ul class="flex flex-col gap-y-3"><li class="flex gap-x-2"><a href="/docs" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Documentation">Documentation</a></li><li class="flex gap-x-2"><a href="/examples" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Examples">Examples</a></li><li class="flex gap-x-2"><a href="/pricing" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Pricing">Pricing</a></li><li class="flex gap-x-2"><a href="/topics" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Realtime A-Z">Realtime A-Z</a></li><li class="flex gap-x-2"><a href="/support" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Support">Support</a></li></ul></div><div class="flex-1 basis-1/3 md:basis-1"><h3 class="ui-text-overline2 text-neutral-700 dark:text-neutral-600 mb-4">Company</h3><ul class="flex flex-col gap-y-3"><li class="flex gap-x-2"><a href="/about" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit About Ably">About Ably</a></li><li class="flex gap-x-2"><a href="/blog" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Blog">Blog</a></li><li class="flex gap-x-2"><a href="/careers" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Careers">Careers</a><div class="inline-flex bg-neutral-100 dark:bg-neutral-1200 rounded-2xl gap-1 items-center focus-base transition-colors select-none px-2 py-0 text-[10px] leading-tight text-neutral-900 dark:text-neutral-400 ui-text-p4 font-[10px]"><span class="whitespace-nowrap tracking-[0.04em] leading-[18px]">WE’RE HIRING</span></div></li><li class="flex gap-x-2"><a href="/contact" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Contact us">Contact us</a></li></ul></div></div></div><div class="pt-6 border-t border-neutral-300 dark:border-neutral-1000"><div class="flex gap-6"><a href="/data-protection" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Data protection">Data protection</a><a href="/privacy" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Privacy">Privacy</a><a href="/legals" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Legals">Legals</a><a href="/privacy" class="ui-text-label3 font-medium transition-colors text-neutral-1000 dark:text-neutral-300 hover:text-neutral-1300 hover:dark:text-neutral-000 active:text-neutral-800 active:dark:text-neutral-400 focus:outline focus:outline-gui-focus" aria-label="Visit Cookies">Cookies</a></div></div></div></footer><div class="fixed transition-[bottom] left-0 p-8 z-50 bottom-[-100px]" data-testid="layout-scroll-to-top"><div class="relative"><a href="#" class="bg-white w-12 h-12 cursor-pointer overflow-hidden rounded border-2 border-mid-grey hover:border-active-orange transition-all duration-[400ms] block"><div class="w-full h-full transition-all duration-[400ms] md:hover:translate-y-[-100%]"><div class="w-full h-full flex flex-row items-center justify-center"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" aria-hidden="true" data-slot="icon" class="text-cool-black" style="width:1.5rem;height:1.5rem"><path stroke-linecap="round" stroke-linejoin="round" d="M8.25 6.75 12 3m0 0 3.75 3.75M12 3v18"></path></svg></div><div class="w-full h-full flex flex-row items-center justify-center"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" aria-hidden="true" data-slot="icon" class="text-cool-black" style="width:1.5rem;height:1.5rem"><path stroke-linecap="round" stroke-linejoin="round" d="M8.25 6.75 12 3m0 0 3.75 3.75M12 3v18"></path></svg></div></div></a></div></div></div><div id="gatsby-announcer" style="position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0" aria-live="assertive" aria-atomic="true"></div></div><script>
(function() {
// Apply initial theme to dark navigation elements
var darkNav = document.getElementById('meganav-theme-dark');
if (darkNav) {
var header = darkNav.querySelector('header');
if (header && !header.classList.contains('ui-theme-dark')) {
header.classList.add('ui-theme-dark');
}
}
var darkMain = document.getElementById('main-theme-dark');
if (darkMain && !darkMain.classList.contains('ui-theme-dark')) {
darkMain.classList.add('ui-theme-dark');
}
})();
</script><script id="gatsby-script-loader">/*<![CDATA[*/window.pagePath="/blog/optimizing-global-message-transit-latency-a-journey-through-tcp-configuration";/*]]>*/</script><!-- slice-start id="_gatsby-scripts-1" -->
<script
id="gatsby-chunk-mapping"
>
window.___chunkMapping="{\"app\":[\"/app-afd9e708c8f5048ea0db.js\"],\"component---src-pages-404-js\":[\"/component---src-pages-404-js-b6e8f7edd8dea77e0c57.js\"],\"component---src-pages-ai-transport-tsx\":[\"/component---src-pages-ai-transport-tsx-4026b54583210f8a2053.js\"],\"component---src-pages-chat-tsx\":[\"/component---src-pages-chat-tsx-b2bdc3f08ae11253c0e3.js\"],\"component---src-pages-contact-tsx\":[\"/component---src-pages-contact-tsx-dc89e20c2190ed888225.js\"],\"component---src-pages-customer-service-chat-tsx\":[\"/component---src-pages-customer-service-chat-tsx-48ef3c126214086f90be.js\"],\"component---src-pages-cx-tech-tsx\":[\"/component---src-pages-cx-tech-tsx-36c4ce5a367e2a78a600.js\"],\"component---src-pages-ed-tech-tsx\":[\"/component---src-pages-ed-tech-tsx-d37da5b8d8490a573dbd.js\"],\"component---src-pages-fan-engagement-tsx\":[\"/component---src-pages-fan-engagement-tsx-8a1c337ce51c17c42fb7.js\"],\"component---src-pages-fin-tech-tsx\":[\"/component---src-pages-fin-tech-tsx-569a7d8f0503b63d5dc2.js\"],\"component---src-pages-four-pillars-of-dependability-tsx\":[\"/component---src-pages-four-pillars-of-dependability-tsx-d7a6e0386690e0cc591b.js\"],\"component---src-pages-global-round-trip-latencies-tsx\":[\"/component---src-pages-global-round-trip-latencies-tsx-b68878661cea17e95b93.js\"],\"component---src-pages-group-chat-tsx\":[\"/component---src-pages-group-chat-tsx-ebd327d243327ee9f8c8.js\"],\"component---src-pages-health-tech-tsx\":[\"/component---src-pages-health-tech-tsx-35d7fa3532326d5e5c7f.js\"],\"component---src-pages-hipaa-compliant-chat-tsx\":[\"/component---src-pages-hipaa-compliant-chat-tsx-acbc2476763adc3c3443.js\"],\"component---src-pages-index-tsx\":[\"/component---src-pages-index-tsx-322545dbcb61532adb0d.js\"],\"component---src-pages-live-streaming-chat-tsx\":[\"/component---src-pages-live-streaming-chat-tsx-e5e76e4db0722d9207ca.js\"],\"component---src-pages-liveobjects-tsx\":[\"/component---src-pages-liveobjects-tsx-28aa815395acdc2ce82e.js\"],\"component---src-pages-livesync-tsx\":[\"/component---src-pages-livesync-tsx-70885b210a264e5141e5.js\"],\"component---src-pages-periodic-table-of-realtime-js\":[\"/component---src-pages-periodic-table-of-realtime-js-d61b7b51a3fc51b8db26.js\"],\"component---src-pages-pricing-contact-tsx\":[\"/component---src-pages-pricing-contact-tsx-6d358f3f151478534903.js\"],\"component---src-pages-pricing-tsx\":[\"/component---src-pages-pricing-tsx-0354dbd8801f5243ece3.js\"],\"component---src-pages-pubsub-tsx\":[\"/component---src-pages-pubsub-tsx-464d839be2211c173921.js\"],\"component---src-pages-react-tsx\":[\"/component---src-pages-react-tsx-87358a2709374aacf56e.js\"],\"component---src-pages-reference-guide-chat-tsx\":[\"/component---src-pages-reference-guide-chat-tsx-8e3b14932c19bcc1e464.js\"],\"component---src-pages-reference-guide-multiplayer-tsx\":[\"/component---src-pages-reference-guide-multiplayer-tsx-9c53536d0cc84d1f7db8.js\"],\"component---src-pages-spaces-tsx\":[\"/component---src-pages-spaces-tsx-c46f420684aae4fb56f5.js\"],\"component---src-pages-websocket-service-tsx\":[\"/component---src-pages-websocket-service-tsx-29bc4c2263ebd30de9ce.js\"],\"component---src-templates-ably-vs-competitors-tsx\":[\"/component---src-templates-ably-vs-competitors-tsx-052a2027d366b1271875.js\"],\"component---src-templates-alternatives-index-tsx\":[\"/component---src-templates-alternatives-index-tsx-935d9b574657fb06b2f7.js\"],\"component---src-templates-alternatives-tsx\":[\"/component---src-templates-alternatives-tsx-5fc8ac958760659bdf83.js\"],\"component---src-templates-business-user-business-user-landing-tsx\":[\"/component---src-templates-business-user-business-user-landing-tsx-a017f6d2e2f718a2eafb.js\"],\"component---src-templates-compare-ably-js\":[\"/component---src-templates-compare-ably-js-6c0f08a75c6b11b3f317.js\"],\"component---src-templates-compare-index-js\":[\"/component---src-templates-compare-index-js-88ec96d0ed1e663d23ca.js\"],\"component---src-templates-compare-v-1-js\":[\"/component---src-templates-compare-v-1-js-d9b82d0110acbb502264.js\"],\"component---src-templates-compare-v-2-tsx\":[\"/component---src-templates-compare-v-2-tsx-4e1cd6756c146c67224d.js\"],\"component---src-templates-customer-story-tsx\":[\"/component---src-templates-customer-story-tsx-c07c97ff42a331eb75ae.js\"],\"component---src-templates-downloadable-gated-resource-downloadable-gated-resource-tsx\":[\"/component---src-templates-downloadable-gated-resource-downloadable-gated-resource-tsx-91158f3c7c6f4d153cd2.js\"],\"component---src-templates-downloadable-gated-resource-non-gated-resources-tsx\":[\"/component---src-templates-downloadable-gated-resource-non-gated-resources-tsx-7f08268bbae47759ffba.js\"],\"component---src-templates-downloadable-gated-resource-thank-you-page-js\":[\"/component---src-templates-downloadable-gated-resource-thank-you-page-js-0d99d8fac0c6445d301a.js\"],\"component---src-templates-flexible-tsx\":[\"/component---src-templates-flexible-tsx-088d00c89eb5a0ed6bfe.js\"],\"component---src-templates-flexible-v-2-tsx\":[\"/component---src-templates-flexible-v-2-tsx-1a6ff88fd8c866a1c57a.js\"],\"component---src-templates-generic-index-tsx\":[\"/component---src-templates-generic-index-tsx-89128b31a07f898b23dc.js\"],\"component---src-templates-ghost-pages-blog-list-blog-author-category-tsx\":[\"/component---src-templates-ghost-pages-blog-list-blog-author-category-tsx-820582c83ba43ebd3312.js\"],\"component---src-templates-ghost-pages-blog-list-blog-author-tsx\":[\"/component---src-templates-ghost-pages-blog-list-blog-author-tsx-eb3d8eb8dd047249efec.js\"],\"component---src-templates-ghost-pages-blog-list-blog-category-tsx\":[\"/component---src-templates-ghost-pages-blog-list-blog-category-tsx-45a6de80e8d75cb9c029.js\"],\"component---src-templates-ghost-pages-blog-list-blog-index-tsx\":[\"/component---src-templates-ghost-pages-blog-list-blog-index-tsx-0c5b2b87757efc90b336.js\"],\"component---src-templates-ghost-pages-blog-post-tsx\":[\"/component---src-templates-ghost-pages-blog-post-tsx-18ffed0af8673bb36129.js\"],\"component---src-templates-migrate-from-js\":[\"/component---src-templates-migrate-from-js-c3c1db295b667b6a61d9.js\"],\"component---src-templates-page-index-page-index-tsx\":[\"/component---src-templates-page-index-page-index-tsx-98b69b9679ce876767c4.js\"],\"component---src-templates-solution-tsx\":[\"/component---src-templates-solution-tsx-d8b78e013cb19083a3c9.js\"],\"component---src-templates-topic-tsx\":[\"/component---src-templates-topic-tsx-f902ddfff741dba97b74.js\"],\"component---src-templates-topics-category-tsx\":[\"/component---src-templates-topics-category-tsx-1433f8d904055ca038df.js\"],\"component---src-templates-topics-tsx\":[\"/component---src-templates-topics-tsx-436fa51625e6743f0934.js\"]}";
</script>
<script>window.___webpackCompilationHash="bd450251d228ea1e62eb";</script><script src="https://voltaire.ably.com/webpack-runtime-835cc4d6ec8a5974e673.js" async></script><script src="https://voltaire.ably.com/framework-5e45d846ab16cc6f99a4.js" async></script><script src="https://voltaire.ably.com/550fdbb7-7d2088aeb95a473eadaf.js" async></script><script src="https://voltaire.ably.com/6c2eb795-43135b94c65e4e3f8c79.js" async></script><script src="https://voltaire.ably.com/67a3bf3c-098d4132adafb49f830f.js" async></script><script src="https://voltaire.ably.com/ce3a710f-71c60be4cf8d98c10377.js" async></script><script src="https://voltaire.ably.com/app-afd9e708c8f5048ea0db.js" async></script><!-- slice-end id="_gatsby-scripts-1" --></body></html>