Files
nexus/sreweekly/articles/419/04-building-application-reliability-on-top-of-infrastructure-unreliabilit.html
2026-09-12 17:23:01 +08:00

320 lines
123 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!doctype html>
<html lang="en">
<head>
<meta name="sentry-trace" content="e998c28883e2d75b5360e4722539cfeb-6a567b40c7ef114d-1"/>
<meta name="baggage" content="sentry-environment=production,sentry-release=dfc3f09000ba6ab71785fa3ffec7abf3be5e69fb,sentry-public_key=cc6a5395b63b8e0fbc289939feb5a883,sentry-trace_id=e998c28883e2d75b5360e4722539cfeb,sentry-transaction=GET%20%2F(site)%2Fblog%2F%5Bslug%5D,sentry-sampled=true,sentry-sample_rand=0.3268026290175423,sentry-sample_rate=1"/>
<meta charset="utf-8" />
<meta name="referrer" content="strict-origin" />
<link rel="icon" href="/favicon.ico" sizes="any" />
<link rel="icon" href="/favicon.svg" type="image/svg+xml" />
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png" />
<link rel="manifest" href="/site.webmanifest" />
<link rel="preconnect" href="https://images.ctfassets.net" />
<link
rel="preload"
href="/fonts/aeonik/Aeonik-Light.woff2"
as="font"
type="font/woff2"
crossorigin
/>
<meta name="viewport" content="width=device-width, initial-scale=1" />
<script>
window.dataLayer = window.dataLayer || [];
// Canonical gtag stub: it must push the `arguments` object (not an array),
// because gtag/Consent Mode command detection relies on it. An arrow
// function pushing a plain array is not recognized as a consent/js command.
function gtag() {
window.dataLayer.push(arguments);
}
window.gtag = gtag;
// Advanced Consent Mode ad-measurement tuning. Affects Google tags only
// (GA4 / Google Ads / Floodlight) and only while ad_storage is denied —
// no effect once ad consent is granted.
// - ads_data_redaction: redacts ad-click identifiers (gclid/dclid) from
// Google Ads pings under denial (they are NOT redacted automatically —
// only cookies are). Pure privacy tightening.
//
// `url_passthrough` is intentionally NOT set. It decorates in-site links
// with a `_gl` blob (the GA client/session id) whenever consent is denied,
// so a visitor who has DECLINED still gets identifiers appended to every
// internal URL — a poor privacy look we don't want. Trade-off: Google Ads
// click-through attribution is no longer carried across navigation for the
// denied cohort (Consent Mode modeling still applies). Do not re-add it
// without weighing that against the declined-visitor URL decoration.
gtag('set', 'ads_data_redaction', true);
gtag('consent', 'default', {
analytics_storage: 'denied',
ad_storage: 'denied',
ad_user_data: 'denied',
ad_personalization: 'denied',
});
// Synchronously read stored consent and update before GTM processes.
// No race condition — this runs in the same script block before GTM loads.
//
// Gate on the record SHAPE, not `decided`: opt-out regimes (us_opt_out)
// persist a default-granted state with `decided: false` (the notice still
// shows once), so gating on `decided` would leave Consent Mode denied on
// every reload until the Svelte app mounts and re-grants — contradicting
// the intended default-grant for opt-out regions. `localStorage.consent`
// is only ever written with real grant states, so any well-formed record
// is safe to restore here. Do not narrow this back to `consent.decided`.
try {
const consent = JSON.parse(localStorage.getItem('consent'));
if (
consent &&
typeof consent.analytics === 'boolean' &&
typeof consent.advertising === 'boolean'
) {
const doNotSell = JSON.parse(localStorage.getItem('do-not-sell'));
const analyticsState = consent.analytics ? 'granted' : 'denied';
const adState =
consent.advertising && !doNotSell ? 'granted' : 'denied';
gtag('consent', 'update', {
analytics_storage: analyticsState,
ad_storage: adState,
ad_user_data: adState,
ad_personalization: adState,
});
}
} catch (e) {
// Best-effort: if localStorage is unavailable or holds malformed JSON,
// leave the denied consent defaults in place rather than blocking render.
}
gtag('js', new Date());
</script>
<link href="../_app/immutable/entry/start.DHXlI2X5.js" rel="modulepreload">
<link href="../_app/immutable/chunks/5enqrggX.js" rel="modulepreload">
<link href="../_app/immutable/chunks/E9LP05Mw.js" rel="modulepreload">
<link href="../_app/immutable/chunks/BoqVLdTb.js" rel="modulepreload">
<link href="../_app/immutable/chunks/CFXj-7Wp.js" rel="modulepreload">
<link href="../_app/immutable/entry/app.CLIeNjXY.js" rel="modulepreload">
<link href="../_app/immutable/chunks/Bpnd7C4r.js" rel="modulepreload">
<link href="../_app/immutable/chunks/Cribp3eQ.js" rel="modulepreload">
<link href="../_app/immutable/chunks/iay7zgLl.js" rel="modulepreload">
<link href="../_app/immutable/chunks/CEeD8KjQ.js" rel="modulepreload">
<link href="../_app/immutable/chunks/C-h7eZjq.js" rel="modulepreload">
<link href="../_app/immutable/chunks/BGFkCP_v.js" rel="modulepreload">
<link href="../_app/immutable/nodes/0.ofPPxsxn.js" rel="modulepreload">
<link href="../_app/immutable/chunks/Bv0b7euL.js" rel="modulepreload">
<link href="../_app/immutable/chunks/BZBUDhCv.js" rel="modulepreload">
<link href="../_app/immutable/chunks/jYEVFyhM.js" rel="modulepreload">
<link href="../_app/immutable/chunks/DfQEPfaM.js" rel="modulepreload">
<link href="../_app/immutable/chunks/Bfa8ybdf.js" rel="modulepreload">
<link href="../_app/immutable/nodes/7.fr6wgwU3.js" rel="modulepreload">
<link href="../_app/immutable/chunks/DZUyXmLw2.js" rel="modulepreload">
<link href="../_app/immutable/chunks/D6wFa1bO.js" rel="modulepreload">
<link href="../_app/immutable/chunks/Buj3FxJi.js" rel="modulepreload">
<link href="../_app/immutable/chunks/D-6oxDHN.js" rel="modulepreload">
<link href="../_app/immutable/chunks/C3V59U9V.js" rel="modulepreload">
<link href="../_app/immutable/chunks/D_K3zgtV.js" rel="modulepreload">
<link href="../_app/immutable/chunks/DcdnLqSs.js" rel="modulepreload">
<link href="../_app/immutable/chunks/C53el5RR.js" rel="modulepreload">
<link href="../_app/immutable/chunks/BVH2kmGN2.js" rel="modulepreload">
<link href="../_app/immutable/chunks/BlqOrNhF2.js" rel="modulepreload">
<link href="../_app/immutable/chunks/BchFKzpt2.js" rel="modulepreload">
<link href="../_app/immutable/chunks/TMlrP8cf2.js" rel="modulepreload">
<link href="../_app/immutable/chunks/ZhFYVuyj2.js" rel="modulepreload">
<link href="../_app/immutable/chunks/D1jaZ42H2.js" rel="modulepreload">
<link href="../_app/immutable/chunks/CQj7sSHo2.js" rel="modulepreload">
<link href="../_app/immutable/nodes/12.DjHXIJxs.js" rel="modulepreload">
<link href="../_app/immutable/chunks/DTCWZ-3d.js" rel="modulepreload">
<link href="../_app/immutable/nodes/46.BHBEXtKt.js" rel="modulepreload">
<link href="../_app/immutable/chunks/BAACmaqg2.js" rel="modulepreload">
<link href="../_app/immutable/chunks/6qIGhfx82.js" rel="modulepreload">
<link href="../_app/immutable/chunks/B0-a6fSW2.js" rel="modulepreload">
<link href="../_app/immutable/chunks/BJ8Dsm2U.js" rel="modulepreload">
<link href="../_app/immutable/chunks/CoMhm74g.js" rel="modulepreload">
<link href="../_app/immutable/chunks/cl-IST5J.js" rel="modulepreload">
<link href="../_app/immutable/chunks/CW8kdd50.js" rel="modulepreload">
<link href="../_app/immutable/chunks/BLW5OeLH.js" rel="modulepreload">
<link href="../_app/immutable/chunks/BewAyoiJ.js" rel="modulepreload">
<link href="../_app/immutable/chunks/CvUg0BuQ.js" rel="modulepreload">
<link href="../_app/immutable/chunks/gXXOdQXL2.js" rel="modulepreload">
<link href="../_app/immutable/chunks/Cfl5O6MR.js" rel="modulepreload">
<link href="../_app/immutable/chunks/B4xB6k1V2.js" rel="modulepreload">
<!--18jd5ta--><!----><script type="application/ld+json">{"@context":"https://schema.org/","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Blog","item":"https://temporal.io/blog"},{"@type":"ListItem","position":2,"name":"How-To","item":"https://temporal.io/blog/categories/how-to"},{"@type":"ListItem","position":3,"name":"Building application reliability on top of infrastructure unreliability","item":"https://temporal.io/blog/building-application-reliability-on-top-of-infrastructure-unreliability"}]}</script><!----> <!----><script type="application/ld+json">{"@context":"https://schema.org","@type":"BlogPosting","mainEntityOfPage":{"@type":"WebPage","@id":"https://temporal.io/blog/building-application-reliability-on-top-of-infrastructure-unreliability"},"headline":"Building application reliability on top of infrastructure unreliability","image":"","author":[{"@type":"Person","name":"Fitz"}],"publisher":{"@id":"https://temporal.io/#organization"},"datePublished":"2024-01-12T00:00-07:00"}</script><!----><!----><!--92clt7--><meta property="og:image" content="https://images.ctfassets.net/0uuz8ydxyd9p/ixYNBwZoX1wu8VwLSDFmS/35e58aea31546f1b0dc42311f8f1de98/social-card-machine.jpg?w=1200&amp;h=630&amp;fit=fill&amp;q=80"/> <meta property="og:image:width" content="1200"/> <meta property="og:image:height" content="630"/> <meta property="og:image:alt" content="Temporal Open Graph Image"/> <meta property="og:image:type" content="image/png"/><!----><!--nppn8n--><!--[0--><!--]--> <!--[0--><meta name="description" content="What do you do when you know, with absolute certainty, that the infrastructure you're running something on is unreliable? Hardware always fails eventually, networks go down, or software needs to be patched."/><!--]--> <!--[0--><link rel="canonical" href="https://temporal.io/blog/building-application-reliability-on-top-of-infrastructure-unreliability"/><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--> <meta name="robots" content="index,follow"/> <meta name="googlebot" content="index,follow"/> <!--[-1--><!--]--> <!--[-1--><!--]--> <!--[0--><!--[--><meta name="twitter:card" content="summary_large_image"/><!--]--><!--]--> <!--[0--><!--[0--><!--[--><!--[0--><meta property="og:type" content="website"/><!--]--> <!--[-1--><!--]--><!--[0--><meta property="og:url" content="https://temporal.io/blog/building-application-reliability-on-top-of-infrastructure-unreliability"/><!--]--> <!--[-1--><!--]--><!--[0--><meta property="og:title" content="Building application reliability on top of infrastructure unreliability"/><!--]--> <!--[-1--><!--]--><!--[0--><meta property="og:description" content="What do you do when you know, with absolute certainty, that the infrastructure you're running something on is unreliable? Hardware always fails eventually, networks go down, or software needs to be patched."/><!--]--> <!--[-1--><!--]--><!--]--><!--]--><!--]--> <!--[-1--><!--]--> <!--[--><!--]--><!----><title>Building application reliability on top of infrastructure unreliability | Temporal</title>
<link href="../_app/immutable/assets/app.CwZIutBK.css" rel="stylesheet">
<link href="../_app/immutable/assets/text.BL3jYVE0.css" rel="stylesheet">
<link href="../_app/immutable/assets/grid.BKppebuq.css" rel="stylesheet">
<link href="../_app/immutable/assets/rich-text.BdlK0XS7.css" rel="stylesheet">
<link href="../_app/immutable/assets/marketo-form.DlwN02SH.css" rel="stylesheet">
<link href="../_app/immutable/assets/button.DB_t02g_.css" rel="stylesheet">
<link href="../_app/immutable/assets/circle-arrow.5ktYyVBC.css" rel="stylesheet">
<link href="../_app/immutable/assets/site-navigation.WwiOQTah.css" rel="stylesheet">
<link href="../_app/immutable/assets/promo-card.CR6Et2a6.css" rel="stylesheet">
<link href="../_app/immutable/assets/social-share.B8UGBDfC.css" rel="stylesheet">
<link href="../_app/immutable/assets/contentful-image.B8tR6945.css" rel="stylesheet">
<link href="../_app/immutable/assets/testimonial.CNHHSnhi.css" rel="stylesheet">
<link href="../_app/immutable/assets/promo-block.BeFTQLbA.css" rel="stylesheet">
<link href="../_app/immutable/assets/blog-header.BXmklgpF.css" rel="stylesheet">
<link href="../_app/immutable/assets/table-of-contents.CdfFhm_i.css" rel="stylesheet">
<link href="../_app/immutable/assets/46.D8NdCl-j.css" rel="stylesheet">
</head>
<!--
data-sveltekit-preload-data="hover" causes false logouts and redirects from the SSU layout when hovering over the link to SSU in the main nav
-->
<body data-sveltekit-preload-data="tap" data-sveltekit-preload-code="hover">
<a href="#main-content" class="skip-to-main-content">
Skip to main content
</a>
<div style="display: contents"><!--[--><!--[0--><!--[--><div style="display: contents" data-sveltekit-reload="off"><!--[0--><!--[--><div class="layout-wrapper"><!--[0--><div class="h-11 announcement-banner z-10 bg-ultraviolet bg-center px-10 text-sm py-3 text-white max-md:hidden svelte-tkkmnz"><div class="flex items-center justify-between"><div class="w-full flex items-center justify-between gap-6"><!--[0--><a href="/reports/state-of-development-2026" class="max-w-xs text-left sm:max-w-sm md:max-w-full"><p>Faster is easy with AI. Better takes more | <span class="pulsar-link">Read the State of Development 2026 to learn what makes AI agents succeed ›</span> </p></a><!--]--> <!--[0--><nav aria-label="top bar menu" class="flex items-center gap-6 shrink-0 max-lg:hidden"><!--[--><a href="/about" class="hover:underline whitespace-nowrap">About</a><a href="/careers" class="hover:underline whitespace-nowrap">Careers</a><a href="https://pages.temporal.io/contact-us" target="_blank" class="hover:underline whitespace-nowrap">Talk to Sales</a><!--]--></nav><!--]--></div></div></div><!--]--><!----> <nav id="site-navigation" class="site-nav site-nav-lg bg-space-black svelte-1rc5go3" style="background-image: none;"><a href="/" aria-label="Home" class="shrink-0 nav-logo"><img src="/images/logos/logo-temporal-with-copy-white-text.svg" alt="Temporal Logo"/></a> <span class="xl:hidden ml-auto"><!----><button class="button group w-fit ghost center regular off-white svelte-103wh5p" aria-label="Open navigation menu"><!--[1--><!--[0--><svg width="24" height="24" class="transition-colors text-off-white group-hover:text-blue-gray-300" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M1 2.5H23V4.9375H1V2.5ZM1 10.625H23V13.0625H1V10.625ZM23 18.75V21.1875H1V18.75H23Z"></path><!--]--><!----><defs><linearGradient id="mist-d1a9bd61-7b76-4a70-9810-7794a7bee0c1"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-d1a9bd61-7b76-4a70-9810-7794a7bee0c1"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-d1a9bd61-7b76-4a70-9810-7794a7bee0c1"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-d1a9bd61-7b76-4a70-9810-7794a7bee0c1"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--><!----></button><!----><!----></span> <ul class="links links-lg nav-center svelte-1rc5go3"><!--[--><!--[1--><li class="nav-link-group-xl group svelte-oqoqiu">Platform <ul class="sub-nav bg-[#141414d9] svelte-oqoqiu"><!--[--><!--[-1--><!--[--><!--[0--><li class="nav-link svelte-oqoqiu"><a href="/product" class="svelte-oqoqiu">Overview</a></li><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><li class="nav-link svelte-oqoqiu"><a href="/how-it-works" class="svelte-oqoqiu">How Temporal Works</a></li><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><li class="nav-link svelte-oqoqiu"><a href="/cloud" class="svelte-oqoqiu">Temporal Cloud</a></li><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><li class="nav-link svelte-oqoqiu"><a href="/security" class="svelte-oqoqiu">Security</a></li><!--]--><!--]--><!--]--><!--]--></ul></li> <li class="nav-link-group-max-xl svelte-oqoqiu"><button class="sub-nav-trigger svelte-oqoqiu" aria-label="Open Platform sub-navigation">Platform <!--[0--><svg width="20" height="20" class="" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M13.25 13.25V22H10.75V13.25H2V10.75H10.75V2H13.25V10.75H22V13.25H13.25Z" fill-rule="evenodd" clip-rule="evenodd"></path><!--]--><!----><defs><linearGradient id="mist-9517b79f-52ef-403e-9b72-c3f2b397e66c"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-9517b79f-52ef-403e-9b72-c3f2b397e66c"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-9517b79f-52ef-403e-9b72-c3f2b397e66c"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-9517b79f-52ef-403e-9b72-c3f2b397e66c"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!----></button> <!--[-1--><!--]--></li><!--]--><!--[0--><li class="nav-link svelte-oqoqiu"><a data-main-nav-link="true" class="xl:hover:text-white xl:focus:text-white svelte-oqoqiu" href="https://docs.temporal.io">Docs</a></li><!--]--><!--[0--><li class="nav-link svelte-oqoqiu"><a data-main-nav-link="true" class="xl:hover:text-white xl:focus:text-white svelte-oqoqiu" href="/pricing">Pricing</a></li><!--]--><!--[1--><li class="nav-link-group-xl group svelte-oqoqiu">Use Cases <ul class="sub-nav bg-[#141414d9] svelte-oqoqiu"><!--[--><!--[-1--><!--[--><!--[0--><li class="nav-link svelte-oqoqiu"><a href="/in-use" class="svelte-oqoqiu">Customer Stories</a></li><!--]--><!--]--><!--]--><!--[0--><li class="sub-nav-section svelte-oqoqiu"><p class="sub-nav-section-title svelte-oqoqiu">By Industry</p> <ul class="sub-nav-section-links svelte-oqoqiu"><!--[--><!--[0--><li class="nav-link svelte-oqoqiu"><a href="/solutions/ai" class="svelte-oqoqiu">AI</a></li><!--]--><!--[0--><li class="nav-link svelte-oqoqiu"><a href="/solutions/financial-services" class="svelte-oqoqiu">Financial Services</a></li><!--]--><!--]--></ul></li><!--]--><!--[0--><li class="sub-nav-section svelte-oqoqiu"><p class="sub-nav-section-title svelte-oqoqiu">By Audience</p> <ul class="sub-nav-section-links svelte-oqoqiu"><!--[--><!--[0--><li class="nav-link svelte-oqoqiu"><a href="/solutions/platform-engineering" class="svelte-oqoqiu">Platform Engineering</a></li><!--]--><!--[0--><li class="nav-link svelte-oqoqiu"><a href="/startup" class="svelte-oqoqiu">Startups</a></li><!--]--><!--[0--><li class="nav-link svelte-oqoqiu"><a href="/partners" class="svelte-oqoqiu">Partners</a></li><!--]--><!--]--></ul></li><!--]--><!--]--></ul></li> <li class="nav-link-group-max-xl svelte-oqoqiu"><button class="sub-nav-trigger svelte-oqoqiu" aria-label="Open Use Cases sub-navigation">Use Cases <!--[0--><svg width="20" height="20" class="" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M13.25 13.25V22H10.75V13.25H2V10.75H10.75V2H13.25V10.75H22V13.25H13.25Z" fill-rule="evenodd" clip-rule="evenodd"></path><!--]--><!----><defs><linearGradient id="mist-f029b1d7-051a-4e00-bf87-9fd90e601b3c"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-f029b1d7-051a-4e00-bf87-9fd90e601b3c"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-f029b1d7-051a-4e00-bf87-9fd90e601b3c"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-f029b1d7-051a-4e00-bf87-9fd90e601b3c"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!----></button> <!--[-1--><!--]--></li><!--]--><!--[1--><li class="nav-link-group-xl group svelte-oqoqiu active">Resources <ul class="sub-nav bg-[#141414d9] svelte-oqoqiu"><!--[--><!--[-1--><!--[--><!--[0--><li class="nav-link svelte-oqoqiu"><a href="/resources" class="svelte-oqoqiu">Resource Library</a></li><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><li class="nav-link svelte-oqoqiu"><a href="https://learn.temporal.io" class="svelte-oqoqiu">Learn Temporal</a></li><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><li class="nav-link svelte-oqoqiu"><a href="/community" class="svelte-oqoqiu">Community</a></li><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><li class="nav-link svelte-oqoqiu"><a href="/code-exchange" class="svelte-oqoqiu">Code Exchange</a></li><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><li class="nav-link svelte-oqoqiu"><a href="/blog" class="svelte-oqoqiu active">Blog</a></li><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><li class="nav-link svelte-oqoqiu"><a href="/changelog" class="svelte-oqoqiu">Changelog</a></li><!--]--><!--]--><!--]--><!--]--></ul></li> <li class="nav-link-group-max-xl svelte-oqoqiu"><button class="sub-nav-trigger svelte-oqoqiu" aria-label="Open Resources sub-navigation">Resources <!--[0--><svg width="20" height="20" class="" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M13.25 13.25V22H10.75V13.25H2V10.75H10.75V2H13.25V10.75H22V13.25H13.25Z" fill-rule="evenodd" clip-rule="evenodd"></path><!--]--><!----><defs><linearGradient id="mist-bdbba6f7-bb20-4cfb-b4e4-ed536e033037"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-bdbba6f7-bb20-4cfb-b4e4-ed536e033037"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-bdbba6f7-bb20-4cfb-b4e4-ed536e033037"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-bdbba6f7-bb20-4cfb-b4e4-ed536e033037"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!----></button> <!--[-1--><!--]--></li><!--]--><!--]--></ul> <div class="nav-actions svelte-1rc5go3"><button class="search-link modal-trigger svelte-1rc5go3" aria-label="Search"><!--[0--><svg width="24" height="24" class="" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M18.0706 9.76908C18.0706 6.85714 16.4706 4.13307 13.9294 2.63014C11.3412 1.17417 8.18824 1.17417 5.64706 2.63014C3.05882 4.13307 1.50588 6.85714 1.50588 9.76908C1.50588 12.728 3.05882 15.4521 5.64706 16.955C8.18824 18.411 11.3412 18.411 13.9294 16.955C16.4706 15.4521 18.0706 12.728 18.0706 9.76908ZM16.1412 17.1898C14.4471 18.6928 12.1882 19.5382 9.78824 19.5382C4.37647 19.5382 0 15.1703 0 9.76908C0 4.41487 4.37647 0 9.78824 0C15.1529 0 19.5765 4.41487 19.5765 9.76908C19.5765 12.2114 18.6824 14.4188 17.2235 16.1566L24 22.9198L22.9176 24L16.1412 17.1898Z"></path><!--]--><!----><defs><linearGradient id="mist-0832278d-e329-4aea-a9e2-34ba5b524ae1"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-0832278d-e329-4aea-a9e2-34ba5b524ae1"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-0832278d-e329-4aea-a9e2-34ba5b524ae1"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-0832278d-e329-4aea-a9e2-34ba5b524ae1"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!----></button> <a class="gh-link svelte-1rc5go3" href="https://github.com/temporalio" aria-label="Temporal on GitHub" target="_blank" rel="noopener noreferrer"><!--[0--><svg width="24" height="24" class="" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M8.15156 18.6281C8.15156 18.7219 8.04375 18.7969 7.90781 18.7969C7.75312 18.8109 7.64531 18.7359 7.64531 18.6281C7.64531 18.5344 7.75313 18.4594 7.88906 18.4594C8.02969 18.4453 8.15156 18.5203 8.15156 18.6281ZM6.69375 18.4172C6.66094 18.5109 6.75469 18.6187 6.89531 18.6469C7.01719 18.6937 7.15781 18.6469 7.18594 18.5531C7.21406 18.4594 7.125 18.3516 6.98438 18.3094C6.8625 18.2766 6.72656 18.3234 6.69375 18.4172ZM8.76562 18.3375C8.62969 18.3703 8.53594 18.4594 8.55 18.5672C8.56406 18.6609 8.68594 18.7219 8.82656 18.6891C8.9625 18.6562 9.05625 18.5672 9.04219 18.4734C9.02812 18.3844 8.90156 18.3234 8.76562 18.3375ZM11.85 0.375C5.34844 0.375 0.375 5.31094 0.375 11.8125C0.375 17.0109 3.64688 21.4594 8.32031 23.025C8.92031 23.1328 9.13125 22.7625 9.13125 22.4578C9.13125 22.1672 9.11719 20.5641 9.11719 19.5797C9.11719 19.5797 5.83594 20.2828 5.14688 18.1828C5.14688 18.1828 4.6125 16.8187 3.84375 16.4672C3.84375 16.4672 2.77031 15.7312 3.91875 15.7453C3.91875 15.7453 5.08594 15.8391 5.72812 16.9547C6.75469 18.7641 8.475 18.2437 9.14531 17.9344C9.25313 17.1844 9.55781 16.6641 9.89531 16.3547C7.275 16.0641 4.63125 15.6844 4.63125 11.175C4.63125 9.88594 4.9875 9.23906 5.7375 8.41406C5.61562 8.10938 5.21719 6.85312 5.85938 5.23125C6.83906 4.92656 9.09375 6.49688 9.09375 6.49688C10.0312 6.23438 11.0391 6.09844 12.0375 6.09844C13.0359 6.09844 14.0438 6.23438 14.9813 6.49688C14.9813 6.49688 17.2359 4.92188 18.2156 5.23125C18.8578 6.85781 18.4594 8.10938 18.3375 8.41406C19.0875 9.24375 19.5469 9.89062 19.5469 11.175C19.5469 15.6984 16.7859 16.0594 14.1656 16.3547C14.5969 16.725 14.9625 17.4281 14.9625 18.5297C14.9625 20.1094 14.9484 22.0641 14.9484 22.4484C14.9484 22.7531 15.1641 23.1234 15.7594 23.0156C20.4469 21.4594 23.625 17.0109 23.625 11.8125C23.625 5.31094 18.3516 0.375 11.85 0.375ZM4.93125 16.5422C4.87031 16.5891 4.88438 16.6969 4.96406 16.7859C5.03906 16.8609 5.14687 16.8938 5.20781 16.8328C5.26875 16.7859 5.25469 16.6781 5.175 16.5891C5.1 16.5141 4.99219 16.4812 4.93125 16.5422ZM4.425 16.1625C4.39219 16.2234 4.43906 16.2984 4.53281 16.3453C4.60781 16.3922 4.70156 16.3781 4.73438 16.3125C4.76719 16.2516 4.72031 16.1766 4.62656 16.1297C4.53281 16.1016 4.45781 16.1156 4.425 16.1625ZM5.94375 17.8312C5.86875 17.8922 5.89687 18.0328 6.00469 18.1219C6.1125 18.2297 6.24844 18.2437 6.30937 18.1688C6.37031 18.1078 6.34219 17.9672 6.24844 17.8781C6.14531 17.7703 6.00469 17.7563 5.94375 17.8312ZM5.40938 17.1422C5.33437 17.1891 5.33437 17.3109 5.40938 17.4188C5.48438 17.5266 5.61094 17.5734 5.67188 17.5266C5.74687 17.4656 5.74687 17.3438 5.67188 17.2359C5.60625 17.1281 5.48438 17.0813 5.40938 17.1422Z"></path><!--]--><!----><defs><linearGradient id="mist-8ba70b0e-265b-4286-8fcf-5fe6592da67c"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-8ba70b0e-265b-4286-8fcf-5fe6592da67c"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-8ba70b0e-265b-4286-8fcf-5fe6592da67c"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-8ba70b0e-265b-4286-8fcf-5fe6592da67c"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!----></a> <div class="flex items-center gap-2"><!--[--><!----><a href="/get-cloud" data-contentful-entry-id="79w1QMsJg2MTAVGlu2jdss" class="button group w-fit primary center small ultraviolet svelte-103wh5p"><!--[-1--><!--]--> <!--[0--><!---->Try Free<!----><!--]--> <!--[-1--><!--]--><!----></a><!----><!----><a href="https://cloud.temporal.io/" data-contentful-entry-id="KAPnScSjIJkz74RbczQ2F" class="button group w-fit secondary center small ultraviolet svelte-103wh5p"><!--[-1--><!--]--> <!--[0--><!---->Log In<!----><!--]--> <!--[-1--><!--]--><!----></a><!----><!--]--></div></div></nav> <!--[-1--><!--]--> <dialog class="modal svelte-1bjjo3r modal-align-top" aria-label="Search" aria-modal="true"><!--[-1--><!--]--></dialog><!----><!----> <main class="main-content" id="main-content" tabindex="-1"><!--[0--><!--[--><!--[-1--><!--[--><!--[0--><!----> <div class="bg-deep-blue"><!----><div class="grid grid-flow-col-dense sm:grid-cols-page sm:gap-x-4 max-sm:grid-cols-1 max-sm:px-5 max-sm:gap-y-4 bg-none bg-transparent bg-deep-blue py-8 lg:py-24 columns temporal-grid svelte-iff7rv"><!----><div class="max-sm:col-span-1 sm:col-span-10 max-sm:col-start-1 sm:col-start-3 col-span-10 col-start-3 lg:col-span-10 lg:col-start-3 bg-deep-blue"><!--[--><div class="blog-header border-bottom deep-blue svelte-1uxl4gb"><!--[0--><div class="breadcrumbs svelte-1uxl4gb"><ul class="svelte-1uxl4gb"><li class="svelte-1uxl4gb"><a href="/blog" class="svelte-1uxl4gb">Blog</a></li> <!--[0--><li class="svelte-1uxl4gb"><a href="/blog/categories/how-to" class="svelte-1uxl4gb">How-To</a></li><!--]--></ul></div><!--]--> <!--[0--><h1 class="title svelte-1uxl4gb">Building application reliability on top of infrastructure unreliability</h1><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div><!--]--><!----></div><!----><!----> <!----><div class="max-sm:col-span-1 sm:col-span-10 max-sm:col-start-1 sm:col-start-3 col-span-10 col-start-3 lg:col-span-2 lg:col-start-3 bg-deep-blue"><!--[--><div class="info-container svelte-18jd5ta"><div class="info-block authors svelte-18jd5ta"><div class="property svelte-18jd5ta">AUTHORS</div> <!--[0--><div class="value svelte-18jd5ta">Fitz</div><!--]--></div> <!--[1--><div class="info-block date svelte-18jd5ta"><div class="property svelte-18jd5ta">PUBLISHED</div> <div class="value svelte-18jd5ta">Jan 12, 2024</div></div><!--]--> <!--[0--><div class="info-block category svelte-18jd5ta"><div class="property svelte-18jd5ta">CATEGORY</div> <div class="value svelte-18jd5ta"><div class="category svelte-18jd5ta"><a href="/blog/categories/how-to" class="svelte-18jd5ta">How-To</a></div></div></div><!--]--> <!--[0--><div class="info-block duration svelte-18jd5ta"><div class="property svelte-18jd5ta">DURATION</div> <div class="value svelte-18jd5ta">12 MIN</div></div><!--]--> <!--[0--><div class="info-block tags svelte-18jd5ta"><ul class="tags-list svelte-18jd5ta" aria-label="Tags"><!--[--><li class="tag svelte-18jd5ta">Industry Events</li><!--]--></ul></div><!--]--> <!--[0--><div class="scroll-progress-bar svelte-1p1vqhf" data-variant="blog" aria-hidden="true"><div class="progress-indicator svelte-1p1vqhf" style="width: 0%;"></div></div><!--]--> <!--[-1--><!--]--><!----></div><!--]--><!----></div><!----><!----> <!----><div class="max-sm:col-span-1 sm:col-span-10 max-sm:col-start-1 sm:col-start-3 col-span-10 col-start-3 lg:col-span-6 lg:col-start-5 bg-deep-blue"><!--[--><article class="blog-body"><div data-contentful-entry-id="4z6WrsBLY38fBGZSaasb8X" data-contentful-field-id="content" class="prose max-w-none prose-dark default paragraph-margin-bottom-8 code-bg-color"><!--[0--><!----><p>What do you do when you know, with absolute certainty, that the infrastructure you're running something on is unreliable? Hardware always fails eventually, networks go down, or software needs to be patched.</p>
<p>Even under perfectly ordinary operation, your runtime environment could drop out from under the running process: traffic drops, causing the autoscaler to downscale and, unless you've tied into <a href="https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-termination" rel="noopener" target="_blank">the termination lifecycle</a>, the process is forcibly killed. But whether purposeful or not, it's a computing fact that processes can abruptly end and be restarted.</p>
<p>For many kinds of services, that's perfectly acceptable; give the process a chance to drain outstanding requests, and then new requests will land on other nodes. Or, if that process unexpectedly dies or is forcibly ended, the client can retry and have its request handled elsewhere.</p>
<p>Put another way, things can be made "effectively reliable" from an infrastructure perspective: for example, Kubernetes helps ensure that nodes and processes exist to run application code, and <a href="https://istio.io/" rel="noopener" target="_blank">Istio</a> improves network reliability and performance. Things still fail, but we can make it tolerable with well-known tools.</p>
<p>But perhaps a more important question is: what's <em>actually</em> being run? As in, what is the full application trying to accomplish?</p>
<h2 id="an-example-application">An Example Application<a href="#an-example-application"><span class="ml-4">#</span></a></h2>
<p>Building something from the bottom up is a natural way to go; assume that things will fail (always a safe assumption), and design the system from there. But this makes it hard to build and keep track of a multi-step application, like an e-commerce store. Let's take a look.</p>
<p>Consider how this application might look from the customer's perspective. It's a relatively short series of steps:</p>
<ol>
<li>Add item to cart</li>
<li>Input payment</li>
<li>Input shipping address</li>
<li>Confirm and checkout</li>
<li>Track order</li>
<li>Receive item</li>
</ol>
<p>Software engineering <a href="https://en.wikipedia.org/wiki/Microservices#History" rel="noopener" target="_blank">over the past couple of decades</a> has taught us that nothing is simple. Namely, because we've made our infrastructures <em>effectively reliable</em> through tools like Kubernetes and patterns like stateless services, a "simple" workflow like the above ends up being composed of dozens of microservices (<a href="https://twitter.com/Werner/status/741673514567143424" rel="noopener" target="_blank">or more</a>).</p>
<p>
<img src="//images.ctfassets.net/0uuz8ydxyd9p/5OHtgsWdkhnR4jCu6lLHyv/11bc45a5478d37956f176e237731b4b7/image2.png-inverted.png" alt="mock ecommerce microservices">
<em>Caption: A hypothetical but realistic microservices connection graph for placing an e-commerce order.</em>
</p>
<p>That's not all, though, because behind and in between these microservices also ends up being a complex mix of databases, caches, event streams, and message queues. While these services all have unique functions, they're all dealing with the same orders, and <em>hopefully</em> each order's details and status are appropriately picked up by the appropriate next service.</p>
<p>Thus, tracing a single order through this dizzying system becomes a daunting or even impossible task. While this is necessary for tasks like ensuring orders aren't dropped, providing customer service, or auditing for regulatory compliance, it's also important to answer a more fundamental question: How reliable is our application?</p>
<h2 id="applications-need-reliability-too">Applications Need Reliability Too<a href="#applications-need-reliability-too"><span class="ml-4">#</span></a></h2>
<p>At some point, that question will come up. And I'm sure you've had the experience of trying to calculate some kind of overall reliability number, only to find that a single number like <code>99.99%</code> doesn't adequately represent the full story of the application.</p>
<p>Let's take a look at a simplified example.</p>
<p>
Suppose you have a two-step process, with each step handled by separate microservices. For the sake of example, let's say the process is painting a bike shed, and the two steps are (1) choose and reserve a paint color and (2) apply paint to the shed. Step 2 cannot happen without Step 1 occurring first. But if we have two sheds, the microservices pattern allows the application to handle the two sheds independently from each other: Step 1 could be working on picking a paint color for Shed B without affecting Step 2's painting of Shed A.
<img src="//images.ctfassets.net/0uuz8ydxyd9p/3DhYNMxnRMXwzjxTWxuv8f/933a570168f0055119971f0f537231f9/image9.png-inverted.png" alt="step 1 - 2">
</p>
<p>Reasoning about the reliability of this application is really asking: What is the likelihood of a given shed successfully making it through both steps?</p>
<p>For a serial system like this, the success probability is calculated as the product of the success probabilities for each individual step. So, if Step 1 has a 98% chance of success and Step 2 has a 99% chance, the overall probability of success (i.e., the "reliability" of the application) would be <code>0.99 * 0.98 = 0.9702</code>, or ~97%.</p>
<p>You might note that this is actually worse than "the weakest link." You can think of the product here as a logical <code>and</code>: <em>both</em> microservices must succeed in order for an overall success to occur. Even though we have a 98% chance of successfully painting the shed, the overall process could still fail if the committee deadlocks on making a decision in Step 1.</p>
<p><strong>Aside:</strong> This then effectively proves what most of us intuitively know to be true. If you add more components to a system, its overall reliability goes down, even if each individual component has a success probability of "<a href="https://en.wikipedia.org/wiki/High_availability#Percentage_calculation" rel="noopener" target="_blank">five nines</a>." Said system would have an overall chance of success of <code>.99999 ^ n</code>, where <code>n</code> is the number of services (assuming they're in serial). Meaning, just adding a third service to our example (say, splitting out choosing the paint color versus procuring the actual paint) reduces the chance of success to four nines, about .99997 or the equivalent of one failure out of every 33,333. <sup><a href="#user-content-fn-1" id="user-content-fnref-1" data-footnote-ref aria-describedby="footnote-label">1</a></sup></p>
<p>
You can make the story a little bit better by adding redundancy to each step.
<img src="//images.ctfassets.net/0uuz8ydxyd9p/6tYZDjZuwQlY1tlsH5bWAm/8558bc7f1abea164e8165a4766688349/image6.png-inverted.png" alt="step 1 - 2 with replicas">
</p>
<p>Even if the backup services are wildly unreliable, it still improves the overall reliability, which is calculated as:</p>
<pre><code>Reliability of step 1 (R1) = 1 - (1 - .99) * (1 - .5) = 0.995
Reliability of step 2 (R2) = 1 - (1 - .98) * (1 - .6) = 0.992
Overall reliability = R1 * R2 = .98704
</code></pre>
<p>That 98.7% chance of success is certainly better than our 97% chance from earlier, but it does come with the extra operational cost of running secondary services.</p>
<p>Why is this important?</p>
<p>
Part of the reason for splitting things out into microservices in the first place is to allow Step 1 to process as many requests as it can independently of Step 2 and vice-versa. To scale this, you might replicate each service as many times as needed, such that you have <code>N</code> instances of Step 1's service, and <code>M</code> instances of Step 2's.
<img src="//images.ctfassets.net/0uuz8ydxyd9p/3BP3uoWrCpMIEwTF3NGH73/d1287911305b5e7537a0f297648ee157/image8.png-inverted.png" alt="step 1 - 2 N replicas">
</p>
<p>While this improves our overall reliability number, we've introduced a new complication.</p>
<p>When the application was just two services wired in serial, once Step 1 was finished it could directly notify Step 2 and then move on to the next incoming request. Each service could maintain its own queue of requests (with possible persistence) and very little other coordination would be needed.</p>
<p>Unfortunately, that pesky little parenthetical tucked in there is the culprit of our complications. The "possible persistence" of a request queue is helpful if you want to not lose requests in the face of, e.g., process restarts. But, it's <em>absolutely required</em> once you scale out beyond n = 1 instances of the service so that each request can be handled effectively once.</p>
<p>
For short-lived requests, this could be handled by some kind of load balancer in front of each step:
<img src="//images.ctfassets.net/0uuz8ydxyd9p/4YPtglXRxFbBVQBnpfpDiZ/31cd260993404ad24a0af4c474f3cb07/image7.png-inverted.png" alt="step 1 - 2 with load balancers">
</p>
<p>The load balancers act as the request queues, divvying requests out to workers as appropriate.</p>
<p>The side effect is that we've now doubled the number of components in the system. Load balancers can fail just like any other service, and so the system reliability becomes</p>
<p>R <sub>LB1</sub> * R <sub>1</sub> * R <sub>LB2</sub> * R <sub>2</sub></p>
<p>As discussed in the <strong>aside</strong> earlier, if all four of these services have a five-nines (99.999%) chance of successfully processing a given request, then the overall reliability is .999994 = 0.99996 or an equivalence of one out of every 25,000 requests failing. That may not seem too bad, but without those load balancers, the success rate would have been .999992 = .99998 or a failure rate of one per <em>50,000</em>. Doubling the number of services in the sequence also doubled our failure rate.</p>
<p>
So, naturally, let's remove one of the load balancers and instead put a cache, database, or message queue in between:
<img src="//images.ctfassets.net/0uuz8ydxyd9p/7sd7JiGTFADQD3FAbkS9lv/680b135414b32b8679db0f86df6db193/image1.png-inverted.png" alt="step 1 - 2 with queue">
</p>
<p>This intermediate queue provides a central place for keeping track of where your various sheds are in their painting lifecycles. That lifecycle is still somewhat disjointed though, and your components (Steps 1 and 2 and the Queue) are still very much wired in serial in a way that one outage affects an untold number of sheds—which ones and how is anyone's guess.</p>
<h2 id="answering-the-bigger-questions">Answering the Bigger Questions<a href="#answering-the-bigger-questions"><span class="ml-4">#</span></a></h2>
<p>This brings us, finally, back to the original question, "How reliable is the application?" The previous section described how to calculate the reliability of an overall system. Being able to say that the combined success rate of all services in the system is X% <em>sort of</em> answers the application-reliability question, but only does so from a fairly conceptual level.</p>
<p>More specifically, while it tells you how often any one random request will fail, it doesn't answer questions like "Where is <em>this</em> request?" or "Where did <em>this</em> request fail?" or "Is <em>this</em> request going to eventually unstick itself or does a human need to step in?"</p>
<p>That is to say, if you're looking at the aforementioned collection of microservices in an e-commerce system and wondering where a particular order is in its lifecycle (footnote: because, for example, you're building a frontend for customer service agents), well.... good luck.</p>
<h3 id="building-reliable-applications-with-temporal">Building Reliable Applications With Temporal<a href="#building-reliable-applications-with-temporal"><span class="ml-4">#</span></a></h3>
<p>
Having a single number representing the reliability of a system isn't nearly as important as understanding what to do when unexpected things happen (as they <em>always</em> will). And, seeing all green across a status dashboard feels delightful.
<img src="//images.ctfassets.net/0uuz8ydxyd9p/KbrAFZZwJmDBE6QqOMP8h/de7708c3b798234dccf51eb4dd522659/image5.png-inverted-green.png" alt="green dashboard">
</p>
<p>Suppose one of your services is backed by a load-balanced cluster of nodes. What's the impact on customer experience if a misconfiguration in your autoscaler causes a repeated up-then-down scaling?</p>
<p>
<img src="//images.ctfassets.net/0uuz8ydxyd9p/3iMSX05mZAD3l0bMXhNZbk/ea29c68fc5ad551ddd7080a99ad87e77/image3.png-inverted.png" alt="mock node count graph">
</p>
<p>Sure, it's a problem worth investigating, but with Temporal you don't have to worry too much about the end-user impact: each user request could match to a Temporal Workflow, which is automatically resumed on another process where it left off if something interrupts the original process it’s running on.</p>
<p>So for example, consider a simplified user flow through your system that looks like:</p>
<ol>
<li>Check fraud</li>
<li>Validate inventory</li>
<li>Process payment</li>
<li>Dispatch / Ship</li>
</ol>
<p>If each of these is a separate microservice, and one of them is having infrastructure problems, you might be tempted to declare, "Oh no! This whole application is unreliable now!" After all, we saw earlier in this post that the weakest link in the chain tanks your overall reliability numbers.</p>
<p>With Temporal, instead of coming in with the assumption that each individual service needs to be made as reliable as possible, we take it a level higher. What if we assumed that dependent services will <em>always</em> fail, and we instead work to make the <em>application</em> reliable?</p>
<p>
<img src="//images.ctfassets.net/0uuz8ydxyd9p/3PcSQ0VkWqrTxk6ThQEjaE/4f9ae2b6cf21051b32c6c7b529492f18/image4.png-inverted.png" alt="demo graphs inverted">
<em>Caption: Four graphs of a demo Temporal application. Top left: the number of active Kubernetes replicas for the Worker fleet; randomly re-sized between 0 and 10. Top right: Workflow successes per second for this application. Bottom left and right: Workflow timeouts and failures, respectively, both flat at zero.</em>
</p>
<p>In these example graphs, depicting a demo Workflow running a simulated load (upper right graph), the Kubernetes cluster (upper left) responsible for actually running the Workflow (as in, the Worker fleet) is concurrently being randomly scaled up or down. This random scaling is to simulate unreliable infrastructure.</p>
<p>Measuring the uptime of this application from the infrastructure perspective – for example, how often are there no Workers available – leads to a pretty bleak number. Roughly calculated, the graph above represents an hour-long window. For a total of eight minutes during that hour, the cluster had no replicas available. 52 / 60 * 100 = 86.666% uptime, or in other words: zero nines.</p>
<p>But, take a closer look at the other graphs. While the <em>rate</em> of Workflow successes dipped to zero during the infrastructure downtime, they did still continue to succeed: Workflow timeouts and Workflow failures remained flat at zero.</p>
<p>With a Workflow as the thing we're measuring the reliability of, the calculation of reliability over the hour looks much nicer:</p>
<pre><code> sum(timeouts_per_second) (T) = 0
sum(failures_per_second) (F) = 0
sum(successes_per_second) (S) = 685
Application reliability = 1 - (T + F) / S = 1 = 100%
</code></pre>
<p>That is to say, even with much worse reliability than you'd like, Temporal gives you <em>better</em> overall application reliability.</p>
<h2 id="summary">Summary<a href="#summary"><span class="ml-4">#</span></a></h2>
<p>Perhaps obviously, the example used here is contrived, but it's not that far from reality with Temporal. Microservices and external dependencies will always fail at some point. Working to make them as reliable as possible is necessary and important work, but no service will ever truly hit 100% reliability over a sufficiently long period of time.</p>
<p>To mitigate that, consider why services exist in the first place: to provide a, well, service—a self-contained piece of functionality—to a bigger application. When you take a step back and consider the reliability of the bigger picture, rather than service-by-service, your whole application can be reliable while underlying services are inconsistent.</p>
<p>For more on Temporal, head over to our documentation and courses:</p>
<ul>
<li><a href="https://docs.temporal.io" rel="noopener" target="_blank">https://docs.temporal.io</a></li>
<li><a href="https://learn.temporal.io" rel="noopener" target="_blank">https://learn.temporal.io</a></li>
</ul>
<hr>
<section data-footnotes class="footnotes">
<h2 class="sr-only" id="footnote-label">Footnotes<a href="#footnote-label"><span class="ml-4">#</span></a></h2>
<ol>
<li id="user-content-fn-1">
<p>Which, to be fair, is still a heck of a lot of successfully painted sheds. <a href="#user-content-fnref-1" data-footnote-backref="" aria-label="Back to reference 1" class="data-footnote-backref">↩</a></p>
</li>
</ol>
</section><!----><!--]--></div> <dialog class="modal svelte-1bjjo3r modal-backdrop-heavy" aria-label="Zoomed image" aria-modal="true"><!--[-1--><!--]--></dialog><!----><!----></article><!--]--><!----></div><!----><!----> <!----><div class="max-sm:col-span-1 sm:col-span-10 max-sm:col-start-1 sm:col-start-3 col-span-10 col-start-3 lg:col-span-2 lg:col-start-11"><!--[--><div class="promo-share-container svelte-18jd5ta"><div class="share-container svelte-111im6b"><div class="share svelte-111im6b">SHARE</div> <div class="social-links svelte-111im6b"><!--[0--><a href="https://temporal.io/blog/feed.xml" target="_blank" rel="noopener noreferrer" aria-label="Share on RSS" title="Share on RSS" class="svelte-111im6b"><!--[0--><svg width="22" height="22" class="" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="m3.42857 0h17.14283c1.875 0 3.4286 1.55357 3.4286 3.42857v17.14283c0 1.9286-1.5536 3.4286-3.4286 3.4286h-17.14283c-1.92857 0-3.42857-1.5-3.42857-3.4286v-17.14283c0-1.875 1.5-3.42857 3.42857-3.42857zm1.71429 4.28571v2.57143c6.58924 0 12.00004 5.41076 12.00004 11.99996h2.5714c0-8.0357-6.5357-14.57139-14.57144-14.57139zm0 5.14286v2.57143c3.75 0 6.85714 3.1071 6.85714 6.8571h2.5714c0-5.1964-4.2321-9.42853-9.42854-9.42853zm0 7.71433c0 .9642.75 1.7142 1.71428 1.7142.91072 0 1.71429-.75 1.71429-1.7142 0-.9108-.80357-1.7143-1.71429-1.7143-.96428 0-1.71428.8035-1.71428 1.7143z"></path><!--]--><!----><defs><linearGradient id="mist-1a720e6d-54c5-4d77-95ce-b538c9e52901"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-1a720e6d-54c5-4d77-95ce-b538c9e52901"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-1a720e6d-54c5-4d77-95ce-b538c9e52901"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-1a720e6d-54c5-4d77-95ce-b538c9e52901"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!----></a><!--]--> <a href="https://x.com/intent/post?url=&amp;text=Building%20application%20reliability%20on%20top%20of%20infrastructure%20unreliability&amp;via=temporalio" target="_blank" rel="noopener noreferrer" aria-label="Share on X" title="Share on X" class="svelte-111im6b"><!--[0--><svg width="22" height="22" class="" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M3.42857 0H20.5714C22.4464 0 24 1.55357 24 3.42857V20.5714C24 22.5 22.4464 24 20.5714 24H3.42857C1.5 24 0 22.5 0 20.5714V3.42857C0 1.55357 1.5 0 3.42857 0ZM19.3393 4.5H16.7679L12.5893 9.32143L8.94643 4.5H3.69643L9.96429 12.75L4.01786 19.5H6.58929L11.1964 14.3036L15.1607 19.5H20.3036L13.7679 10.875L19.3393 4.5ZM17.3036 18H15.8571L6.69643 5.94643H8.19643L17.3036 18Z"></path><!--]--><!----><defs><linearGradient id="mist-d9554345-c693-4a15-9452-fa5bf404f701"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-d9554345-c693-4a15-9452-fa5bf404f701"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-d9554345-c693-4a15-9452-fa5bf404f701"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-d9554345-c693-4a15-9452-fa5bf404f701"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!----></a> <a href="https://bsky.app/intent/compose?text=Building%20application%20reliability%20on%20top%20of%20infrastructure%20unreliability%0A" target="_blank" rel="noopener noreferrer" aria-label="Share on Bluesky" title="Share on Bluesky" class="svelte-111im6b"><!--[0--><svg width="28" height="22" class="" viewBox="0 0 32 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="m6.93438 1.61961c3.67152 2.34123 7.61962 7.09868 9.06562 9.65419 1.446-2.55015 5.3941-7.3076 9.0656-9.65419 2.6468-1.6929661 6.9344-3.0002 6.9344 1.16794 0 .83041-.5595 6.99153-.8864 7.99335-1.1442 3.4717-5.3061 4.361-9.0091 3.8199 6.4692.9376 8.1163 4.0449 4.558 7.1523-6.7521 5.9039-9.7006-1.4787-10.455-3.3699-.1069-.2625-.1635-.4179-.2075-.4179s-.1006.1608-.2075.4179c-.7544 1.8912-3.7092 9.2738-10.45498 3.3699-3.55206-3.1074-1.91119-6.2147 4.55796-7.1523-3.70295.5411-7.86483-.3482-9.002749-3.8252-.333203-1.00188-.892731-7.163-.892731-7.99341 0-4.16813 4.28762-2.8609038 6.93438-1.16794z"></path><!--]--><!----><defs><linearGradient id="mist-738ec8fd-677a-4bb8-83da-6d309ba33828"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-738ec8fd-677a-4bb8-83da-6d309ba33828"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-738ec8fd-677a-4bb8-83da-6d309ba33828"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-738ec8fd-677a-4bb8-83da-6d309ba33828"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!----></a> <a href="https://www.linkedin.com/shareArticle?url=" target="_blank" rel="noopener noreferrer" aria-label="Share on LinkedIn" title="Share on LinkedIn" class="svelte-111im6b"><!--[0--><svg width="22" height="22" class="" viewBox="0 0 22 22" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M6.20062 21H1.84688V6.97971H6.20062V21ZM4.02141 5.06721C2.62922 5.06721 1.5 3.91408 1.5 2.52189C1.5 1.85318 1.76565 1.21185 2.2385 0.738991C2.71136 0.266136 3.35269 0.000488281 4.02141 0.000488281C4.69012 0.000488281 5.33145 0.266136 5.80431 0.738991C6.27716 1.21185 6.54281 1.85318 6.54281 2.52189C6.54281 3.91408 5.41313 5.06721 4.02141 5.06721ZM22.4953 21H18.1509V14.175C18.1509 12.5485 18.1181 10.4625 15.8873 10.4625C13.6237 10.4625 13.2769 12.2297 13.2769 14.0578V21H8.92781V6.97971H13.1034V8.89221H13.1644C13.7456 7.79064 15.1655 6.62814 17.2838 6.62814C21.69 6.62814 22.5 9.52971 22.5 13.2985V21H22.4953Z"></path><!--]--><!----><defs><linearGradient id="mist-915e6fbb-9111-40ac-8775-5533d49fdcc5"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-915e6fbb-9111-40ac-8775-5533d49fdcc5"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-915e6fbb-9111-40ac-8775-5533d49fdcc5"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-915e6fbb-9111-40ac-8775-5533d49fdcc5"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!----></a> <a href="https://www.hackernews.com/submitlink?u=&amp;t=Building%20application%20reliability%20on%20top%20of%20infrastructure%20unreliability" target="_blank" rel="noopener noreferrer" aria-label="Share on Hacker News" title="Share on Hacker News" class="svelte-111im6b"><!--[0--><svg width="22" height="22" class="" viewBox="0 0 22 22" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="m21 0v21h-21v-21zm-14.8125 5.12109 3.60449 6.75191v4.4366h1.38671v-4.3545l3.6367-6.83401h-1.5332l-2.1523 4.27344c-.1087.22827-.2151.44819-.3184.66017-.1033.212-.1981.4108-.2851.5957-.0326-.0869-.0709-.182-.1143-.2852-.0435-.1032-.0921-.2097-.1465-.3183-.0108-.0217-.0223-.0461-.0332-.07327-.0109-.02718-.0271-.05722-.0488-.08984-.0326-.08699-.0708-.17138-.1143-.25293-.0434-.08151-.08162-.16026-.11422-.23633l-2.12012-4.27344z"></path><!--]--><!----><defs><linearGradient id="mist-902cd692-002d-47cc-9331-3c5eb4ae2429"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-902cd692-002d-47cc-9331-3c5eb4ae2429"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-902cd692-002d-47cc-9331-3c5eb4ae2429"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-902cd692-002d-47cc-9331-3c5eb4ae2429"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!----></a></div></div><!----> <div class="promo-container sticky svelte-18jd5ta"><!--[0--><div class="promo-card max-md:py-4 md:py-8 max-md:px-3 md:px-6 flex flex-col gap-2 svelte-1pycpkl" data-contentful-id="2SJj3BBBzkr6FefI6ZfRox"><!--[0--><!----><p data-variant="" data-size="" class="text s eyebrow text-indigo-200 uppercase svelte-1i5vmsf" data-contentful-field-id="eyebrow"><!--[--><!---->Temporal Cloud<!--]--><!----></p><!----><!--]--> <!--[0--><!----><p data-variant="heading" data-size="24" class="text s24 heading svelte-1i5vmsf" data-contentful-field-id="heading"><!--[--><!---->Ready to see for yourself?<!--]--><!----></p><!----><!--]--> <!--[0--><div data-contentful-entry-id="2SJj3BBBzkr6FefI6ZfRox" data-contentful-field-id="content" class="prose max-w-none prose-dark default promo-card-content"><!--[0--><!----><p>Sign up for Temporal Cloud today and get $1,000 in free credits.</p><!----><!--]--></div> <dialog class="modal svelte-1bjjo3r modal-backdrop-heavy" aria-label="Zoomed image" aria-modal="true"><!--[-1--><!--]--></dialog><!----><!--]--> <!--[0--><div class="flex flex-wrap gap-2 pt-4"><!--[--><!----><a href="/get-cloud" data-contentful-entry-id="7wKJdxc4JeeNXHCliDfqVV" class="button group primary center regular pink w-full max-lg:w-auto svelte-103wh5p" text="Get started"><!--[-1--><!--]--> <!--[0--><!---->Get started<!----><!--]--> <!--[0--><!--[0--><svg width="24" height="24" class="transition-colors group-hover:text-pink-100" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M14 18L20 12M20 12L14 6M20 12L4 12" stroke="currentcolor" stroke-linecap="round" stroke-linejoin="round"></path><!--]--><!----><defs><linearGradient id="mist-0a400740-cc18-48f3-b16c-9099b8f83402"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-0a400740-cc18-48f3-b16c-9099b8f83402"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-0a400740-cc18-48f3-b16c-9099b8f83402"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-0a400740-cc18-48f3-b16c-9099b8f83402"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!--]--><!----></a><!----><!--]--></div><!--]--></div><!--]--><!----></div></div><!--]--><!----></div><!----><!----><!----><!----></div><!----><!----></div> <!----><div class="grid grid-flow-col-dense sm:grid-cols-page sm:gap-x-4 max-sm:grid-cols-1 max-sm:px-5 max-sm:gap-y-4 bg-deep-darker-blue blog-footer lg:gap-x-4 py-10 lg:py-18 columns temporal-grid svelte-iff7rv"><!----><div class="max-sm:col-span-1 sm:col-span-10 max-sm:col-start-1 sm:col-start-3 col-span-10 col-start-3 lg:col-span-8 lg:col-start-5"><!--[--><div class="blog-footer svelte-18jd5ta"><h2 class="svelte-18jd5ta">More Posts</h2> <!--[0--><ul><!--[--><li class="related-post svelte-18jd5ta"><div class="related-post-info svelte-18jd5ta"><!--[0--><span class="date svelte-18jd5ta">Jun 25, 2026</span><!--]--> <!--[0--><a class="category svelte-18jd5ta" href="/blog/categories/how-to">How-To</a><!--]--> <!--[0--><span class="duration svelte-18jd5ta">11 MIN READ</span><!--]--></div> <!--[0--><p class="related-post-title svelte-18jd5ta"><a href="/blog/getting-the-most-out-of-the-billable-action-count-metric" class="svelte-18jd5ta">Getting the most out of the Billable Action Count metric</a></p><!--]--></li><li class="related-post svelte-18jd5ta"><div class="related-post-info svelte-18jd5ta"><!--[0--><span class="date svelte-18jd5ta">Jun 03, 2026</span><!--]--> <!--[0--><a class="category svelte-18jd5ta" href="/blog/categories/how-to">How-To</a><!--]--> <!--[0--><span class="duration svelte-18jd5ta">6 MIN READ</span><!--]--></div> <!--[0--><p class="related-post-title svelte-18jd5ta"><a href="/blog/how-to-build-deep-research-agents-using-temporal-and-braintrust" class="svelte-18jd5ta">How to build deep research agents using Temporal and Braintrust</a></p><!--]--></li><!--]--></ul><!--]--></div><!--]--><!----></div><!----><!----><!----></div><!----><!----> <!--[0--><!----><div class="grid grid-flow-col-dense sm:grid-cols-page sm:gap-x-4 max-sm:grid-cols-1 max-sm:px-5 max-sm:gap-y-4 bg-customer-hype-rainbow no-padding gap-y-12 max-md:py-24 md:py-48 xl:py-80 columns temporal-grid svelte-iff7rv"><!----><div class="max-sm:col-span-1 sm:col-span-10 max-sm:col-start-1 sm:col-start-3 flex flex-col flex-wrap justify-center items-center text-center gap-8"><!--[--><!--[-1--><!--]--> <!--[0--><!----><p data-variant="display" data-size="128" class="text s128 customer-hype-rainbow display text-off-white svelte-1i5vmsf" data-contentful-entry-id="promo-block" data-contentful-field-id="title"><!--[--><!---->Build invincible applications<!--]--><!----></p><!----><!--]--> <!--[0--><!----><p data-variant="body" data-size="24" class="text s24 customer-hype-rainbow body text-off-white svelte-1i5vmsf" data-contentful-entry-id="promo-block" data-contentful-field-id="description"><!--[--><!---->It sounds like magic, we promise it's not.<!--]--><!----></p><!----><!--]--> <!--[0--><div class="flex max-xl:flex-col xl:flex-row items-center justify-center gap-4 mt-4"><!--[--><!----><a href="https://docs.temporal.io" data-contentful-entry-id="call-to-action-documentation" class="button group w-fit secondary center regular ultraviolet svelte-103wh5p"><!--[1--><!--[0--><svg width="24" height="24" class="transition-colors text-indigo-300 group-hover:text-off-white" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M1.5 4.125C1.5 1.84688 3.34688 0 5.625 0H21.375H22.5V1.125V17.625V18.75H21.375H21V21.75H21.375H22.5V24H21.375H5.25C3.17812 24 1.5 22.3219 1.5 20.25C1.5 20.1234 1.50469 19.9969 1.51875 19.875H1.5V4.125ZM3.75 20.25C3.75 21.0797 4.42031 21.75 5.25 21.75H18.75V18.75H5.25C4.42031 18.75 3.75 19.4203 3.75 20.25ZM3.75 16.8141C4.20937 16.6125 4.71563 16.5 5.25 16.5H20.25V2.25H5.625C4.58906 2.25 3.75 3.08906 3.75 4.125V16.8141ZM9 5.25L9.75 3.75L10.5 5.25L12 6L10.5 6.75L9.75 8.25L9 6.75L7.5 6L9 5.25ZM13.875 10.125L15 7.5L16.125 10.125L18.75 11.25L16.125 12.375L15 15L13.875 12.375L11.25 11.25L13.875 10.125Z"></path><!--]--><!----><defs><linearGradient id="mist-0bf91616-081e-4fcb-a9f7-6cc005fdacb7"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-0bf91616-081e-4fcb-a9f7-6cc005fdacb7"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-0bf91616-081e-4fcb-a9f7-6cc005fdacb7"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-0bf91616-081e-4fcb-a9f7-6cc005fdacb7"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!--]--> <!--[0--><!---->Documentation<!----><!--]--> <!--[-1--><!--]--><!----></a><!----><!----><a href="https://github.com/temporalio" target="_blank" rel="noopener noreferrer nofollow" data-contentful-entry-id="call-to-action-codebase" class="button group w-fit secondary center regular ultraviolet svelte-103wh5p"><!--[1--><!--[0--><svg width="24" height="24" class="transition-colors text-indigo-300 group-hover:text-off-white" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M3.75 2.25V5.25H6.75V2.25H3.75ZM1.5 0H3.75H6.75H9V2.25V5.25V7.5H6.75H6.375V12C7.31719 11.2922 8.48438 10.875 9.75 10.875H14.25C16.1156 10.875 17.625 9.36562 17.625 7.5H17.25H15V5.25V2.25V0H17.25H20.25H22.5V2.25V5.25V7.5H20.25H19.875C19.875 10.6078 17.3578 13.125 14.25 13.125H9.75C7.88438 13.125 6.375 14.6344 6.375 16.5H6.75H9V18.75V21.75V24H6.75H3.75H1.5V21.75V18.75V16.5H3.75H4.125V7.5H3.75H1.5V5.25V2.25V0ZM6.75 18.75H3.75V21.75H6.75V18.75ZM20.25 2.25H17.25V5.25H20.25V2.25Z"></path><!--]--><!----><defs><linearGradient id="mist-e629c1e3-b67e-4366-97e0-3440999562ba"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-e629c1e3-b67e-4366-97e0-3440999562ba"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-e629c1e3-b67e-4366-97e0-3440999562ba"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-e629c1e3-b67e-4366-97e0-3440999562ba"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!--]--> <!--[0--><!---->Code Base<!----><!--]--> <!--[-1--><!--]--><!----></a><!----><!----><a href="https://learn.temporal.io/examples" data-contentful-entry-id="call-to-action-samples" class="button group w-fit secondary center regular ultraviolet svelte-103wh5p"><!--[1--><!--[0--><svg width="24" height="24" class="transition-colors text-indigo-300 group-hover:text-off-white" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M14.1188 3.0675L13.8451 3.9225L8.44509 20.7225L8.17134 21.5775L9.88509 22.1288L10.1588 21.2738L15.5588 4.47375L15.8326 3.61875L14.1188 3.0675ZM16.4288 8.14126L17.0851 8.75625L21.1838 12.6L17.0851 16.4438L16.4288 17.0588L17.6588 18.3713L18.3151 17.7563L23.1151 13.2563L23.8163 12.6L23.1151 11.9438L18.3151 7.44375L17.6588 6.82875L16.4288 8.14126ZM6.34134 6.82875L5.68509 7.44375L0.885088 11.9438L0.183838 12.6L0.885088 13.2563L5.68509 17.7563L6.34134 18.3713L7.57134 17.0588L6.91509 16.4438L2.81634 12.6L6.91509 8.75625L7.57134 8.14126L6.34134 6.82875Z"></path><!--]--><!----><defs><linearGradient id="mist-b61f4f64-8bf4-458f-8c30-03b95f5b5424"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-b61f4f64-8bf4-458f-8c30-03b95f5b5424"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-b61f4f64-8bf4-458f-8c30-03b95f5b5424"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-b61f4f64-8bf4-458f-8c30-03b95f5b5424"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!--]--> <!--[0--><!---->Samples<!----><!--]--> <!--[-1--><!--]--><!----></a><!----><!--]--></div><!--]--><!--]--><!----></div><!----><!----> <!--[-1--><!--]--><!----><!----></div><!----><!--]--><!----><!--]--><!--]--><!--]--><!----><!--]--><!--]--><!----></main> <footer id="site-footer" class="footer flex flex-col bg-space-black"><div class="footer-content flex flex-col py-8 lg:py-20"><!----><div class="grid grid-flow-col-dense sm:grid-cols-page sm:gap-x-4 max-sm:grid-cols-1 max-sm:px-5 max-sm:gap-y-4 bg-none bg-transparent columns temporal-grid svelte-iff7rv"><!----><div class="max-sm:col-span-1 sm:col-span-10 max-sm:col-start-1 sm:col-start-3 flex flex-col md:flex-row items-start md:items-end justify-between mb-8 md:mb-16 gap-4 md:gap-8"><!--[--><div class="flex flex-col gap-4"><a href="https://status.temporal.io/" class="status-badge none svelte-19yr774"><!----><p data-variant="mono" data-size="16" class="text s16 mono !font-light font-mono !text-xs !tracking-[0.1rem] text-mint svelte-1i5vmsf"><!--[--><!---->ALL SYSTEMS OPERATIONAL<!--]--><!----></p><!----><!----></a><!----> <!----><p data-variant="body" data-size="24" class="text s24 body svelte-1i5vmsf"><!--[--><!---->99.999% trailing 30-day uptime<!--]--><!----></p><!----><!----></div> <!--[0--><div class="w-full sm:w-auto"><!--[0--><!--[0--><div data-contentful-entry-id="6vvH5NYYvowiSs0UETAzJT" data-contentful-field-id="introContent" class="prose max-w-none default text-off-white"><!--[0--><!---->Sign up for our newsletter<!----><!--]--></div> <dialog class="modal svelte-1bjjo3r modal-backdrop-heavy" aria-label="Zoomed image" aria-modal="true"><!--[-1--><!--]--></dialog><!----><!--]--> <div class="newsletter-signup small svelte-qgnxzt"><!--[-1--><!--]--> <form class="dark svelte-qgnxzt"><label class="sr-only" for="1001">Email Address</label> <input value="" type="email" data-lpignore="" data-1p-ignore="" placeholder="Email Address" id="1001" required="" class="svelte-qgnxzt"/> <!--[-1--><button class="dark svelte-qgnxzt" type="submit" aria-label="Subscribe"><!--[0--><svg width="24" height="24" class="" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M0 13.5L2.60625 14.6156L9 17.3578V17.625V20.625V23.25L11.25 24L14.6859 19.7953L18.8531 21.5812L21 22.5L21.3094 20.1844L23.6297 2.78906L24 0L21.5438 1.37812L2.47031 12.1125L0 13.5ZM19.1625 19.2656L12.5812 16.4437L20.8547 6.57187L19.1625 19.2656ZM10.4203 15.5203L5.07656 13.2281L18.8203 5.49375L10.4203 15.5203Z"></path><!--]--><!----><defs><linearGradient id="mist-b7c61a32-9bed-47ea-a574-67d897aa97fd"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-b7c61a32-9bed-47ea-a574-67d897aa97fd"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-b7c61a32-9bed-47ea-a574-67d897aa97fd"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-b7c61a32-9bed-47ea-a574-67d897aa97fd"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!----></button><!--]--></form> <!----></div><!----><!--]--> <!--[-1--><!--]--> <!--[-1--><!--]--></div><!--]--><!--]--><!----></div><!----><!----><!----></div><!----><!----> <!----><div class="grid grid-flow-col-dense sm:grid-cols-page sm:gap-x-4 max-sm:grid-cols-1 max-sm:px-5 max-sm:gap-y-4 bg-none bg-transparent columns temporal-grid svelte-iff7rv"><!----><div class="max-sm:col-span-1 sm:col-span-10 max-sm:col-start-1 sm:col-start-3 flex flex-wrap flex-col sm:flex-row justify-start xl:justify-between mb-8 md:mb-20 gap-6 sm:gap-x-12 lg:gap-x-20"><!--[--><!--[--><div class="footer-links svelte-wkopkf"><!----><p data-variant="body" data-size="20" class="text s20 body text-off-white svelte-1i5vmsf" data-contentful-entry-id="4epztNOZG4zdkSTcNzPL0s" data-contentful-field-id="title"><!--[--><!---->Discover<!--]--><!----></p><!----><!----> <!--[--><!--[-1--><!--[--><!--[0--><a href="/product" class="footer-link svelte-wkopkf">Overview</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="/how-it-works" class="footer-link svelte-wkopkf">How Temporal Works</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="/cloud" class="footer-link svelte-wkopkf">Temporal Cloud</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="/pricing" class="footer-link svelte-wkopkf">Pricing</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="/security" class="footer-link svelte-wkopkf">Security</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="https://trust.temporal.io" class="footer-link svelte-wkopkf">Trust Center</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="/startup" class="footer-link svelte-wkopkf">Startups</a><!--]--><!--]--><!--]--><!--]--> <div class="flex flex-col gap-4 pt-8 max-lg:hidden"><!--[--><!--]--></div></div><div class="footer-links svelte-wkopkf"><!----><p data-variant="body" data-size="20" class="text s20 body text-off-white svelte-1i5vmsf" data-contentful-entry-id="4TSqxuyMrVLtBrzEaGTAfA" data-contentful-field-id="title"><!--[--><!---->Explore<!--]--><!----></p><!----><!----> <!--[--><!--[-1--><!--[--><!--[0--><a href="/in-use" class="footer-link svelte-wkopkf">Customer Stories</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="https://learn.temporal.io/tutorials/" class="footer-link svelte-wkopkf">Project-based tutorials</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="https://learn.temporal.io/examples/" class="footer-link svelte-wkopkf">Example applications</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="/code-exchange" class="footer-link svelte-wkopkf">Code Exchange</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="/replay/2026" class="footer-link svelte-wkopkf">Replay 2026 Recap</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="https://pages.temporal.io/ask-an-expert" class="footer-link svelte-wkopkf">Ask an expert</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="/blog" class="footer-link svelte-wkopkf">Blog</a><!--]--><!--]--><!--]--><!--]--> <div class="flex flex-col gap-4 pt-8 max-lg:hidden"><!--[--><!--]--></div></div><div class="footer-links svelte-wkopkf"><!----><p data-variant="body" data-size="20" class="text s20 body text-off-white svelte-1i5vmsf" data-contentful-entry-id="7CY4xLfQDevf6lTYNpIQeN" data-contentful-field-id="title"><!--[--><!---->Developers<!--]--><!----></p><!----><!----> <!--[--><!--[-1--><!--[--><!--[0--><a href="https://docs.temporal.io/quickstarts" class="footer-link svelte-wkopkf">Getting Started with Temporal</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="https://docs.temporal.io/develop" class="footer-link svelte-wkopkf">Start building your next app</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="https://docs.temporal.io/cloud" class="footer-link svelte-wkopkf">Temporal Cloud docs</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="https://docs.temporal.io/production-deployment" class="footer-link svelte-wkopkf">Production deployments</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="https://learn.temporal.io/courses/temporal_101/" class="footer-link svelte-wkopkf">Temporal 101</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="https://learn.temporal.io/courses/temporal_102/" class="footer-link svelte-wkopkf">Temporal 102</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="https://learn.temporal.io/courses/intro_to_temporal_cloud/" class="footer-link svelte-wkopkf">Introduction to Temporal Cloud</a><!--]--><!--]--><!--]--><!--]--> <div class="flex flex-col gap-4 pt-8 max-lg:hidden"><!--[--><!--]--></div></div><div class="footer-links svelte-wkopkf"><!----><p data-variant="body" data-size="20" class="text s20 body text-off-white svelte-1i5vmsf" data-contentful-entry-id="4NydwQZlTc68NC2MfMnwu2" data-contentful-field-id="title"><!--[--><!---->Community<!--]--><!----></p><!----><!----> <!--[--><!--[-1--><!--[--><!--[0--><a href="https://t.mp/slack" target="_blank" rel="noopener noreferrer nofollow" class="footer-link svelte-wkopkf">Join our Slack group</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="/community" class="footer-link svelte-wkopkf">Find a meetup near you</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="https://community.temporal.io/" class="footer-link svelte-wkopkf">Community forum</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="/events" class="footer-link svelte-wkopkf">Events</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="/cpfc" class="footer-link svelte-wkopkf">Crystal Palace</a><!--]--><!--]--><!--]--><!--]--> <div class="flex flex-col gap-4 pt-8 max-lg:hidden"><!--[--><!--]--></div></div><div class="footer-links svelte-wkopkf"><!----><p data-variant="body" data-size="20" class="text s20 body text-off-white svelte-1i5vmsf" data-contentful-entry-id="53AeODgxpn6IHQwIaED2p8" data-contentful-field-id="title"><!--[--><!---->Company<!--]--><!----></p><!----><!----> <!--[--><!--[-1--><!--[--><!--[0--><a href="/about" class="footer-link svelte-wkopkf">About</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="/careers" class="footer-link svelte-wkopkf">Careers</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="/news" class="footer-link svelte-wkopkf">News</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="https://pages.temporal.io/contact-us" class="footer-link svelte-wkopkf">Contact us</a><!--]--><!--]--><!--]--><!--[-1--><!--[--><!--[0--><a href="/partners" class="footer-link svelte-wkopkf">Partners</a><!--]--><!--]--><!--]--><!--]--> <div class="flex flex-col gap-4 pt-8 max-lg:hidden"><!--[--><!--]--></div></div><!--]--><!--]--><!----></div><!----><!----><!----></div><!----><!----> <!----><div class="grid grid-flow-col-dense sm:grid-cols-page sm:gap-x-4 max-sm:grid-cols-1 max-sm:px-5 max-sm:gap-y-4 bg-none bg-transparent columns temporal-grid svelte-iff7rv"><!----><div class="max-sm:col-span-1 sm:col-span-10 max-sm:col-start-1 sm:col-start-3 flex flex-col-reverse md:flex-row gap-4 md:gap-6 md:items-center justify-between"><!--[--><div class="copyright font-light font-sans text-blue-gray-700 text-xs"><p>2026 © Temporal Technologies. All Rights Reserved.</p></div> <div class="legal svelte-1t4iaj7"><a href="/global-privacy-policy" class="svelte-1t4iaj7">Privacy Policy</a> <a href="/candidate-privacy-policy" class="svelte-1t4iaj7">Candidate Privacy Policy</a> <a href="/terms-of-service" class="svelte-1t4iaj7">Terms of Service</a> <a href="/code-of-conduct" class="svelte-1t4iaj7">Code of Conduct</a> <a href="/sitemap" class="svelte-1t4iaj7">Sitemap</a> <button type="button" class="svelte-1t4iaj7"><svg xmlns="http://www.w3.org/2000/svg" width="30" version="1.1" x="0px" y="0px" viewBox="0 0 30 14" aria-hidden="true" focusable="false"><g transform="translate(-1275.000000, -200.000000)"><g transform="translate(1275.000000, 200.000000)"><path d="M7.4,12.8h6.8l3.1-11.6H7.4C4.2,1.2,1.6,3.8,1.6,7S4.2,12.8,7.4,12.8z" fill="#fff"></path><path d="M22.6,0H7.4c-3.9,0-7,3.1-7,7s3.1,7,7,7h15.2c3.9,0,7-3.1,7-7S26.4,0,22.6,0z M1.6,7c0-3.2,2.6-5.8,5.8-5.8h9.9l-3.1,11.6H7.4C4.2,12.8,1.6,10.2,1.6,7z" fill="#0066FF"></path><path d="M24.6,4c0.2,0.2,0.2,0.6,0,0.8l0,0L22.5,7l2.2,2.2c0.2,0.2,0.2,0.6,0,0.8c-0.2,0.2-0.6,0.2-0.8,0l0,0l-2.2-2.2L19.5,10c-0.2,0.2-0.6,0.2-0.8,0c-0.2-0.2-0.2-0.6,0-0.8l0,0L20.8,7l-2.2-2.2c-0.2-0.2-0.2-0.6,0-0.8c0.2-0.2,0.6-0.2,0.8,0l0,0l2.2,2.2L23.8,4C24,3.8,24.4,3.8,24.6,4z" fill="#fff"></path><path d="M12.7,4.1c0.2,0.2,0.3,0.6,0.1,0.8l0,0L8.6,9.8C8.5,9.9,8.4,10,8.3,10c-0.2,0.1-0.5,0.1-0.7-0.1l0,0L5.4,7.7c-0.2-0.2-0.2-0.6,0-0.8c0.2-0.2,0.6-0.2,0.8,0l0,0L8,8.6l3.8-4.5C12,3.9,12.4,3.9,12.7,4.1z" fill="#0066FF"></path></g></g></svg> Your Privacy Choices</button></div><!----> <div class="social-links svelte-69xid7"><a class="social-link svelte-69xid7" target="_blank" href="https://www.youtube.com/temporalio" aria-label="Follow us on YouTube"><!--[0--><svg width="24" height="24" class="" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M 23.498438,5.81639 C 23.222439,4.7078 22.409253,3.8347 21.376599,3.53841 19.504978,3 12.000023,3 12.000023,3 c 0,0 -7.5050005,0 -9.3766687,0.53841 C 1.590775,3.83475 0.7775235,4.7078 0.5015053,5.81639 0,7.82578 0,12.0182 0,12.0182 c 0,0 0,4.1924 0.5015053,6.2018 0.2760182,1.1086 1.0892697,1.9453 2.121849,2.2416 C 4.4950225,21 12.000023,21 12.000023,21 c 0,0 7.504955,0 9.376576,-0.5384 1.032654,-0.2963 1.84584,-1.133 2.121839,-2.2416 C 24,16.2106 24,12.0182 24,12.0182 c 0,0 0,-4.19242 -0.501562,-6.20181 z M 9.545466,15.8246 V 8.2118 l 6.272706,3.8065 z"></path><!--]--><!----><defs><linearGradient id="mist-00a77574-83a7-4823-bb0b-bbce7013e332"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-00a77574-83a7-4823-bb0b-bbce7013e332"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-00a77574-83a7-4823-bb0b-bbce7013e332"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-00a77574-83a7-4823-bb0b-bbce7013e332"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!----></a> <a class="social-link svelte-69xid7" target="_blank" href="https://twitter.com/temporalio" aria-label="Follow us on Twitter"><!--[0--><svg width="24" height="24" class="" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M18.2437 2.25H21.5531L14.325 10.5094L22.8281 21.75H16.1719L10.9547 14.9344L4.99216 21.75H1.6781L9.40779 12.9141L1.25623 2.25H8.08123L12.7922 8.47969L18.2437 2.25ZM17.0812 19.7719H18.914L7.08279 4.125H5.11404L17.0812 19.7719Z"></path><!--]--><!----><defs><linearGradient id="mist-cccb4cc3-2200-4c2c-972c-976ef1fd6400"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-cccb4cc3-2200-4c2c-972c-976ef1fd6400"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-cccb4cc3-2200-4c2c-972c-976ef1fd6400"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-cccb4cc3-2200-4c2c-972c-976ef1fd6400"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!----></a> <a class="social-link svelte-69xid7" target="_blank" href="https://github.com/temporalio" aria-label="Follow us on GitHub"><!--[0--><svg width="24" height="24" class="" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M8.15156 18.6281C8.15156 18.7219 8.04375 18.7969 7.90781 18.7969C7.75312 18.8109 7.64531 18.7359 7.64531 18.6281C7.64531 18.5344 7.75313 18.4594 7.88906 18.4594C8.02969 18.4453 8.15156 18.5203 8.15156 18.6281ZM6.69375 18.4172C6.66094 18.5109 6.75469 18.6187 6.89531 18.6469C7.01719 18.6937 7.15781 18.6469 7.18594 18.5531C7.21406 18.4594 7.125 18.3516 6.98438 18.3094C6.8625 18.2766 6.72656 18.3234 6.69375 18.4172ZM8.76562 18.3375C8.62969 18.3703 8.53594 18.4594 8.55 18.5672C8.56406 18.6609 8.68594 18.7219 8.82656 18.6891C8.9625 18.6562 9.05625 18.5672 9.04219 18.4734C9.02812 18.3844 8.90156 18.3234 8.76562 18.3375ZM11.85 0.375C5.34844 0.375 0.375 5.31094 0.375 11.8125C0.375 17.0109 3.64688 21.4594 8.32031 23.025C8.92031 23.1328 9.13125 22.7625 9.13125 22.4578C9.13125 22.1672 9.11719 20.5641 9.11719 19.5797C9.11719 19.5797 5.83594 20.2828 5.14688 18.1828C5.14688 18.1828 4.6125 16.8187 3.84375 16.4672C3.84375 16.4672 2.77031 15.7312 3.91875 15.7453C3.91875 15.7453 5.08594 15.8391 5.72812 16.9547C6.75469 18.7641 8.475 18.2437 9.14531 17.9344C9.25313 17.1844 9.55781 16.6641 9.89531 16.3547C7.275 16.0641 4.63125 15.6844 4.63125 11.175C4.63125 9.88594 4.9875 9.23906 5.7375 8.41406C5.61562 8.10938 5.21719 6.85312 5.85938 5.23125C6.83906 4.92656 9.09375 6.49688 9.09375 6.49688C10.0312 6.23438 11.0391 6.09844 12.0375 6.09844C13.0359 6.09844 14.0438 6.23438 14.9813 6.49688C14.9813 6.49688 17.2359 4.92188 18.2156 5.23125C18.8578 6.85781 18.4594 8.10938 18.3375 8.41406C19.0875 9.24375 19.5469 9.89062 19.5469 11.175C19.5469 15.6984 16.7859 16.0594 14.1656 16.3547C14.5969 16.725 14.9625 17.4281 14.9625 18.5297C14.9625 20.1094 14.9484 22.0641 14.9484 22.4484C14.9484 22.7531 15.1641 23.1234 15.7594 23.0156C20.4469 21.4594 23.625 17.0109 23.625 11.8125C23.625 5.31094 18.3516 0.375 11.85 0.375ZM4.93125 16.5422C4.87031 16.5891 4.88438 16.6969 4.96406 16.7859C5.03906 16.8609 5.14687 16.8938 5.20781 16.8328C5.26875 16.7859 5.25469 16.6781 5.175 16.5891C5.1 16.5141 4.99219 16.4812 4.93125 16.5422ZM4.425 16.1625C4.39219 16.2234 4.43906 16.2984 4.53281 16.3453C4.60781 16.3922 4.70156 16.3781 4.73438 16.3125C4.76719 16.2516 4.72031 16.1766 4.62656 16.1297C4.53281 16.1016 4.45781 16.1156 4.425 16.1625ZM5.94375 17.8312C5.86875 17.8922 5.89687 18.0328 6.00469 18.1219C6.1125 18.2297 6.24844 18.2437 6.30937 18.1688C6.37031 18.1078 6.34219 17.9672 6.24844 17.8781C6.14531 17.7703 6.00469 17.7563 5.94375 17.8312ZM5.40938 17.1422C5.33437 17.1891 5.33437 17.3109 5.40938 17.4188C5.48438 17.5266 5.61094 17.5734 5.67188 17.5266C5.74687 17.4656 5.74687 17.3438 5.67188 17.2359C5.60625 17.1281 5.48438 17.0813 5.40938 17.1422Z"></path><!--]--><!----><defs><linearGradient id="mist-316d7cfb-66ac-4a0c-86e3-6ebd821c5e16"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-316d7cfb-66ac-4a0c-86e3-6ebd821c5e16"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-316d7cfb-66ac-4a0c-86e3-6ebd821c5e16"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-316d7cfb-66ac-4a0c-86e3-6ebd821c5e16"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!----></a> <a class="social-link svelte-69xid7" target="_blank" href="https://www.linkedin.com/company/temporal-technologies" aria-label="Follow us on Linkedin"><!--[0--><svg width="24" height="24" class="" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M6.20062 21H1.84688V6.97971H6.20062V21ZM4.02141 5.06721C2.62922 5.06721 1.5 3.91408 1.5 2.52189C1.5 1.85318 1.76565 1.21185 2.2385 0.738991C2.71136 0.266136 3.35269 0.000488281 4.02141 0.000488281C4.69012 0.000488281 5.33145 0.266136 5.80431 0.738991C6.27716 1.21185 6.54281 1.85318 6.54281 2.52189C6.54281 3.91408 5.41313 5.06721 4.02141 5.06721ZM22.4953 21H18.1509V14.175C18.1509 12.5485 18.1181 10.4625 15.8873 10.4625C13.6237 10.4625 13.2769 12.2297 13.2769 14.0578V21H8.92781V6.97971H13.1034V8.89221H13.1644C13.7456 7.79064 15.1655 6.62814 17.2838 6.62814C21.69 6.62814 22.5 9.52971 22.5 13.2985V21H22.4953Z"></path><!--]--><!----><defs><linearGradient id="mist-aa29f992-c09b-473e-a194-2c643fff7da4"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-aa29f992-c09b-473e-a194-2c643fff7da4"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-aa29f992-c09b-473e-a194-2c643fff7da4"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-aa29f992-c09b-473e-a194-2c643fff7da4"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!----></a> <a class="social-link svelte-69xid7" target="_blank" href="https://t.mp/slack" aria-label="Join us on Slack"><!--[0--><svg width="24" height="24" class="" viewBox="0 0 24 24" fill="currentcolor" stroke-width="2" xmlns="http://www.w3.org/2000/svg" role="img" data-contentful-entry-id="" data-contentful-field-id=""><!--[--><path d="M5.91188 14.7703C5.91188 15.9844 4.92 16.9762 3.70594 16.9762C2.49188 16.9762 1.5 15.9844 1.5 14.7703C1.5 13.5563 2.49188 12.5644 3.70594 12.5644H5.91188V14.7703ZM7.02375 14.7703C7.02375 13.5563 8.01562 12.5644 9.22969 12.5644C10.4437 12.5644 11.4356 13.5563 11.4356 14.7703V20.2941C11.4356 21.5081 10.4437 22.5 9.22969 22.5C8.01562 22.5 7.02375 21.5081 7.02375 20.2941V14.7703ZM9.22969 5.91188C8.01562 5.91188 7.02375 4.92 7.02375 3.70594C7.02375 2.49187 8.01562 1.5 9.22969 1.5C10.4437 1.5 11.4356 2.49187 11.4356 3.70594V5.91188H9.22969ZM9.22969 7.02375C10.4437 7.02375 11.4356 8.01562 11.4356 9.22969C11.4356 10.4437 10.4437 11.4356 9.22969 11.4356H3.70594C2.49188 11.4356 1.5 10.4437 1.5 9.22969C1.5 8.01562 2.49188 7.02375 3.70594 7.02375H9.22969ZM18.0881 9.22969C18.0881 8.01562 19.08 7.02375 20.2941 7.02375C21.5081 7.02375 22.5 8.01562 22.5 9.22969C22.5 10.4437 21.5081 11.4356 20.2941 11.4356H18.0881V9.22969ZM16.9762 9.22969C16.9762 10.4437 15.9844 11.4356 14.7703 11.4356C13.5563 11.4356 12.5644 10.4437 12.5644 9.22969V3.70594C12.5644 2.49187 13.5563 1.5 14.7703 1.5C15.9844 1.5 16.9762 2.49187 16.9762 3.70594V9.22969ZM14.7703 18.0881C15.9844 18.0881 16.9762 19.08 16.9762 20.2941C16.9762 21.5081 15.9844 22.5 14.7703 22.5C13.5563 22.5 12.5644 21.5081 12.5644 20.2941V18.0881H14.7703ZM14.7703 16.9762C13.5563 16.9762 12.5644 15.9844 12.5644 14.7703C12.5644 13.5563 13.5563 12.5644 14.7703 12.5644H20.2941C21.5081 12.5644 22.5 13.5563 22.5 14.7703C22.5 15.9844 21.5081 16.9762 20.2941 16.9762H14.7703Z"></path><!--]--><!----><defs><linearGradient id="mist-a514320d-1d3a-4d60-bee2-26b2925e2875"><stop offset="0%" stop-color="#34D399"></stop><stop offset="100%" stop-color="#FF6BFF"></stop></linearGradient><linearGradient id="green-gradient-a514320d-1d3a-4d60-bee2-26b2925e2875"><stop offset="0%" stop-color="#C3FF62"></stop><stop offset="100%" stop-color="#1FF1A5"></stop></linearGradient><linearGradient id="pink-gradient-a514320d-1d3a-4d60-bee2-26b2925e2875"><stop offset="0%" stop-color="#FF5555"></stop><stop offset="100%" stop-color="#B664FF"></stop></linearGradient><linearGradient id="purple-ultraviolet-gradient-a514320d-1d3a-4d60-bee2-26b2925e2875"><stop offset="0%" stop-color="#B664FF"></stop><stop offset="100%" stop-color="#444CE7"></stop></linearGradient></defs></svg><!--]--><!----></a></div><!----><!--]--><!----></div><!----><!----><!----></div><!----><!----></div></footer><!----></div> <temporal-consent-banner geo-endpoint="/api/geo"></temporal-consent-banner><!----><!--]--><!--]--><!----></div> <!----> <!--[-1--><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
<script>
{
__sveltekit_m7vlw0 = {
base: new URL("..", location).pathname.slice(0, -1)
};
const element = document.currentScript.parentElement;
Promise.all([
import("../_app/immutable/entry/start.DHXlI2X5.js"),
import("../_app/immutable/entry/app.CLIeNjXY.js")
]).then(([kit, app]) => {
kit.start(app, element, {
node_ids: [0, 7, 12, 46],
data: [null,(function(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z,A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R){a.linkText="Overview";a.link="/product";a.entityId="5d7x2oU3GnKl2PwRHfFMuR";a.contentType="footerLink";b.linkText="How Temporal Works";b.link="/how-it-works";b.entityId="3MXMlddUcrW8nOsJCpEDUx";b.contentType="footerLink";c.linkText="Temporal Cloud";c.link="/cloud";c.entityId="2JeJYErsmcssv06hMrfwiF";c.contentType="footerLink";d.linkText="Security";d.link="/security";d.entityId="4meokZHNYwid3e236uZKIJ";d.contentType="footerLink";e.linkText="Customer Stories";e.link="/in-use";e.entityId="5NHM0cToFOIOfWEKhILN5K";e.contentType="footerLink";f[0]={linkText:"AI",link:"/solutions/ai",entityId:"3TlMgadJMFwa4WE2jYMXjZ",contentType:"footerLink"};f[1]={linkText:"Financial Services",link:"/solutions/financial-services",entityId:"2aD760BTctiSuChV74SVwl",contentType:"footerLink"};g[0]={linkText:"Platform Engineering",link:"/solutions/platform-engineering",entityId:"3H3pNmqq7jm9FQ6UK4100V",contentType:"footerLink"};g[1]={linkText:"Startups",link:"/startup",entityId:"4xqwrRciKtVF6u7s1H09QE",contentType:"footerLink"};g[2]={linkText:"Partners",link:"/partners",entityId:"22Jz9PiLUIpO3te9ZZm2mb",contentType:"footerLink"};h.linkText="Resource Library";h.link="/resources";h.entityId="2jjwhzfbZvR90Zd1ylRlgS";h.contentType="footerLink";i.linkText="Learn Temporal";i.link="https://learn.temporal.io";i.entityId="6AG3qtWngBbv7iyqhZfUZv";i.contentType="footerLink";j.linkText="Community";j.link="/community";j.entityId="2ZOhNmQn108V8jKiPBQ1QD";j.contentType="footerLink";k.linkText="Code Exchange";k.link="/code-exchange";k.entityId="4E03OqVVyu2407sYn3Cmb3";k.contentType="footerLink";l.linkText="Blog";l.link="/blog";l.entityId="48YgmuEOMCfyNiOG6fKuIi";l.contentType="footerLink";m.linkText="Changelog";m.link="/changelog";m.entityId="6D5newtErYVln01crlUkJ3";m.contentType="footerLink";n.linkText="Overview";n.link="/product";n.entityId="5d7x2oU3GnKl2PwRHfFMuR";n.contentType="footerLink";o.linkText="How Temporal Works";o.link="/how-it-works";o.entityId="5JvBIqkZjDTveG8rBLGk98";o.contentType="footerLink";p.linkText="Temporal Cloud";p.link="/cloud";p.entityId="7a4w9eJq6SDHLzFBCbPS4";p.contentType="footerLink";q.linkText="Pricing";q.link="/pricing";q.entityId="4mAmb12lsMPKYsVZhppKwL";q.contentType="footerLink";r.linkText="Security";r.link="/security";r.entityId="2jeP3NX1QybaIff6VqswXu";r.contentType="footerLink";s.linkText="Trust Center";s.link="https://trust.temporal.io";s.entityId="6JvstGMvrnsU3k7jr4X44s";s.contentType="footerLink";t.linkText="Startups";t.link="/startup";t.entityId="4xqwrRciKtVF6u7s1H09QE";t.contentType="footerLink";u.linkText="Customer Stories";u.link="/in-use";u.entityId="5NHM0cToFOIOfWEKhILN5K";u.contentType="footerLink";v.linkText="Project-based tutorials";v.link="https://learn.temporal.io/tutorials/";v.entityId="49m9nbzR6ESqyTPDQes3Bq";v.contentType="footerLink";w.linkText="Example applications";w.link="https://learn.temporal.io/examples/";w.entityId="4gSShzYEMpRvjMVTm1Z2Zm";w.contentType="footerLink";x.linkText="Code Exchange";x.link="/code-exchange";x.entityId="4E03OqVVyu2407sYn3Cmb3";x.contentType="footerLink";y.linkText="Replay 2026 Recap";y.link="/replay/2026";y.entityId="WKxFa7VtBYoFpxbXBmLmE";y.contentType="footerLink";z.linkText="Ask an expert";z.link="https://pages.temporal.io/ask-an-expert";z.entityId="8EwBCGoohcDZnaSssqVwo";z.contentType="footerLink";A.linkText="Blog";A.link="/blog";A.entityId="48YgmuEOMCfyNiOG6fKuIi";A.contentType="footerLink";B.linkText="Getting Started with Temporal";B.link="https://docs.temporal.io/quickstarts";B.entityId="3MakYxt6SSrjcQoHlsMVNd";B.contentType="footerLink";C.linkText="Start building your next app";C.link="https://docs.temporal.io/develop";C.entityId="ReUR12o5GJ7TtRT9hqUtb";C.contentType="footerLink";D.linkText="Temporal Cloud docs";D.link="https://docs.temporal.io/cloud";D.entityId="RzIgM3E3E06xvtI47tS4w";D.contentType="footerLink";E.linkText="Production deployments";E.link="https://docs.temporal.io/production-deployment";E.entityId="2OeOkTFbu4dsa3o86XzvP2";E.contentType="footerLink";F.linkText="Temporal 101";F.link="https://learn.temporal.io/courses/temporal_101/";F.entityId="4AhlRbcIYx2cdSgCGmUK8p";F.contentType="footerLink";G.linkText="Temporal 102";G.link="https://learn.temporal.io/courses/temporal_102/";G.entityId="3XL0r7EBbIw273NdTkW3F2";G.contentType="footerLink";H.linkText="Introduction to Temporal Cloud";H.link="https://learn.temporal.io/courses/intro_to_temporal_cloud/";H.entityId="3cCsE7bvSqYTOzUMeNk4Ml";H.contentType="footerLink";I.linkText="Join our Slack group";I.link="https://t.mp/slack";I.entityId="1xMVykhhiEWRodXOuPnFIW";I.contentType="footerLink";J.linkText="Find a meetup near you";J.link="/community";J.entityId="2tvkQdG09YTz1isJPXD1U1";J.contentType="footerLink";K.linkText="Community forum";K.link="https://community.temporal.io/";K.entityId="7oVm0JFGanegsXT4bOOjCJ";K.contentType="footerLink";L.linkText="Events";L.link="/events";L.entityId="1iTUuLxKMc9djxa9WuGGM2";L.contentType="footerLink";M.linkText="Crystal Palace";M.link="/cpfc";M.entityId="2xC1ZeJ7tRqhZJZS708fMe";M.contentType="footerLink";N.linkText="About";N.link="/about";N.entityId="1elL6e3d8pGbpBqDKKaJ4h";N.contentType="footerLink";O.linkText="Careers";O.link="/careers";O.entityId="17PUUqXrw2P6vdNHdMKuWp";O.contentType="footerLink";P.linkText="News";P.link="/news";P.entityId="1jpUgFD5yQTsIGkbEGTTa7";P.contentType="footerLink";Q.linkText="Contact us";Q.link="https://pages.temporal.io/contact-us";Q.entityId="2BPNJwBWqxI8tta08ppxou";Q.contentType="footerLink";R.linkText="Partners";R.link="/partners";R.entityId="22Jz9PiLUIpO3te9ZZm2mb";R.contentType="footerLink";return {type:"data",data:{globalSettings:{mainMenu:[{title:"Platform",links:[a,b,c,d],sections:[{links:[a]},{links:[b]},{links:[c]},{links:[d]}],callsToAction:[],entityId:"1rq371AgLqwEnh8eclWZ4z",contentType:"footerLinkGroup"},{linkText:"Docs",link:"https://docs.temporal.io",entityId:"5GIUmSUc4tuKTWmFIx7iJb",contentType:"footerLink"},{linkText:"Pricing",link:"/pricing",entityId:"4mAmb12lsMPKYsVZhppKwL",contentType:"footerLink"},{title:"Use Cases",links:[e,{title:"By Industry",links:f,entityId:"1X2fJCkbFD446C75dbaJWC",contentType:"footerLinkGroup"},{title:"By Audience",links:g,entityId:"5nw7mkYFNip0G99aGTfyNQ",contentType:"footerLinkGroup"}],sections:[{links:[e]},{title:"By Industry",links:f},{title:"By Audience",links:g}],callsToAction:[],entityId:"1e95s28bgM2FkcA50IHhyo",contentType:"footerLinkGroup"},{title:"Resources",links:[h,i,j,k,l,m],sections:[{links:[h]},{links:[i]},{links:[j]},{links:[k]},{links:[l]},{links:[m]}],callsToAction:[],entityId:"5MBgd98sk9VXlbqC9UhGJS",contentType:"footerLinkGroup"}],topBarMenu:[{linkText:"About",link:"/about",entityId:"1elL6e3d8pGbpBqDKKaJ4h",contentType:"footerLink"},{linkText:"Careers",link:"/careers",entityId:"lAfwVDQKE4T4PGM7gaeZy",contentType:"footerLink"},{linkText:"Talk to Sales",link:"https://pages.temporal.io/contact-us",entityId:"1kW37omCzrZFFzaoBwW2VG",contentType:"footerLink"}],menuCtas:[{text:"Try Free",href:"/get-cloud",variant:"primary",leadingIcon:null,trailingIcon:null,entityId:"79w1QMsJg2MTAVGlu2jdss",size:"small",theme:"ultraviolet"},{text:"Log In",href:"https://cloud.temporal.io/",variant:"secondary",leadingIcon:null,trailingIcon:null,entityId:"KAPnScSjIJkz74RbczQ2F",size:"small",theme:"ultraviolet"}],footerLinks:[{title:"Discover",links:[n,o,p,q,r,s,t],sections:[{links:[n]},{links:[o]},{links:[p]},{links:[q]},{links:[r]},{links:[s]},{links:[t]}],callsToAction:[],entityId:"4epztNOZG4zdkSTcNzPL0s",contentType:"footerLinkGroup"},{title:"Explore",links:[u,v,w,x,y,z,A],sections:[{links:[u]},{links:[v]},{links:[w]},{links:[x]},{links:[y]},{links:[z]},{links:[A]}],callsToAction:[],entityId:"4TSqxuyMrVLtBrzEaGTAfA",contentType:"footerLinkGroup"},{title:"Developers",links:[B,C,D,E,F,G,H],sections:[{links:[B]},{links:[C]},{links:[D]},{links:[E]},{links:[F]},{links:[G]},{links:[H]}],callsToAction:[],entityId:"7CY4xLfQDevf6lTYNpIQeN",contentType:"footerLinkGroup"},{title:"Community",links:[I,J,K,L,M],sections:[{links:[I]},{links:[J]},{links:[K]},{links:[L]},{links:[M]}],callsToAction:[],entityId:"4NydwQZlTc68NC2MfMnwu2",contentType:"footerLinkGroup"},{title:"Company",links:[N,O,P,Q,R],sections:[{links:[N]},{links:[O]},{links:[P]},{links:[Q]},{links:[R]}],callsToAction:[],entityId:"53AeODgxpn6IHQwIaED2p8",contentType:"footerLinkGroup"}],footerForm:{formId:1001,title:"Newsletter Form",errorMessage:"Please enter a valid email address.",successMessage:"Thank you for signing up for our newsletter!",introContent:"\u003Cp>Sign up for our community newsletter\u003C/p>",background:"dark",alignment:"left",contentType:"marketoForm",entityId:"6vvH5NYYvowiSs0UETAzJT",postRollPageUrl:void 0,anchor:void 0},defaultBlogPromoCard:{title:"Get Started with Cloud Promo",eyebrow:"Temporal Cloud",heading:"Ready to see for yourself?",content:"Sign up for Temporal Cloud today and get $1,000 in free credits.",callsToAction:[{text:"Get started",href:"/get-cloud",variant:"primary",leadingIcon:null,trailingIcon:"arrow-right",entityId:"7wKJdxc4JeeNXHCliDfqVV",size:"regular",theme:"pink"}],entityId:"2SJj3BBBzkr6FefI6ZfRox",contentType:"noise"},resourcesPreFooter:void 0},banner:{preLinkText:"Faster is easy with AI. Better takes more | ",link:"/reports/state-of-development-2026",linkToText:"Read the State of Development 2026 to learn what makes AI agents succeed ›",postLinkText:void 0,topBarMenu:[],publishDate:"2026-08-31T06:00-08:00",backgroundColor:void 0,textColor:void 0,linkColor:void 0,backgroundImage:void 0,contentType:"banner"},githubStats:{stars:22983,forks:1894,watchers:123,pulls:8441,issues:1083}},uses:{}}}({},{},{},{},{},Array(2),Array(3),{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{})),{type:"data",data:{post:{title:"Building application reliability on top of infrastructure unreliability",content:"\u003Cp>What do you do when you know, with absolute certainty, that the infrastructure you're running something on is unreliable? Hardware always fails eventually, networks go down, or software needs to be patched.\u003C/p>\n\u003Cp>Even under perfectly ordinary operation, your runtime environment could drop out from under the running process: traffic drops, causing the autoscaler to downscale and, unless you've tied into \u003Ca href=\"https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-termination\" rel=\"noopener\" target=\"_blank\">the termination lifecycle\u003C/a>, the process is forcibly killed. But whether purposeful or not, it's a computing fact that processes can abruptly end and be restarted.\u003C/p>\n\u003Cp>For many kinds of services, that's perfectly acceptable; give the process a chance to drain outstanding requests, and then new requests will land on other nodes. Or, if that process unexpectedly dies or is forcibly ended, the client can retry and have its request handled elsewhere.\u003C/p>\n\u003Cp>Put another way, things can be made \"effectively reliable\" from an infrastructure perspective: for example, Kubernetes helps ensure that nodes and processes exist to run application code, and \u003Ca href=\"https://istio.io/\" rel=\"noopener\" target=\"_blank\">Istio\u003C/a> improves network reliability and performance. Things still fail, but we can make it tolerable with well-known tools.\u003C/p>\n\u003Cp>But perhaps a more important question is: what's \u003Cem>actually\u003C/em> being run? As in, what is the full application trying to accomplish?\u003C/p>\n\u003Ch2 id=\"an-example-application\">An Example Application\u003Ca href=\"#an-example-application\">\u003Cspan class=\"ml-4\">#\u003C/span>\u003C/a>\u003C/h2>\n\u003Cp>Building something from the bottom up is a natural way to go; assume that things will fail (always a safe assumption), and design the system from there. But this makes it hard to build and keep track of a multi-step application, like an e-commerce store. Let's take a look.\u003C/p>\n\u003Cp>Consider how this application might look from the customer's perspective. It's a relatively short series of steps:\u003C/p>\n\u003Col>\n \u003Cli>Add item to cart\u003C/li>\n \u003Cli>Input payment\u003C/li>\n \u003Cli>Input shipping address\u003C/li>\n \u003Cli>Confirm and checkout\u003C/li>\n \u003Cli>Track order\u003C/li>\n \u003Cli>Receive item\u003C/li>\n\u003C/ol>\n\u003Cp>Software engineering \u003Ca href=\"https://en.wikipedia.org/wiki/Microservices#History\" rel=\"noopener\" target=\"_blank\">over the past couple of decades\u003C/a> has taught us that nothing is simple. Namely, because we've made our infrastructures \u003Cem>effectively reliable\u003C/em> through tools like Kubernetes and patterns like stateless services, a \"simple\" workflow like the above ends up being composed of dozens of microservices (\u003Ca href=\"https://twitter.com/Werner/status/741673514567143424\" rel=\"noopener\" target=\"_blank\">or more\u003C/a>).\u003C/p>\n\u003Cp>\n \u003Cimg src=\"//images.ctfassets.net/0uuz8ydxyd9p/5OHtgsWdkhnR4jCu6lLHyv/11bc45a5478d37956f176e237731b4b7/image2.png-inverted.png\" alt=\"mock ecommerce microservices\">\n \u003Cem>Caption: A hypothetical but realistic microservices connection graph for placing an e-commerce order.\u003C/em>\n\u003C/p>\n\u003Cp>That's not all, though, because behind and in between these microservices also ends up being a complex mix of databases, caches, event streams, and message queues. While these services all have unique functions, they're all dealing with the same orders, and \u003Cem>hopefully\u003C/em> each order's details and status are appropriately picked up by the appropriate next service.\u003C/p>\n\u003Cp>Thus, tracing a single order through this dizzying system becomes a daunting or even impossible task. While this is necessary for tasks like ensuring orders aren't dropped, providing customer service, or auditing for regulatory compliance, it's also important to answer a more fundamental question: How reliable is our application?\u003C/p>\n\u003Ch2 id=\"applications-need-reliability-too\">Applications Need Reliability Too\u003Ca href=\"#applications-need-reliability-too\">\u003Cspan class=\"ml-4\">#\u003C/span>\u003C/a>\u003C/h2>\n\u003Cp>At some point, that question will come up. And I'm sure you've had the experience of trying to calculate some kind of overall reliability number, only to find that a single number like \u003Ccode>99.99%\u003C/code> doesn't adequately represent the full story of the application.\u003C/p>\n\u003Cp>Let's take a look at a simplified example.\u003C/p>\n\u003Cp>\n Suppose you have a two-step process, with each step handled by separate microservices. For the sake of example, let's say the process is painting a bike shed, and the two steps are (1) choose and reserve a paint color and (2) apply paint to the shed. Step 2 cannot happen without Step 1 occurring first. But if we have two sheds, the microservices pattern allows the application to handle the two sheds independently from each other: Step 1 could be working on picking a paint color for Shed B without affecting Step 2's painting of Shed A.\n \n \u003Cimg src=\"//images.ctfassets.net/0uuz8ydxyd9p/3DhYNMxnRMXwzjxTWxuv8f/933a570168f0055119971f0f537231f9/image9.png-inverted.png\" alt=\"step 1 - 2\">\n\u003C/p>\n\u003Cp>Reasoning about the reliability of this application is really asking: What is the likelihood of a given shed successfully making it through both steps?\u003C/p>\n\u003Cp>For a serial system like this, the success probability is calculated as the product of the success probabilities for each individual step. So, if Step 1 has a 98% chance of success and Step 2 has a 99% chance, the overall probability of success (i.e., the \"reliability\" of the application) would be \u003Ccode>0.99 * 0.98 = 0.9702\u003C/code>, or ~97%.\u003C/p>\n\u003Cp>You might note that this is actually worse than \"the weakest link.\" You can think of the product here as a logical \u003Ccode>and\u003C/code>: \u003Cem>both\u003C/em> microservices must succeed in order for an overall success to occur. Even though we have a 98% chance of successfully painting the shed, the overall process could still fail if the committee deadlocks on making a decision in Step 1.\u003C/p>\n\u003Cp>\u003Cstrong>Aside:\u003C/strong> This then effectively proves what most of us intuitively know to be true. If you add more components to a system, its overall reliability goes down, even if each individual component has a success probability of \"\u003Ca href=\"https://en.wikipedia.org/wiki/High_availability#Percentage_calculation\" rel=\"noopener\" target=\"_blank\">five nines\u003C/a>.\" Said system would have an overall chance of success of \u003Ccode>.99999 ^ n\u003C/code>, where \u003Ccode>n\u003C/code> is the number of services (assuming they're in serial). Meaning, just adding a third service to our example (say, splitting out choosing the paint color versus procuring the actual paint) reduces the chance of success to four nines, about .99997 or the equivalent of one failure out of every 33,333. \u003Csup>\u003Ca href=\"#user-content-fn-1\" id=\"user-content-fnref-1\" data-footnote-ref aria-describedby=\"footnote-label\">1\u003C/a>\u003C/sup>\u003C/p>\n\u003Cp>\n You can make the story a little bit better by adding redundancy to each step.\n \n \u003Cimg src=\"//images.ctfassets.net/0uuz8ydxyd9p/6tYZDjZuwQlY1tlsH5bWAm/8558bc7f1abea164e8165a4766688349/image6.png-inverted.png\" alt=\"step 1 - 2 with replicas\">\n\u003C/p>\n\u003Cp>Even if the backup services are wildly unreliable, it still improves the overall reliability, which is calculated as:\u003C/p>\n\u003Cpre>\u003Ccode>Reliability of step 1 (R1) = 1 - (1 - .99) * (1 - .5) = 0.995\nReliability of step 2 (R2) = 1 - (1 - .98) * (1 - .6) = 0.992\nOverall reliability = R1 * R2 = .98704\n\u003C/code>\u003C/pre>\n\u003Cp>That 98.7% chance of success is certainly better than our 97% chance from earlier, but it does come with the extra operational cost of running secondary services.\u003C/p>\n\u003Cp>Why is this important?\u003C/p>\n\u003Cp>\n Part of the reason for splitting things out into microservices in the first place is to allow Step 1 to process as many requests as it can independently of Step 2 and vice-versa. To scale this, you might replicate each service as many times as needed, such that you have \u003Ccode>N\u003C/code> instances of Step 1's service, and \u003Ccode>M\u003C/code> instances of Step 2's.\n \n \u003Cimg src=\"//images.ctfassets.net/0uuz8ydxyd9p/3BP3uoWrCpMIEwTF3NGH73/d1287911305b5e7537a0f297648ee157/image8.png-inverted.png\" alt=\"step 1 - 2 N replicas\">\n\u003C/p>\n\u003Cp>While this improves our overall reliability number, we've introduced a new complication.\u003C/p>\n\u003Cp>When the application was just two services wired in serial, once Step 1 was finished it could directly notify Step 2 and then move on to the next incoming request. Each service could maintain its own queue of requests (with possible persistence) and very little other coordination would be needed.\u003C/p>\n\u003Cp>Unfortunately, that pesky little parenthetical tucked in there is the culprit of our complications. The \"possible persistence\" of a request queue is helpful if you want to not lose requests in the face of, e.g., process restarts. But, it's \u003Cem>absolutely required\u003C/em> once you scale out beyond n = 1 instances of the service so that each request can be handled effectively once.\u003C/p>\n\u003Cp>\n For short-lived requests, this could be handled by some kind of load balancer in front of each step:\n \n \u003Cimg src=\"//images.ctfassets.net/0uuz8ydxyd9p/4YPtglXRxFbBVQBnpfpDiZ/31cd260993404ad24a0af4c474f3cb07/image7.png-inverted.png\" alt=\"step 1 - 2 with load balancers\">\n\u003C/p>\n\u003Cp>The load balancers act as the request queues, divvying requests out to workers as appropriate.\u003C/p>\n\u003Cp>The side effect is that we've now doubled the number of components in the system. Load balancers can fail just like any other service, and so the system reliability becomes\u003C/p>\n\u003Cp>R \u003Csub>LB1\u003C/sub> * R \u003Csub>1\u003C/sub> * R \u003Csub>LB2\u003C/sub> * R \u003Csub>2\u003C/sub>\u003C/p>\n\u003Cp>As discussed in the \u003Cstrong>aside\u003C/strong> earlier, if all four of these services have a five-nines (99.999%) chance of successfully processing a given request, then the overall reliability is .999994 = 0.99996 or an equivalence of one out of every 25,000 requests failing. That may not seem too bad, but without those load balancers, the success rate would have been .999992 = .99998 or a failure rate of one per \u003Cem>50,000\u003C/em>. Doubling the number of services in the sequence also doubled our failure rate.\u003C/p>\n\u003Cp>\n So, naturally, let's remove one of the load balancers and instead put a cache, database, or message queue in between:\n \n \u003Cimg src=\"//images.ctfassets.net/0uuz8ydxyd9p/7sd7JiGTFADQD3FAbkS9lv/680b135414b32b8679db0f86df6db193/image1.png-inverted.png\" alt=\"step 1 - 2 with queue\">\n\u003C/p>\n\u003Cp>This intermediate queue provides a central place for keeping track of where your various sheds are in their painting lifecycles. That lifecycle is still somewhat disjointed though, and your components (Steps 1 and 2 and the Queue) are still very much wired in serial in a way that one outage affects an untold number of sheds—which ones and how is anyone's guess.\u003C/p>\n\u003Ch2 id=\"answering-the-bigger-questions\">Answering the Bigger Questions\u003Ca href=\"#answering-the-bigger-questions\">\u003Cspan class=\"ml-4\">#\u003C/span>\u003C/a>\u003C/h2>\n\u003Cp>This brings us, finally, back to the original question, \"How reliable is the application?\" The previous section described how to calculate the reliability of an overall system. Being able to say that the combined success rate of all services in the system is X% \u003Cem>sort of\u003C/em> answers the application-reliability question, but only does so from a fairly conceptual level.\u003C/p>\n\u003Cp>More specifically, while it tells you how often any one random request will fail, it doesn't answer questions like \"Where is \u003Cem>this\u003C/em> request?\" or \"Where did \u003Cem>this\u003C/em> request fail?\" or \"Is \u003Cem>this\u003C/em> request going to eventually unstick itself or does a human need to step in?\"\u003C/p>\n\u003Cp>That is to say, if you're looking at the aforementioned collection of microservices in an e-commerce system and wondering where a particular order is in its lifecycle (footnote: because, for example, you're building a frontend for customer service agents), well.... good luck.\u003C/p>\n\u003Ch3 id=\"building-reliable-applications-with-temporal\">Building Reliable Applications With Temporal\u003Ca href=\"#building-reliable-applications-with-temporal\">\u003Cspan class=\"ml-4\">#\u003C/span>\u003C/a>\u003C/h3>\n\u003Cp>\n Having a single number representing the reliability of a system isn't nearly as important as understanding what to do when unexpected things happen (as they \u003Cem>always\u003C/em> will). And, seeing all green across a status dashboard feels delightful.\n \n \u003Cimg src=\"//images.ctfassets.net/0uuz8ydxyd9p/KbrAFZZwJmDBE6QqOMP8h/de7708c3b798234dccf51eb4dd522659/image5.png-inverted-green.png\" alt=\"green dashboard\">\n\u003C/p>\n\u003Cp>Suppose one of your services is backed by a load-balanced cluster of nodes. What's the impact on customer experience if a misconfiguration in your autoscaler causes a repeated up-then-down scaling?\u003C/p>\n\u003Cp>\n \u003Cimg src=\"//images.ctfassets.net/0uuz8ydxyd9p/3iMSX05mZAD3l0bMXhNZbk/ea29c68fc5ad551ddd7080a99ad87e77/image3.png-inverted.png\" alt=\"mock node count graph\">\n\u003C/p>\n\u003Cp>Sure, it's a problem worth investigating, but with Temporal you don't have to worry too much about the end-user impact: each user request could match to a Temporal Workflow, which is automatically resumed on another process where it left off if something interrupts the original process it’s running on.\u003C/p>\n\u003Cp>So for example, consider a simplified user flow through your system that looks like:\u003C/p>\n\u003Col>\n \u003Cli>Check fraud\u003C/li>\n \u003Cli>Validate inventory\u003C/li>\n \u003Cli>Process payment\u003C/li>\n \u003Cli>Dispatch / Ship\u003C/li>\n\u003C/ol>\n\u003Cp>If each of these is a separate microservice, and one of them is having infrastructure problems, you might be tempted to declare, \"Oh no! This whole application is unreliable now!\" After all, we saw earlier in this post that the weakest link in the chain tanks your overall reliability numbers.\u003C/p>\n\u003Cp>With Temporal, instead of coming in with the assumption that each individual service needs to be made as reliable as possible, we take it a level higher. What if we assumed that dependent services will \u003Cem>always\u003C/em> fail, and we instead work to make the \u003Cem>application\u003C/em> reliable?\u003C/p>\n\u003Cp>\n \u003Cimg src=\"//images.ctfassets.net/0uuz8ydxyd9p/3PcSQ0VkWqrTxk6ThQEjaE/4f9ae2b6cf21051b32c6c7b529492f18/image4.png-inverted.png\" alt=\"demo graphs inverted\">\n \u003Cem>Caption: Four graphs of a demo Temporal application. Top left: the number of active Kubernetes replicas for the Worker fleet; randomly re-sized between 0 and 10. Top right: Workflow successes per second for this application. Bottom left and right: Workflow timeouts and failures, respectively, both flat at zero.\u003C/em>\n\u003C/p>\n\u003Cp>In these example graphs, depicting a demo Workflow running a simulated load (upper right graph), the Kubernetes cluster (upper left) responsible for actually running the Workflow (as in, the Worker fleet) is concurrently being randomly scaled up or down. This random scaling is to simulate unreliable infrastructure.\u003C/p>\n\u003Cp>Measuring the uptime of this application from the infrastructure perspective – for example, how often are there no Workers available – leads to a pretty bleak number. Roughly calculated, the graph above represents an hour-long window. For a total of eight minutes during that hour, the cluster had no replicas available. 52 / 60 * 100 = 86.666% uptime, or in other words: zero nines.\u003C/p>\n\u003Cp>But, take a closer look at the other graphs. While the \u003Cem>rate\u003C/em> of Workflow successes dipped to zero during the infrastructure downtime, they did still continue to succeed: Workflow timeouts and Workflow failures remained flat at zero.\u003C/p>\n\u003Cp>With a Workflow as the thing we're measuring the reliability of, the calculation of reliability over the hour looks much nicer:\u003C/p>\n\u003Cpre>\u003Ccode>\tsum(timeouts_per_second) (T) = 0\n\tsum(failures_per_second) (F) = 0\n\tsum(successes_per_second) (S) = 685\n\n\tApplication reliability = 1 - (T + F) / S = 1 = 100%\n\u003C/code>\u003C/pre>\n\u003Cp>That is to say, even with much worse reliability than you'd like, Temporal gives you \u003Cem>better\u003C/em> overall application reliability.\u003C/p>\n\u003Ch2 id=\"summary\">Summary\u003Ca href=\"#summary\">\u003Cspan class=\"ml-4\">#\u003C/span>\u003C/a>\u003C/h2>\n\u003Cp>Perhaps obviously, the example used here is contrived, but it's not that far from reality with Temporal. Microservices and external dependencies will always fail at some point. Working to make them as reliable as possible is necessary and important work, but no service will ever truly hit 100% reliability over a sufficiently long period of time.\u003C/p>\n\u003Cp>To mitigate that, consider why services exist in the first place: to provide a, well, service—a self-contained piece of functionality—to a bigger application. When you take a step back and consider the reliability of the bigger picture, rather than service-by-service, your whole application can be reliable while underlying services are inconsistent.\u003C/p>\n\u003Cp>For more on Temporal, head over to our documentation and courses:\u003C/p>\n\u003Cul>\n \u003Cli>\u003Ca href=\"https://docs.temporal.io\" rel=\"noopener\" target=\"_blank\">https://docs.temporal.io\u003C/a>\u003C/li>\n \u003Cli>\u003Ca href=\"https://learn.temporal.io\" rel=\"noopener\" target=\"_blank\">https://learn.temporal.io\u003C/a>\u003C/li>\n\u003C/ul>\n\u003Chr>\n\u003Csection data-footnotes class=\"footnotes\">\n \u003Ch2 class=\"sr-only\" id=\"footnote-label\">Footnotes\u003Ca href=\"#footnote-label\">\u003Cspan class=\"ml-4\">#\u003C/span>\u003C/a>\u003C/h2>\n \u003Col>\n \u003Cli id=\"user-content-fn-1\">\n \u003Cp>Which, to be fair, is still a heck of a lot of successfully painted sheds. \u003Ca href=\"#user-content-fnref-1\" data-footnote-backref=\"\" aria-label=\"Back to reference 1\" class=\"data-footnote-backref\">↩\u003C/a>\u003C/p>\n \u003C/li>\n \u003C/ol>\n\u003C/section>",featureImage:{title:"social-card-machine",description:"Close-up of dark industrial machinery with blue-tinted metal gears, pipes, and mechanical components in low lighting.",url:"https://images.ctfassets.net/0uuz8ydxyd9p/ixYNBwZoX1wu8VwLSDFmS/35e58aea31546f1b0dc42311f8f1de98/social-card-machine.jpg",aspectRatio:1.5},publishDate:"2024-01-12T00:00-07:00",updateDate:void 0,metaDescription:"What do you do when you know, with absolute certainty, that the infrastructure you're running something on is unreliable? Hardware always fails eventually, networks go down, or software needs to be patched.",metaTitle:"Building application reliability on top of infrastructure unreliability",socialCard:{title:"social-card-machine",description:"Close-up of dark industrial machinery with blue-tinted metal gears, pipes, and mechanical components in low lighting.",url:"https://images.ctfassets.net/0uuz8ydxyd9p/ixYNBwZoX1wu8VwLSDFmS/35e58aea31546f1b0dc42311f8f1de98/social-card-machine.jpg",aspectRatio:1.5},tags:"Industry Events",slug:"building-application-reliability-on-top-of-infrastructure-unreliability",contentType:"blogPost",entityId:"4z6WrsBLY38fBGZSaasb8X",authors:[{id:"1V2nWXuO9AHzYzI4qmqAJ5",name:"Fitz",slug:"fitz",jobTitle:"Developer Advocate",photograph:{title:"Fitz",description:"Person with bright blue spiky hair styled upward, wearing a gray and black shirt, taking a selfie indoors.",url:"https://images.ctfassets.net/0uuz8ydxyd9p/5rtdEzcJFPLV5Ce29uPyFb/951227c4a68cb13b1bd03c8856b1f740/fitzface_shadowed_blue-2.jpg",aspectRatio:1},biography:void 0,twitterUrl:void 0,linkedInUrl:void 0,website:void 0,gitHubUrl:void 0,company:void 0,contentType:"person"}],authorsString:"Fitz",category:"How-To",relatedPosts:void 0,promoCard:{title:"Get Started with Cloud Promo",eyebrow:"Temporal Cloud",heading:"Ready to see for yourself?",content:"\u003Cp>Sign up for Temporal Cloud today and get $1,000 in free credits.\u003C/p>",callsToAction:[{text:"Get started",href:"/get-cloud",variant:"primary",leadingIcon:null,trailingIcon:"arrow-right",entityId:"7wKJdxc4JeeNXHCliDfqVV",size:"regular",theme:"pink"}],entityId:"2SJj3BBBzkr6FefI6ZfRox",contentType:"noise"},readingTime:12},morePosts:[{title:"Getting the most out of the Billable Action Count metric",slug:"getting-the-most-out-of-the-billable-action-count-metric",category:"How-To",publishDate:"2026-06-25",readingTime:11},{title:"How to build deep research agents using Temporal and Braintrust",slug:"how-to-build-deep-research-agents-using-temporal-and-braintrust",category:"How-To",publishDate:"2026-06-03",readingTime:6}],domain:"https://temporal.io"},uses:{params:["slug"]}},{type:"data",data:null,uses:{}}],
form: null,
error: null
});
});
}
</script>
</div>
</body>
</html>