Files
nexus/sreweekly/articles/353/06-intermittent-downtime-from-repeated-crashes.html
2026-09-12 17:23:01 +08:00

83 lines
242 KiB
HTML
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html data-dpl-id="dpl_BBRoCePwqZvFD5t2NWjeL4RH9mwm" lang="en" data-scroll-behavior="smooth" class="geist_mono_54069fc3-module__PcBEcq__variable kalam_48bfd9a2-module__JSifUG__variable stkbureausans_1e54e897-module__zxjcxa__variable stkbureauserif_2525014a-module__zkXytW__variable stkbureauserifdeferred_742a2474-module__ukGqna__variable pixelify_sans_81b7d097-module__1Rm0Lq__variable scroll-smooth"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" href="/_next/static/immutable/media/stkbureau_serif_book-s.p.0_js8gy1zifgl.woff2" as="font" crossorigin="" type="font/woff2"/><link rel="preload" as="image" imageSrcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=60&amp;q=75 60w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=64&amp;q=75 64w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=96&amp;q=75 96w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=128&amp;q=75 128w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=256&amp;q=75 256w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=384&amp;q=75 384w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=640&amp;q=75 640w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=750&amp;q=75 750w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=828&amp;q=75 828w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=1080&amp;q=75 1080w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=1200&amp;q=75 1200w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=1920&amp;q=75 1920w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=2048&amp;q=75 2048w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=3840&amp;q=75 3840w" imageSizes="100vw"/><link rel="stylesheet" href="/_next/static/immutable/chunks/1wus27ic5fsbj.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/immutable/chunks/0ok2xz0peykn3.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/immutable/chunks/3_pcvzakx73f_.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/immutable/chunks/1f8tzf816i32j.js"/><script src="/_next/static/immutable/chunks/17il0mq9qt26b.js" async=""></script><script src="/_next/static/immutable/chunks/07v142o5owpy6.js" async=""></script><script src="/_next/static/immutable/chunks/1eat_vqbhn7ur.js" async=""></script><script src="/_next/static/immutable/chunks/16wtr1pupryzq.js" async=""></script><script src="/_next/static/immutable/chunks/turbopack-1yl6v3yt8k5c5.js" async=""></script><script src="/_next/static/immutable/chunks/0esgme6slw2os.js" async=""></script><script src="/_next/static/immutable/chunks/0-l0sxraekmer.js" async=""></script><script src="/_next/static/immutable/chunks/1vj2-813oq6tg.js" async=""></script><script src="/_next/static/immutable/chunks/1vvlruwewhgbt.js" async=""></script><script src="/_next/static/immutable/chunks/27fxx2z424byj.js" async=""></script><script src="/_next/static/immutable/chunks/3c4tjp_3gm_rj.js" async=""></script><script src="/_next/static/immutable/chunks/3toyqodzjr-9d.js" async=""></script><script src="/_next/static/immutable/chunks/2hdxyct5upc3z.js" async=""></script><script src="/_next/static/immutable/chunks/1ah2s79s4e7mm.js" async=""></script><script src="/_next/static/immutable/chunks/24dbkag80ciqd.js" async=""></script><script src="/_next/static/immutable/chunks/34-yxyyt1i1n8.js" async=""></script><script src="/_next/static/immutable/chunks/1zi8qxi1p_fsn.js" async=""></script><script src="/_next/static/immutable/chunks/2eophjajue3_e.js" async=""></script><script src="/_next/static/immutable/chunks/3d48p4uv87yqq.js" async=""></script><script src="/_next/static/immutable/chunks/2y03n43y-fjfs.js" async=""></script><script src="/_next/static/immutable/chunks/39pfb5kk8her1.js" async=""></script><script src="/_next/static/immutable/chunks/3qz9rwp_7jbnb.js" async=""></script><script src="/_next/static/immutable/chunks/1my_9vnng6gxj.js" async=""></script><script src="/_next/static/immutable/chunks/040yti440pn1b.js" async=""></script><script src="/_next/static/immutable/chunks/0v4j5g1ppvtg_.js" async=""></script><script src="/_next/static/immutable/chunks/3buml8s4fdaii.js" async=""></script><script src="/_next/static/immutable/chunks/2_frah5pspzdw.js" async=""></script><script src="/_next/static/immutable/chunks/1mcb8cvy5_n5b.js" async=""></script><script src="/_next/static/immutable/chunks/132szer1wwfxa.js" async=""></script><script src="/_next/static/immutable/chunks/2jjhnej_crwty.js" async=""></script><script src="/_next/static/immutable/chunks/19t-07qzgwjic.js" async=""></script><script src="/_next/static/immutable/chunks/1wpznbhwql3dd.js" async=""></script><script src="/_next/static/immutable/chunks/20izjjpsu0ag8.js" async=""></script><link rel="preload" href="https://embeds.iubenda.com/widgets/9d001c02-6db6-11ee-8bfc-5ad8d8c564c0.js" as="script"/><meta name="next-size-adjust" content=""/><meta http-equiv="x-ua-compatible" content="ie=edge"/><link rel="alternate" type="application/rss+xml" title="Blog" href="https://incident.io/blog.xml"/><link rel="alternate" type="application/rss+xml" title="Changelog" href="https://incident.io/changelog.xml"/><link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png"/><link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png"/><link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png"/><title>Intermittent downtime from repeated crashes | Blog | incident.io</title><meta name="description" content="This is a technical write-up of an incident on Friday 18th November 2022 where we experienced 13 minutes of downtime from intermittent crashes."/><meta name="keywords" content="incident,incident management,incident response,post mortem,outage,slack incident,incident channel"/><meta name="slack-app-id" content="A01DEGPUHHC"/><link rel="canonical" href="https://incident.io/blog/intermittent-downtime"/><link rel="alternate" type="text/markdown" href="https://incident.io/blog/intermittent-downtime.md"/><meta property="og:title" content="Intermittent downtime from repeated crashes | Blog | incident.io"/><meta property="og:description" content="This is a technical write-up of an incident on Friday 18th November 2022 where we experienced 13 minutes of downtime from intermittent crashes."/><meta property="og:image" content="https://incident.io/socialcard/blog/intermittent-downtime?1687509000000"/><meta property="og:image:width" content="1200"/><meta property="og:image:height" content="628"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:site" content="@incident_io"/><meta name="twitter:creator" content="@incident_io"/><meta name="twitter:title" content="Intermittent downtime from repeated crashes | Blog"/><meta name="twitter:description" content="This is a technical write-up of an incident on Friday 18th November 2022 where we experienced 13 minutes of downtime from intermittent crashes."/><meta name="twitter:image" content="https://incident.io/socialcard/blog/intermittent-downtime?1687509000000"/><meta name="twitter:image:width" content="1200"/><meta name="twitter:image:height" content="628"/><link rel="icon" href="/icons/icon-48x48.png"/><link rel="apple-touch-icon" href="/icons/icon-48x48.png" sizes="48x48"/><link rel="apple-touch-icon" href="/icons/icon-72x72.png" sizes="72x72"/><link rel="apple-touch-icon" href="/icons/icon-96x96.png" sizes="96x96"/><link rel="apple-touch-icon" href="/icons/icon-144x144.png" sizes="144x144"/><link rel="apple-touch-icon" href="/icons/icon-192x192.png" sizes="192x192"/><link rel="apple-touch-icon" href="/icons/icon-256x256.png" sizes="256x256"/><link rel="apple-touch-icon" href="/icons/icon-384x384.png" sizes="384x384"/><link rel="apple-touch-icon" href="/icons/icon-512x512.png" sizes="512x512"/><script src="/_next/static/immutable/chunks/0c0hxoamwjsbw.js" noModule=""></script></head><body><div hidden=""><!--$--><!--/$--></div><script type="application/ld+json" id="organization-jsonld-Organization">{"@context":"https://schema.org","@type":"Organization","name":"incident.io","url":"https://incident.io","logo":"https://incident.io/icons/icon-512x512.png","sameAs":["https://x.com/incident_io","https://www.linkedin.com/company/incident-io","https://www.youtube.com/@incident-io","https://bsky.app/profile/incident.io"]}</script><!--$--><!--/$--><script type="application/ld+json" id="article-jsonld-BlogPosting">{"@context":"https://schema.org","@type":"BlogPosting","headline":"Intermittent downtime from repeated crashes","url":"https://incident.io/blog/intermittent-downtime","author":[{"@type":"Person","name":"Lawrence Jones"}],"datePublished":"2022-11-30T00:00:00.000Z","dateModified":"2022-11-30T00:00:00.000Z","image":["https://cdn.sanity.io/images/oqy5aexb/production/ee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png?w=2000&fit=max&auto=format"],"publisher":{"@type":"Organization","name":"incident.io","url":"https://incident.io"},"description":"This is a technical write-up of an incident on Friday 18th November 2022 where we experienced 13 minutes of downtime from intermittent crashes."}</script><nav class="h-20 sticky top-0 w-full z-[100]"><div class="lg:hidden fixed top-0 bg-white text-slate-900 overflow-y-auto overscroll-contain left-0 w-full h-[100dvh] flex flex-col z-[100] pb-6 transition-transform duration-200 ease-out translate-x-full pointer-events-none"><div class="bg-white"><div class="h-20 mx-auto" data-mobile-menu-open="false"><div class="relative flex justify-between h-full z-[99]"><div class="max-w-[1512px] mx-auto px-5 flex-1 flex items-center justify-between"><div class="flex-shrink-0 flex items-center relative transition z-[1]"><a class="transition hover:opacity-80 relative top-[-8px]" href="/"><span class="sr-only">incident.io</span><div class="w-[130px] h-[32px]"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 1000 248" class="h-8 w-auto absolute inset-0 dark:opacity-0 [[data-mobile-menu-open=true]_&amp;]:opacity-100"><g clip-path="url(#workmark__black_svg__a)"><g clip-path="url(#workmark__black_svg__b)"><path fill="#F25533" fill-rule="evenodd" d="M48.803 140.978v23.429C20.788 157.71 0 133.023 0 103.594 0 86.626 7.168 73.165 18.131 59.96 27.222 49.01 49.67 20.01 56.735 4.009c1.716-3.885 6.09-3.735 7.976-1.647 6.033 6.677 13.47 21.26 15.902 37.802.496 3.37.7 6.123.862 8.309.348 4.691.502 6.777 2.93 6.777 3.816 0 6.313-5.795 6.861-11.776.352-3.833 4.148-5.115 6.765-3.833 12.59 6.17 25.437 35.537 28.543 49.949.974 4.52 1.609 9.19 1.609 14.004 0 29.363-20.695 54.006-48.616 60.768v-23.384zm15.289-9.338c9.845 0 17.826-7.567 17.826-16.901 0-14.888-10.959-25.526-16.42-28.76-.369-.218-.553-.328-1.055-.314-.367.01-.91.24-1.172.498-.358.351-.453.752-.643 1.552-.968 4.078-4.015 6.986-7.222 10.045-4.42 4.217-9.141 8.723-9.141 16.979 0 9.334 7.981 16.901 17.827 16.901" clip-rule="evenodd"></path><path fill="#000" d="M158.057 138.153c-13.535 0-23.788 4.919-29.325 12.503v-9.633h-30.76v102.48h30.76v-55.954c0-14.552 7.793-21.111 19.071-21.111 9.843 0 17.841 5.944 17.841 18.651v58.414h30.76V180.58c0-27.67-17.636-42.427-38.347-42.427M262.641 246.373c20.097 0 37.527-10.454 46.346-26.44l-26.864-15.372c-3.281 7.173-10.869 11.477-19.892 11.477-13.329 0-23.172-9.632-23.172-23.775s9.843-23.775 23.172-23.775c9.023 0 16.406 4.303 19.892 11.477l26.864-15.577c-8.819-15.782-26.454-26.235-46.346-26.235-31.169 0-54.342 23.571-54.342 54.11s23.173 54.11 54.342 54.11M332.645 132.005c10.047 0 18.456-8.404 18.456-18.448 0-10.042-8.409-18.446-18.456-18.446-10.049 0-18.456 8.404-18.456 18.446 0 10.044 8.407 18.448 18.456 18.448m-15.38 111.498h30.76v-102.48h-30.76zM440.871 100.031v50.625c-6.767-7.788-16.815-12.503-30.555-12.503-26.863 0-49.01 23.571-49.01 54.11s22.147 54.11 49.01 54.11c13.74 0 23.788-4.714 30.555-12.503v9.634h30.76V100.031zm-24.402 117.238c-14.15 0-24.403-9.634-24.403-25.006s10.253-25.005 24.403-25.005 24.402 9.633 24.402 25.005-10.253 25.006-24.402 25.006M517.576 204.561h72.799q1.23-5.841 1.23-12.298c0-30.949-22.147-54.11-52.086-54.11-32.401 0-54.548 23.571-54.548 54.11s21.736 54.11 56.803 54.11c19.482 0 34.656-7.174 44.705-21.111l-24.609-14.143c-4.101 4.509-11.278 7.789-19.686 7.789-11.278 0-20.711-3.69-24.608-14.347m-.615-22.956c2.871-10.453 10.663-16.192 22.352-16.192 9.228 0 18.456 4.304 21.737 16.192zM665.034 138.153c-13.534 0-23.787 4.919-29.324 12.503v-9.633h-30.76v102.48h30.76v-55.954c0-14.552 7.793-21.111 19.071-21.111 9.843 0 17.841 5.944 17.841 18.651v58.414h30.76V180.58c0-27.67-17.636-42.427-38.348-42.427M780.49 170.537v-29.514h-21.123v-28.695l-30.759 9.224v19.471h-16.407v29.514h16.407v36.278c0 28.695 11.689 40.787 51.882 36.688v-27.875c-13.535.82-21.123 0-21.123-8.813v-36.278zM813.237 246.373c10.663 0 19.48-8.813 19.48-19.472 0-10.657-8.817-19.471-19.48-19.471s-19.482 8.814-19.482 19.471c0 10.659 8.818 19.472 19.482 19.472M862.014 132.005c10.047 0 18.455-8.404 18.455-18.448 0-10.042-8.408-18.446-18.455-18.446-10.049 0-18.456 8.404-18.456 18.446 0 10.044 8.407 18.448 18.456 18.448m-15.381 111.498h30.761v-102.48h-30.761zM945.017 246.373c30.144 0 54.343-23.571 54.343-54.11s-24.199-54.11-54.343-54.11-54.342 23.571-54.342 54.11 24.197 54.11 54.342 54.11m0-29.925c-13.329 0-23.582-9.633-23.582-24.185 0-14.553 10.253-24.185 23.582-24.185s23.583 9.632 23.583 24.185-10.253 24.185-23.583 24.185M48.804 243.508h30.76v-78.701s-30.76 5.525-30.76 34.995z"></path></g></g><defs><clipPath id="workmark__black_svg__a"><path fill="#fff" d="M0 0h1000v248H0z"></path></clipPath><clipPath id="workmark__black_svg__b"><path fill="#fff" d="M0 .91h1000v246.18H0z"></path></clipPath></defs></svg><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 1000 248" class="h-8 w-auto absolute inset-0 opacity-0 dark:opacity-100 [[data-mobile-menu-open=true]_&amp;]:opacity-0"><g fill="#fff" clip-path="url(#workmark__fullwhite_svg__a)"><path fill-rule="evenodd" d="M48.803 140.978v23.429C20.788 157.71 0 133.023 0 103.594 0 86.626 7.168 73.165 18.131 59.96 27.222 49.01 49.67 20.01 56.735 4.009c1.716-3.885 6.09-3.735 7.976-1.647 6.033 6.677 13.47 21.26 15.902 37.802.496 3.37.7 6.123.862 8.309.348 4.691.502 6.777 2.93 6.777 3.816 0 6.313-5.795 6.861-11.776.352-3.833 4.148-5.115 6.765-3.833 12.59 6.17 25.437 35.537 28.543 49.949.974 4.52 1.609 9.19 1.609 14.004 0 29.363-20.695 54.006-48.616 60.768v-23.384zm15.289-9.338c9.845 0 17.826-7.567 17.826-16.901 0-14.888-10.959-25.526-16.42-28.76-.369-.218-.553-.328-1.055-.314-.367.01-.91.24-1.172.498-.358.351-.453.752-.643 1.552-.968 4.078-4.015 6.986-7.222 10.045-4.42 4.217-9.141 8.723-9.141 16.979 0 9.334 7.981 16.901 17.827 16.901" clip-rule="evenodd"></path><path d="M158.057 138.153c-13.535 0-23.788 4.919-29.325 12.503v-9.633h-30.76v102.48h30.76v-55.954c0-14.552 7.793-21.111 19.071-21.111 9.843 0 17.841 5.944 17.841 18.651v58.414h30.76V180.58c0-27.67-17.636-42.427-38.347-42.427M262.641 246.373c20.097 0 37.527-10.454 46.346-26.44l-26.864-15.372c-3.281 7.173-10.869 11.477-19.892 11.477-13.329 0-23.172-9.632-23.172-23.775s9.843-23.775 23.172-23.775c9.023 0 16.406 4.303 19.892 11.477l26.864-15.577c-8.819-15.782-26.454-26.235-46.346-26.235-31.169 0-54.342 23.571-54.342 54.11s23.173 54.11 54.342 54.11M332.645 132.005c10.047 0 18.456-8.404 18.456-18.448 0-10.042-8.409-18.446-18.456-18.446-10.049 0-18.456 8.404-18.456 18.446 0 10.044 8.407 18.448 18.456 18.448m-15.38 111.498h30.76v-102.48h-30.76zM440.871 100.031v50.625c-6.767-7.788-16.815-12.503-30.555-12.503-26.863 0-49.01 23.571-49.01 54.11s22.147 54.11 49.01 54.11c13.74 0 23.788-4.714 30.555-12.503v9.634h30.76V100.031zm-24.402 117.238c-14.15 0-24.403-9.634-24.403-25.006s10.253-25.005 24.403-25.005 24.402 9.633 24.402 25.005-10.253 25.006-24.402 25.006M517.576 204.561h72.799q1.23-5.841 1.23-12.298c0-30.949-22.147-54.11-52.086-54.11-32.401 0-54.548 23.571-54.548 54.11s21.736 54.11 56.803 54.11c19.482 0 34.656-7.174 44.705-21.111l-24.609-14.143c-4.101 4.509-11.278 7.789-19.686 7.789-11.278 0-20.711-3.69-24.608-14.347m-.615-22.956c2.871-10.453 10.663-16.192 22.352-16.192 9.228 0 18.456 4.304 21.737 16.192zM665.034 138.153c-13.534 0-23.787 4.919-29.324 12.503v-9.633h-30.76v102.48h30.76v-55.954c0-14.552 7.793-21.111 19.071-21.111 9.843 0 17.841 5.944 17.841 18.651v58.414h30.76V180.58c0-27.67-17.636-42.427-38.348-42.427M780.49 170.537v-29.514h-21.123v-28.695l-30.759 9.224v19.471h-16.407v29.514h16.407v36.278c0 28.695 11.689 40.787 51.882 36.688v-27.875c-13.535.82-21.123 0-21.123-8.813v-36.278zM813.237 246.373c10.663 0 19.48-8.813 19.48-19.472 0-10.657-8.817-19.471-19.48-19.471s-19.482 8.814-19.482 19.471c0 10.659 8.818 19.472 19.482 19.472M862.014 132.005c10.047 0 18.455-8.404 18.455-18.448 0-10.042-8.408-18.446-18.455-18.446-10.049 0-18.456 8.404-18.456 18.446 0 10.044 8.407 18.448 18.456 18.448m-15.381 111.498h30.761v-102.48h-30.761zM945.017 246.373c30.144 0 54.343-23.571 54.343-54.11s-24.199-54.11-54.343-54.11-54.342 23.571-54.342 54.11 24.197 54.11 54.342 54.11m0-29.925c-13.329 0-23.582-9.633-23.582-24.185 0-14.553 10.253-24.185 23.582-24.185s23.583 9.632 23.583 24.185-10.253 24.185-23.583 24.185M48.804 243.508h30.76v-78.701s-30.76 5.525-30.76 34.995z"></path></g><defs><clipPath id="workmark__fullwhite_svg__a"><path fill="#fff" d="M0 .91h1000v246.18H0z"></path></clipPath></defs></svg></div></a><div class="border border-stroke-primary bg-white rounded-lg shadow-lg p-1 flex flex-col gap-1 absolute top-full mt-2 left-0 z-50 w-[250px] transition-opacity duration-200 opacity-0 pointer-events-none"><div class="p-2 pt-1 text-body-xs-emphasis border-b border-stroke-primary">Brand Assets</div><div class="flex flex-col gap-1 text-body-xs text-content-secondary"><a href="/brand/resources/incidentio-brand-pngs.zip" class="p-2 flex gap-2 items-center group rounded hover:bg-surface-secondary transition"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 16 16" class="shrink-0 size-4 text-content-primary"><path fill="#fff" d="M2.8 1.6A1.6 1.6 0 0 1 4.4 0h6.4c.127 0 .25.05.34.141l4.318 4.318c.09.09.141.213.141.341v9.6A1.6 1.6 0 0 1 14 16H4.4a1.6 1.6 0 0 1-1.6-1.6z"></path><path stroke="#161618" stroke-opacity="0.08" stroke-width="0.6" d="M4.399.3h6.4c.049 0 .095.02.13.054l4.317 4.317c.034.034.053.08.053.129v9.6a1.3 1.3 0 0 1-1.3 1.3H4.4a1.3 1.3 0 0 1-1.3-1.3V1.6a1.3 1.3 0 0 1 1.3-1.3Z"></path><path stroke="#161618" stroke-opacity="0.08" stroke-width="0.6" d="M10.8.2v3a1.6 1.6 0 0 0 1.6 1.6h3"></path><rect width="11.4" height="6.6" x="0.3" y="7.1" fill="#F25533" rx="0.8"></rect><path fill="#fff" d="M1.966 11.9V8.99h1.147q.332 0 .564.127.234.126.355.348.124.222.124.512t-.125.511a.86.86 0 0 1-.362.345 1.2 1.2 0 0 1-.571.124h-.732v-.493h.632a.6.6 0 0 0 .293-.061.4.4 0 0 0 .173-.172.54.54 0 0 0 .059-.254q0-.145-.059-.253a.4.4 0 0 0-.173-.17.63.63 0 0 0-.295-.06H2.58V11.9zm5.023-2.91v2.91h-.531l-1.266-1.831H5.17V11.9h-.615V8.99h.54l1.255 1.83h.026V8.99zm2.437.941a.6.6 0 0 0-.084-.183.567.567 0 0 0-.308-.223.8.8 0 0 0-.217-.03.7.7 0 0 0-.392.111.73.73 0 0 0-.262.322 1.3 1.3 0 0 0-.094.515q0 .303.093.517.091.213.261.325.17.11.4.11a.8.8 0 0 0 .356-.073.53.53 0 0 0 .227-.212.63.63 0 0 0 .08-.322l.125.018h-.75v-.463h1.217v.367q0 .383-.162.659a1.1 1.1 0 0 1-.446.423 1.4 1.4 0 0 1-.65.148q-.41 0-.72-.18a1.25 1.25 0 0 1-.482-.516 1.7 1.7 0 0 1-.172-.796q0-.354.102-.63.104-.279.29-.472.186-.194.433-.294t.536-.1q.246 0 .46.072.213.07.378.201a1.1 1.1 0 0 1 .406.706z"></path></svg><div class="grow">Download .PNG logos</div><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 25 26" class="shrink-0 size-4"><path fill="currentColor" fill-rule="evenodd" d="M10.274 7.024a.937.937 0 0 1 1.325 0l5.313 5.313a.94.94 0 0 1 0 1.325l-5.313 5.312a.937.937 0 0 1-1.325-1.325l4.65-4.65-4.65-4.65a.937.937 0 0 1 0-1.325" clip-rule="evenodd"></path></svg></a><a href="/brand/resources/incidentio-brand-svgs.zip" class="p-2 flex gap-2 items-center group rounded hover:bg-surface-secondary transition"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 16 16" class="shrink-0 size-4 text-content-primary"><path fill="#fff" d="M2.8 1.6A1.6 1.6 0 0 1 4.4 0h6.4c.128 0 .251.05.342.141l4.317 4.318c.09.09.141.213.141.341v9.6A1.6 1.6 0 0 1 14 16H4.4a1.6 1.6 0 0 1-1.6-1.6z"></path><path stroke="#161618" stroke-opacity="0.08" stroke-width="0.6" d="M4.4.3h6.4q.076.002.13.054l4.317 4.317c.034.034.053.08.053.129v9.6a1.3 1.3 0 0 1-1.3 1.3H4.4a1.3 1.3 0 0 1-1.3-1.3V1.6A1.3 1.3 0 0 1 4.4.3Z"></path><path stroke="#161618" stroke-opacity="0.08" stroke-width="0.6" d="M10.8.2v3a1.6 1.6 0 0 0 1.6 1.6h3"></path><rect width="11.4" height="6.6" x="0.299" y="7.1" fill="#F25533" rx="0.8"></rect><path fill="#fff" d="M3.584 9.828a.36.36 0 0 0-.146-.267.58.58 0 0 0-.35-.096.7.7 0 0 0-.255.043.36.36 0 0 0-.16.115.28.28 0 0 0-.053.168.24.24 0 0 0 .032.136.34.34 0 0 0 .101.1.8.8 0 0 0 .148.073q.084.03.179.051l.261.063q.19.042.35.114.16.07.275.174a.73.73 0 0 1 .248.567.8.8 0 0 1-.137.463.87.87 0 0 1-.386.303q-.252.106-.607.106-.353 0-.613-.108a.9.9 0 0 1-.407-.32.96.96 0 0 1-.152-.526h.596a.45.45 0 0 0 .083.244q.075.096.2.146.127.049.286.049a.7.7 0 0 0 .271-.046.4.4 0 0 0 .18-.126.3.3 0 0 0 .065-.186q0-.098-.058-.165a.44.44 0 0 0-.168-.114 2 2 0 0 0-.269-.085l-.316-.08a1.3 1.3 0 0 1-.581-.28.65.65 0 0 1-.212-.512.76.76 0 0 1 .14-.462.95.95 0 0 1 .394-.308q.25-.11.568-.11.324 0 .566.11a.9.9 0 0 1 .378.308q.135.197.139.458zm1.569-.837.703 2.21h.027l.704-2.21h.682L6.266 11.9h-.792L4.47 8.99zm4.231.94a.6.6 0 0 0-.083-.183.566.566 0 0 0-.308-.223.8.8 0 0 0-.218-.03.7.7 0 0 0-.392.111.73.73 0 0 0-.261.322 1.3 1.3 0 0 0-.094.515q0 .303.092.517.093.213.261.325.17.11.4.11a.8.8 0 0 0 .356-.073.53.53 0 0 0 .227-.212.63.63 0 0 0 .08-.322l.125.018h-.75v-.463h1.217v.367q0 .383-.162.659a1.1 1.1 0 0 1-.446.423 1.4 1.4 0 0 1-.65.148q-.41 0-.719-.18a1.25 1.25 0 0 1-.483-.516 1.7 1.7 0 0 1-.172-.796q0-.354.102-.63.105-.279.29-.472.186-.194.433-.294.248-.1.536-.1.247 0 .46.072.213.07.378.201a1.1 1.1 0 0 1 .406.706z"></path></svg><div class="grow">Download .SVG logos</div><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 25 26" class="shrink-0 size-4"><path fill="currentColor" fill-rule="evenodd" d="M10.274 7.024a.937.937 0 0 1 1.325 0l5.313 5.313a.94.94 0 0 1 0 1.325l-5.313 5.312a.937.937 0 0 1-1.325-1.325l4.65-4.65-4.65-4.65a.937.937 0 0 1 0-1.325" clip-rule="evenodd"></path></svg></a><a href="/downloads/incidentio-brand-guidelines.pdf?v=20260826" class="p-2 flex gap-2 items-center group rounded hover:bg-surface-secondary transition"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="shrink-0 size-4 text-alarmalade-500"><path fill="currentColor" d="m16.487 6.427-3.914-3.914A1.74 1.74 0 0 0 11.336 2H5.75A2.75 2.75 0 0 0 3 4.75v10.5A2.75 2.75 0 0 0 5.75 18h8.5A2.75 2.75 0 0 0 17 15.25V7.664c0-.467-.182-.907-.513-1.237M6.75 7h2a.75.75 0 0 1 0 1.5h-2a.75.75 0 0 1 0-1.5m6.5 7.5h-6.5a.75.75 0 0 1 0-1.5h6.5a.75.75 0 0 1 0 1.5m0-3h-6.5a.75.75 0 0 1 0-1.5h6.5a.75.75 0 0 1 0 1.5m2.182-4H12.5c-.55 0-1-.45-1-1V3.579l.013-.005 3.922 3.921z"></path></svg><div class="grow">Download Brand Guidelines</div><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 25 26" class="shrink-0 size-4"><path fill="currentColor" fill-rule="evenodd" d="M10.274 7.024a.937.937 0 0 1 1.325 0l5.313 5.313a.94.94 0 0 1 0 1.325l-5.313 5.312a.937.937 0 0 1-1.325-1.325l4.65-4.65-4.65-4.65a.937.937 0 0 1 0-1.325" clip-rule="evenodd"></path></svg></a><a href="/brand" class="p-2 flex gap-2 items-center group rounded hover:bg-surface-secondary transition"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 21" class="shrink-0 size-4 text-content-primary"><path fill="currentColor" d="M14.25 3.5h-8.5A2.75 2.75 0 0 0 3 6.25v6.568c0 1.336 1.616 2.006 2.56 1.06L10.44 9H7.75a.75.75 0 0 1 0-1.5h4.5a.75.75 0 0 1 .75.75v4.5a.75.75 0 0 1-1.5 0V10.06L6.622 14.94c-.945.945-.276 2.56 1.06 2.56h6.568A2.75 2.75 0 0 0 17 14.75v-8.5a2.75 2.75 0 0 0-2.75-2.75"></path></svg><div class="grow">Visit brand center</div><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 25 26" class="shrink-0 size-4"><path fill="currentColor" fill-rule="evenodd" d="M10.274 7.024a.937.937 0 0 1 1.325 0l5.313 5.313a.94.94 0 0 1 0 1.325l-5.313 5.312a.937.937 0 0 1-1.325-1.325l4.65-4.65-4.65-4.65a.937.937 0 0 1 0-1.325" clip-rule="evenodd"></path></svg></a></div></div></div><nav aria-label="Main" data-orientation="horizontal" dir="ltr" class="hidden flex-1 lg:flex items-stretch justify-center h-full grow absolute inset-x-0"><div style="position:relative"><ul data-orientation="horizontal" class="h-full flex items-stretch justify-center gap-0" dir="ltr"><li><button id="radix-_R_4laav5ubsraivb_-trigger-products" data-state="closed" aria-expanded="false" class="relative font-medium border-transparent inline-flex items-center leading-none whitespace-nowrap transition-colors outline-0 dark:text-white focus-visible:text-orange-600 hover:text-orange-600 data-[state=open]:text-orange-600 text-gray-900 px-3 py-2 h-full group" data-radix-collection-item="">Products</button></li><li><button id="radix-_R_4laav5ubsraivb_-trigger-solutions" data-state="closed" aria-expanded="false" class="relative font-medium border-transparent inline-flex items-center leading-none whitespace-nowrap transition-colors outline-0 dark:text-white focus-visible:text-orange-600 hover:text-orange-600 data-[state=open]:text-orange-600 text-gray-900 px-3 py-2 h-full group" data-radix-collection-item="">Solutions</button></li><li><button id="radix-_R_4laav5ubsraivb_-trigger-resources" data-state="closed" aria-expanded="false" class="relative font-medium border-transparent inline-flex items-center leading-none whitespace-nowrap transition-colors outline-0 dark:text-white focus-visible:text-orange-600 hover:text-orange-600 data-[state=open]:text-orange-600 text-gray-900 px-3 py-2 h-full group" data-radix-collection-item="">Resources</button></li><li><button id="radix-_R_4laav5ubsraivb_-trigger-customers" data-state="closed" aria-expanded="false" class="relative font-medium border-transparent inline-flex items-center leading-none whitespace-nowrap transition-colors outline-0 dark:text-white focus-visible:text-orange-600 hover:text-orange-600 data-[state=open]:text-orange-600 text-gray-900 px-3 py-2 h-full group" data-radix-collection-item="">Customers</button></li><li><a class="relative font-medium border-transparent inline-flex items-center leading-none whitespace-nowrap transition-colors outline-0 dark:text-white focus-visible:text-orange-600 hover:text-orange-600 data-[state=open]:text-orange-600 text-gray-900 px-3 py-2 h-full" data-radix-collection-item="" href="/pricing">Pricing</a></li><li><a class="relative font-medium border-transparent inline-flex items-center leading-none whitespace-nowrap transition-colors outline-0 dark:text-white focus-visible:text-orange-600 hover:text-orange-600 data-[state=open]:text-orange-600 text-gray-900 px-3 py-2 h-full" data-radix-collection-item="" href="/careers">Careers</a></li></ul></div></nav></div><div class="absolute inset-y-0 right-5 flex lg:hidden items-center lg:static lg:inset-auto lg:ml-6 lg:pr-0"><div class="relative h-10"><button class="inline-flex items-center justify-center p-2 -mr-2 rounded-md text-gray-500 hover:text-gray-600 hover:bg-gray-100 focus:outline-none focus:ring-2 focus:ring-inset focus:ring-indigo-500 h-10 dark:text-white"><span class="sr-only">Open main menu</span><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" aria-hidden="true" data-slot="icon" class="block h-6 w-6"><path stroke-linecap="round" stroke-linejoin="round" d="M3.75 6.75h16.5M3.75 12h16.5m-16.5 5.25h16.5"></path></svg></button></div></div></div></div></div><div class="grow relative"><div class="w-full px-4 pt-4 absolute transition ease-in-out duration-250 transform translate-x-full"><ul class="flex flex-col gap-2 text-body-emphasis grow mb-20"><li class=""><button class="w-full text-left py-2.5 flex items-center gap-4"><div class="p-4 rounded-md flex items-center justify-center text-alarmalade-500 bg-surface-secondary group-hover:bg-surface-tertiary transition"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="shrink-0 size-5"><path fill="currentColor" d="M9.165 11.667c1.074 0 1.944.87 1.944 1.944v2.222c0 1.074-.87 1.944-1.944 1.944h-3.89a1.945 1.945 0 0 1-1.944-1.944v-2.222c0-1.073.87-1.944 1.944-1.944zm7.777-7.223c1.074 0 1.945.87 1.945 1.945v7.222c0 1.074-.87 1.945-1.945 1.945h-2.221a1.945 1.945 0 0 1-1.946-1.945V6.39c0-1.074.872-1.945 1.946-1.945zM8.61 2.223c1.074 0 1.945.87 1.945 1.944v3.889c0 1.073-.87 1.944-1.945 1.944H3.054a1.944 1.944 0 0 1-1.945-1.944V4.167c0-1.074.87-1.944 1.945-1.944z"></path></svg></div><div class="grow">Products</div><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 25 26" class="shrink-0 size-[25px] text-content-tertiary"><path fill="currentColor" fill-rule="evenodd" d="M10.274 7.024a.937.937 0 0 1 1.325 0l5.313 5.313a.94.94 0 0 1 0 1.325l-5.313 5.312a.937.937 0 0 1-1.325-1.325l4.65-4.65-4.65-4.65a.937.937 0 0 1 0-1.325" clip-rule="evenodd"></path></svg></button></li><li class=""><button class="w-full text-left py-2.5 flex items-center gap-4"><div class="p-4 rounded-md flex items-center justify-center text-alarmalade-500 bg-surface-secondary group-hover:bg-surface-tertiary transition"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="shrink-0 size-5"><path fill="currentColor" d="M10 2c-4.411 0-8 3.589-8 8s3.589 8 8 8 8-3.589 8-8-3.589-8-8-8m3.947 5.641c-1.859 1.382-3.435 3.29-4.683 5.669a.75.75 0 0 1-1.331-.005c-.575-1.117-1.218-2.025-1.965-2.776a.75.75 0 1 1 1.064-1.058 11.6 11.6 0 0 1 1.563 1.972c1.239-2.045 2.734-3.726 4.458-5.007a.75.75 0 0 1 .894 1.205"></path></svg></div><div class="grow">Solutions</div><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 25 26" class="shrink-0 size-[25px] text-content-tertiary"><path fill="currentColor" fill-rule="evenodd" d="M10.274 7.024a.937.937 0 0 1 1.325 0l5.313 5.313a.94.94 0 0 1 0 1.325l-5.313 5.312a.937.937 0 0 1-1.325-1.325l4.65-4.65-4.65-4.65a.937.937 0 0 1 0-1.325" clip-rule="evenodd"></path></svg></button></li><li class=""><button class="w-full text-left py-2.5 flex items-center gap-4"><div class="p-4 rounded-md flex items-center justify-center text-alarmalade-500 bg-surface-secondary group-hover:bg-surface-tertiary transition"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="shrink-0 size-5"><path fill="currentColor" d="M3.146 3.76c.411-.314.933-.416 1.431-.278l4.719 1.303v11.303l-5.59-1.543A1.646 1.646 0 0 1 2.5 12.963V5.064c0-.516.236-.992.646-1.305m12.277-.278a1.63 1.63 0 0 1 1.43.277c.412.313.647.789.647 1.305v7.898c0 .736-.495 1.387-1.204 1.582l-5.593 1.543V4.785z"></path></svg></div><div class="grow">Resources</div><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 25 26" class="shrink-0 size-[25px] text-content-tertiary"><path fill="currentColor" fill-rule="evenodd" d="M10.274 7.024a.937.937 0 0 1 1.325 0l5.313 5.313a.94.94 0 0 1 0 1.325l-5.313 5.312a.937.937 0 0 1-1.325-1.325l4.65-4.65-4.65-4.65a.937.937 0 0 1 0-1.325" clip-rule="evenodd"></path></svg></button></li><li class=""><button class="w-full text-left py-2.5 flex items-center gap-4"><div class="p-4 rounded-md flex items-center justify-center text-alarmalade-500 bg-surface-secondary group-hover:bg-surface-tertiary transition"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="shrink-0 size-5"><path fill="currentColor" d="M12.914 3A4.62 4.62 0 0 0 9.75 4.306 4.64 4.64 0 0 0 6.572 3C4.042 3.015 1.99 5.084 2 7.609c0 5.253 5.306 8.429 6.932 9.278a1.77 1.77 0 0 0 1.635 0c1.626-.848 6.933-4.024 6.933-9.275.009-2.528-2.042-4.597-4.586-4.612"></path></svg></div><div class="grow">Customers</div><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 25 26" class="shrink-0 size-[25px] text-content-tertiary"><path fill="currentColor" fill-rule="evenodd" d="M10.274 7.024a.937.937 0 0 1 1.325 0l5.313 5.313a.94.94 0 0 1 0 1.325l-5.313 5.312a.937.937 0 0 1-1.325-1.325l4.65-4.65-4.65-4.65a.937.937 0 0 1 0-1.325" clip-rule="evenodd"></path></svg></button></li><li class=""><button class="w-full text-left py-2.5 flex items-center gap-4"><div class="p-4 rounded-md flex items-center justify-center text-alarmalade-500 bg-surface-secondary group-hover:bg-surface-tertiary transition"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="shrink-0 size-5"><path fill="currentColor" d="M16.916 10.663q-.031 0-.062-.002a.75.75 0 0 1-.687-.808 1.25 1.25 0 0 0-.362-.986l-5.792-5.794A.26.26 0 0 0 9.836 3H4.957A.75.75 0 0 1 3.5 2.75v-.5a.75.75 0 0 1 .75-.75h5.586c.461 0 .911.187 1.237.513l5.793 5.793c.571.572.862 1.362.796 2.167a.75.75 0 0 1-.746.689z"></path><path fill="currentColor" d="M15.366 10.806 9.573 5.013A1.76 1.76 0 0 0 8.336 4.5H2.75a.75.75 0 0 0-.75.75v5.586c0 .467.182.907.513 1.237l5.793 5.793a2.74 2.74 0 0 0 1.944.804 2.74 2.74 0 0 0 1.944-.804l3.172-3.172c.52-.519.806-1.209.806-1.944s-.286-1.425-.806-1.944M6.25 10C5.561 10 5 9.439 5 8.75S5.561 7.5 6.25 7.5s1.25.561 1.25 1.25S6.939 10 6.25 10"></path></svg></div><div class="grow">Pricing</div><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 25 26" class="shrink-0 size-[25px] text-content-tertiary"><path fill="currentColor" fill-rule="evenodd" d="M10.274 7.024a.937.937 0 0 1 1.325 0l5.313 5.313a.94.94 0 0 1 0 1.325l-5.313 5.312a.937.937 0 0 1-1.325-1.325l4.65-4.65-4.65-4.65a.937.937 0 0 1 0-1.325" clip-rule="evenodd"></path></svg></button></li><li class=""><button class="w-full text-left py-2.5 flex items-center gap-4"><div class="p-4 rounded-md flex items-center justify-center text-alarmalade-500 bg-surface-secondary group-hover:bg-surface-tertiary transition"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 21 20" class="shrink-0 size-5"><path fill="currentColor" d="M10.664 10.75a2.75 2.75 0 1 0 0-5.5 2.75 2.75 0 0 0 0 5.5M15.304 15.482a4.775 4.775 0 0 0-4.64-3.732c-2.216 0-4.168 1.57-4.64 3.732-.19.864.348 1.749 1.224 2.013a11.8 11.8 0 0 0 6.832 0c.876-.265 1.413-1.15 1.224-2.014M5.914 6.5a2.25 2.25 0 1 0 0-4.5 2.25 2.25 0 0 0 0 4.5M7.682 11.022A4.23 4.23 0 0 1 6.414 8c0-.166.03-.323.049-.484-.132-.007-.262-.015-.406-.019l-.143-.003a4.33 4.33 0 0 0-4.2 3.263c-.224.881.33 1.81 1.236 2.072a9.6 9.6 0 0 0 2.394.404 6.3 6.3 0 0 1 2.338-2.21M15.414 6.5a2.25 2.25 0 1 0 0-4.5 2.25 2.25 0 0 0 0 4.5M13.646 11.022A4.23 4.23 0 0 0 14.914 8c0-.166-.03-.323-.049-.484.132-.007.262-.015.406-.019l.143-.003a4.33 4.33 0 0 1 4.2 3.263c.224.881-.33 1.81-1.236 2.072a9.6 9.6 0 0 1-2.394.404 6.3 6.3 0 0 0-2.338-2.21"></path></svg></div><div class="grow">Careers</div><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 25 26" class="shrink-0 size-[25px] text-content-tertiary"><path fill="currentColor" fill-rule="evenodd" d="M10.274 7.024a.937.937 0 0 1 1.325 0l5.313 5.313a.94.94 0 0 1 0 1.325l-5.313 5.312a.937.937 0 0 1-1.325-1.325l4.65-4.65-4.65-4.65a.937.937 0 0 1 0-1.325" clip-rule="evenodd"></path></svg></button></li></ul><div class="ml-auto flex flex-col lg:flex-row gap-2 lg:gap-2 flex-shrink-0 transition duration-500 text-gray-600"><a class="relative inline-flex gap-2 items-center justify-center rounded-full focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:opacity-60 group/button cursor-pointer bg-alarmalade-500 text-content-invert hover:bg-alarmalade-600 active:bg-alarmalade-700 text-body-sm-emphasis h-[38px] px-4 py-2 transition-all duration-500 opacity-100 pointer-events-auto" id="nav-bar-request-a-demo" href="/demo">Get a demo</a><a href="https://app.incident.io/login" target="_blank" class="relative inline-flex gap-2 items-center justify-center rounded-full focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:opacity-60 group/button cursor-pointer transition-[color,background-color,border-color,opacity,transform] bg-surface-secondary hover:bg-surface-tertiary active:bg-surface-quaternary text-content-primary text-body-sm-emphasis h-[38px] px-4 py-2">Login</a></div></div></div></div><div class="relative h-full transition-[background-color,backdrop-filter,transform] duration-300 ease-[cubic-bezier(0.2,0,0,1)] z-10 backdrop-blur-md bg-white/80" style="view-transition-name:navigation"><div class="h-20 mx-auto" data-mobile-menu-open="false"><div class="relative flex justify-between h-full z-[99]"><div class="max-w-[1512px] mx-auto px-5 flex-1 flex items-center justify-between"><div class="flex-shrink-0 flex items-center relative transition z-[1]"><a class="transition hover:opacity-80 relative top-[-8px]" href="/"><span class="sr-only">incident.io</span><div class="w-[130px] h-[32px]"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 1000 248" class="h-8 w-auto absolute inset-0 dark:opacity-0 [[data-mobile-menu-open=true]_&amp;]:opacity-100"><g clip-path="url(#workmark__black_svg__a)"><g clip-path="url(#workmark__black_svg__b)"><path fill="#F25533" fill-rule="evenodd" d="M48.803 140.978v23.429C20.788 157.71 0 133.023 0 103.594 0 86.626 7.168 73.165 18.131 59.96 27.222 49.01 49.67 20.01 56.735 4.009c1.716-3.885 6.09-3.735 7.976-1.647 6.033 6.677 13.47 21.26 15.902 37.802.496 3.37.7 6.123.862 8.309.348 4.691.502 6.777 2.93 6.777 3.816 0 6.313-5.795 6.861-11.776.352-3.833 4.148-5.115 6.765-3.833 12.59 6.17 25.437 35.537 28.543 49.949.974 4.52 1.609 9.19 1.609 14.004 0 29.363-20.695 54.006-48.616 60.768v-23.384zm15.289-9.338c9.845 0 17.826-7.567 17.826-16.901 0-14.888-10.959-25.526-16.42-28.76-.369-.218-.553-.328-1.055-.314-.367.01-.91.24-1.172.498-.358.351-.453.752-.643 1.552-.968 4.078-4.015 6.986-7.222 10.045-4.42 4.217-9.141 8.723-9.141 16.979 0 9.334 7.981 16.901 17.827 16.901" clip-rule="evenodd"></path><path fill="#000" d="M158.057 138.153c-13.535 0-23.788 4.919-29.325 12.503v-9.633h-30.76v102.48h30.76v-55.954c0-14.552 7.793-21.111 19.071-21.111 9.843 0 17.841 5.944 17.841 18.651v58.414h30.76V180.58c0-27.67-17.636-42.427-38.347-42.427M262.641 246.373c20.097 0 37.527-10.454 46.346-26.44l-26.864-15.372c-3.281 7.173-10.869 11.477-19.892 11.477-13.329 0-23.172-9.632-23.172-23.775s9.843-23.775 23.172-23.775c9.023 0 16.406 4.303 19.892 11.477l26.864-15.577c-8.819-15.782-26.454-26.235-46.346-26.235-31.169 0-54.342 23.571-54.342 54.11s23.173 54.11 54.342 54.11M332.645 132.005c10.047 0 18.456-8.404 18.456-18.448 0-10.042-8.409-18.446-18.456-18.446-10.049 0-18.456 8.404-18.456 18.446 0 10.044 8.407 18.448 18.456 18.448m-15.38 111.498h30.76v-102.48h-30.76zM440.871 100.031v50.625c-6.767-7.788-16.815-12.503-30.555-12.503-26.863 0-49.01 23.571-49.01 54.11s22.147 54.11 49.01 54.11c13.74 0 23.788-4.714 30.555-12.503v9.634h30.76V100.031zm-24.402 117.238c-14.15 0-24.403-9.634-24.403-25.006s10.253-25.005 24.403-25.005 24.402 9.633 24.402 25.005-10.253 25.006-24.402 25.006M517.576 204.561h72.799q1.23-5.841 1.23-12.298c0-30.949-22.147-54.11-52.086-54.11-32.401 0-54.548 23.571-54.548 54.11s21.736 54.11 56.803 54.11c19.482 0 34.656-7.174 44.705-21.111l-24.609-14.143c-4.101 4.509-11.278 7.789-19.686 7.789-11.278 0-20.711-3.69-24.608-14.347m-.615-22.956c2.871-10.453 10.663-16.192 22.352-16.192 9.228 0 18.456 4.304 21.737 16.192zM665.034 138.153c-13.534 0-23.787 4.919-29.324 12.503v-9.633h-30.76v102.48h30.76v-55.954c0-14.552 7.793-21.111 19.071-21.111 9.843 0 17.841 5.944 17.841 18.651v58.414h30.76V180.58c0-27.67-17.636-42.427-38.348-42.427M780.49 170.537v-29.514h-21.123v-28.695l-30.759 9.224v19.471h-16.407v29.514h16.407v36.278c0 28.695 11.689 40.787 51.882 36.688v-27.875c-13.535.82-21.123 0-21.123-8.813v-36.278zM813.237 246.373c10.663 0 19.48-8.813 19.48-19.472 0-10.657-8.817-19.471-19.48-19.471s-19.482 8.814-19.482 19.471c0 10.659 8.818 19.472 19.482 19.472M862.014 132.005c10.047 0 18.455-8.404 18.455-18.448 0-10.042-8.408-18.446-18.455-18.446-10.049 0-18.456 8.404-18.456 18.446 0 10.044 8.407 18.448 18.456 18.448m-15.381 111.498h30.761v-102.48h-30.761zM945.017 246.373c30.144 0 54.343-23.571 54.343-54.11s-24.199-54.11-54.343-54.11-54.342 23.571-54.342 54.11 24.197 54.11 54.342 54.11m0-29.925c-13.329 0-23.582-9.633-23.582-24.185 0-14.553 10.253-24.185 23.582-24.185s23.583 9.632 23.583 24.185-10.253 24.185-23.583 24.185M48.804 243.508h30.76v-78.701s-30.76 5.525-30.76 34.995z"></path></g></g><defs><clipPath id="workmark__black_svg__a"><path fill="#fff" d="M0 0h1000v248H0z"></path></clipPath><clipPath id="workmark__black_svg__b"><path fill="#fff" d="M0 .91h1000v246.18H0z"></path></clipPath></defs></svg><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 1000 248" class="h-8 w-auto absolute inset-0 opacity-0 dark:opacity-100 [[data-mobile-menu-open=true]_&amp;]:opacity-0"><g fill="#fff" clip-path="url(#workmark__fullwhite_svg__a)"><path fill-rule="evenodd" d="M48.803 140.978v23.429C20.788 157.71 0 133.023 0 103.594 0 86.626 7.168 73.165 18.131 59.96 27.222 49.01 49.67 20.01 56.735 4.009c1.716-3.885 6.09-3.735 7.976-1.647 6.033 6.677 13.47 21.26 15.902 37.802.496 3.37.7 6.123.862 8.309.348 4.691.502 6.777 2.93 6.777 3.816 0 6.313-5.795 6.861-11.776.352-3.833 4.148-5.115 6.765-3.833 12.59 6.17 25.437 35.537 28.543 49.949.974 4.52 1.609 9.19 1.609 14.004 0 29.363-20.695 54.006-48.616 60.768v-23.384zm15.289-9.338c9.845 0 17.826-7.567 17.826-16.901 0-14.888-10.959-25.526-16.42-28.76-.369-.218-.553-.328-1.055-.314-.367.01-.91.24-1.172.498-.358.351-.453.752-.643 1.552-.968 4.078-4.015 6.986-7.222 10.045-4.42 4.217-9.141 8.723-9.141 16.979 0 9.334 7.981 16.901 17.827 16.901" clip-rule="evenodd"></path><path d="M158.057 138.153c-13.535 0-23.788 4.919-29.325 12.503v-9.633h-30.76v102.48h30.76v-55.954c0-14.552 7.793-21.111 19.071-21.111 9.843 0 17.841 5.944 17.841 18.651v58.414h30.76V180.58c0-27.67-17.636-42.427-38.347-42.427M262.641 246.373c20.097 0 37.527-10.454 46.346-26.44l-26.864-15.372c-3.281 7.173-10.869 11.477-19.892 11.477-13.329 0-23.172-9.632-23.172-23.775s9.843-23.775 23.172-23.775c9.023 0 16.406 4.303 19.892 11.477l26.864-15.577c-8.819-15.782-26.454-26.235-46.346-26.235-31.169 0-54.342 23.571-54.342 54.11s23.173 54.11 54.342 54.11M332.645 132.005c10.047 0 18.456-8.404 18.456-18.448 0-10.042-8.409-18.446-18.456-18.446-10.049 0-18.456 8.404-18.456 18.446 0 10.044 8.407 18.448 18.456 18.448m-15.38 111.498h30.76v-102.48h-30.76zM440.871 100.031v50.625c-6.767-7.788-16.815-12.503-30.555-12.503-26.863 0-49.01 23.571-49.01 54.11s22.147 54.11 49.01 54.11c13.74 0 23.788-4.714 30.555-12.503v9.634h30.76V100.031zm-24.402 117.238c-14.15 0-24.403-9.634-24.403-25.006s10.253-25.005 24.403-25.005 24.402 9.633 24.402 25.005-10.253 25.006-24.402 25.006M517.576 204.561h72.799q1.23-5.841 1.23-12.298c0-30.949-22.147-54.11-52.086-54.11-32.401 0-54.548 23.571-54.548 54.11s21.736 54.11 56.803 54.11c19.482 0 34.656-7.174 44.705-21.111l-24.609-14.143c-4.101 4.509-11.278 7.789-19.686 7.789-11.278 0-20.711-3.69-24.608-14.347m-.615-22.956c2.871-10.453 10.663-16.192 22.352-16.192 9.228 0 18.456 4.304 21.737 16.192zM665.034 138.153c-13.534 0-23.787 4.919-29.324 12.503v-9.633h-30.76v102.48h30.76v-55.954c0-14.552 7.793-21.111 19.071-21.111 9.843 0 17.841 5.944 17.841 18.651v58.414h30.76V180.58c0-27.67-17.636-42.427-38.348-42.427M780.49 170.537v-29.514h-21.123v-28.695l-30.759 9.224v19.471h-16.407v29.514h16.407v36.278c0 28.695 11.689 40.787 51.882 36.688v-27.875c-13.535.82-21.123 0-21.123-8.813v-36.278zM813.237 246.373c10.663 0 19.48-8.813 19.48-19.472 0-10.657-8.817-19.471-19.48-19.471s-19.482 8.814-19.482 19.471c0 10.659 8.818 19.472 19.482 19.472M862.014 132.005c10.047 0 18.455-8.404 18.455-18.448 0-10.042-8.408-18.446-18.455-18.446-10.049 0-18.456 8.404-18.456 18.446 0 10.044 8.407 18.448 18.456 18.448m-15.381 111.498h30.761v-102.48h-30.761zM945.017 246.373c30.144 0 54.343-23.571 54.343-54.11s-24.199-54.11-54.343-54.11-54.342 23.571-54.342 54.11 24.197 54.11 54.342 54.11m0-29.925c-13.329 0-23.582-9.633-23.582-24.185 0-14.553 10.253-24.185 23.582-24.185s23.583 9.632 23.583 24.185-10.253 24.185-23.583 24.185M48.804 243.508h30.76v-78.701s-30.76 5.525-30.76 34.995z"></path></g><defs><clipPath id="workmark__fullwhite_svg__a"><path fill="#fff" d="M0 .91h1000v246.18H0z"></path></clipPath></defs></svg></div></a><div class="border border-stroke-primary bg-white rounded-lg shadow-lg p-1 flex flex-col gap-1 absolute top-full mt-2 left-0 z-50 w-[250px] transition-opacity duration-200 opacity-0 pointer-events-none"><div class="p-2 pt-1 text-body-xs-emphasis border-b border-stroke-primary">Brand Assets</div><div class="flex flex-col gap-1 text-body-xs text-content-secondary"><a href="/brand/resources/incidentio-brand-pngs.zip" class="p-2 flex gap-2 items-center group rounded hover:bg-surface-secondary transition"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 16 16" class="shrink-0 size-4 text-content-primary"><path fill="#fff" d="M2.8 1.6A1.6 1.6 0 0 1 4.4 0h6.4c.127 0 .25.05.34.141l4.318 4.318c.09.09.141.213.141.341v9.6A1.6 1.6 0 0 1 14 16H4.4a1.6 1.6 0 0 1-1.6-1.6z"></path><path stroke="#161618" stroke-opacity="0.08" stroke-width="0.6" d="M4.399.3h6.4c.049 0 .095.02.13.054l4.317 4.317c.034.034.053.08.053.129v9.6a1.3 1.3 0 0 1-1.3 1.3H4.4a1.3 1.3 0 0 1-1.3-1.3V1.6a1.3 1.3 0 0 1 1.3-1.3Z"></path><path stroke="#161618" stroke-opacity="0.08" stroke-width="0.6" d="M10.8.2v3a1.6 1.6 0 0 0 1.6 1.6h3"></path><rect width="11.4" height="6.6" x="0.3" y="7.1" fill="#F25533" rx="0.8"></rect><path fill="#fff" d="M1.966 11.9V8.99h1.147q.332 0 .564.127.234.126.355.348.124.222.124.512t-.125.511a.86.86 0 0 1-.362.345 1.2 1.2 0 0 1-.571.124h-.732v-.493h.632a.6.6 0 0 0 .293-.061.4.4 0 0 0 .173-.172.54.54 0 0 0 .059-.254q0-.145-.059-.253a.4.4 0 0 0-.173-.17.63.63 0 0 0-.295-.06H2.58V11.9zm5.023-2.91v2.91h-.531l-1.266-1.831H5.17V11.9h-.615V8.99h.54l1.255 1.83h.026V8.99zm2.437.941a.6.6 0 0 0-.084-.183.567.567 0 0 0-.308-.223.8.8 0 0 0-.217-.03.7.7 0 0 0-.392.111.73.73 0 0 0-.262.322 1.3 1.3 0 0 0-.094.515q0 .303.093.517.091.213.261.325.17.11.4.11a.8.8 0 0 0 .356-.073.53.53 0 0 0 .227-.212.63.63 0 0 0 .08-.322l.125.018h-.75v-.463h1.217v.367q0 .383-.162.659a1.1 1.1 0 0 1-.446.423 1.4 1.4 0 0 1-.65.148q-.41 0-.72-.18a1.25 1.25 0 0 1-.482-.516 1.7 1.7 0 0 1-.172-.796q0-.354.102-.63.104-.279.29-.472.186-.194.433-.294t.536-.1q.246 0 .46.072.213.07.378.201a1.1 1.1 0 0 1 .406.706z"></path></svg><div class="grow">Download .PNG logos</div><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 25 26" class="shrink-0 size-4"><path fill="currentColor" fill-rule="evenodd" d="M10.274 7.024a.937.937 0 0 1 1.325 0l5.313 5.313a.94.94 0 0 1 0 1.325l-5.313 5.312a.937.937 0 0 1-1.325-1.325l4.65-4.65-4.65-4.65a.937.937 0 0 1 0-1.325" clip-rule="evenodd"></path></svg></a><a href="/brand/resources/incidentio-brand-svgs.zip" class="p-2 flex gap-2 items-center group rounded hover:bg-surface-secondary transition"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 16 16" class="shrink-0 size-4 text-content-primary"><path fill="#fff" d="M2.8 1.6A1.6 1.6 0 0 1 4.4 0h6.4c.128 0 .251.05.342.141l4.317 4.318c.09.09.141.213.141.341v9.6A1.6 1.6 0 0 1 14 16H4.4a1.6 1.6 0 0 1-1.6-1.6z"></path><path stroke="#161618" stroke-opacity="0.08" stroke-width="0.6" d="M4.4.3h6.4q.076.002.13.054l4.317 4.317c.034.034.053.08.053.129v9.6a1.3 1.3 0 0 1-1.3 1.3H4.4a1.3 1.3 0 0 1-1.3-1.3V1.6A1.3 1.3 0 0 1 4.4.3Z"></path><path stroke="#161618" stroke-opacity="0.08" stroke-width="0.6" d="M10.8.2v3a1.6 1.6 0 0 0 1.6 1.6h3"></path><rect width="11.4" height="6.6" x="0.299" y="7.1" fill="#F25533" rx="0.8"></rect><path fill="#fff" d="M3.584 9.828a.36.36 0 0 0-.146-.267.58.58 0 0 0-.35-.096.7.7 0 0 0-.255.043.36.36 0 0 0-.16.115.28.28 0 0 0-.053.168.24.24 0 0 0 .032.136.34.34 0 0 0 .101.1.8.8 0 0 0 .148.073q.084.03.179.051l.261.063q.19.042.35.114.16.07.275.174a.73.73 0 0 1 .248.567.8.8 0 0 1-.137.463.87.87 0 0 1-.386.303q-.252.106-.607.106-.353 0-.613-.108a.9.9 0 0 1-.407-.32.96.96 0 0 1-.152-.526h.596a.45.45 0 0 0 .083.244q.075.096.2.146.127.049.286.049a.7.7 0 0 0 .271-.046.4.4 0 0 0 .18-.126.3.3 0 0 0 .065-.186q0-.098-.058-.165a.44.44 0 0 0-.168-.114 2 2 0 0 0-.269-.085l-.316-.08a1.3 1.3 0 0 1-.581-.28.65.65 0 0 1-.212-.512.76.76 0 0 1 .14-.462.95.95 0 0 1 .394-.308q.25-.11.568-.11.324 0 .566.11a.9.9 0 0 1 .378.308q.135.197.139.458zm1.569-.837.703 2.21h.027l.704-2.21h.682L6.266 11.9h-.792L4.47 8.99zm4.231.94a.6.6 0 0 0-.083-.183.566.566 0 0 0-.308-.223.8.8 0 0 0-.218-.03.7.7 0 0 0-.392.111.73.73 0 0 0-.261.322 1.3 1.3 0 0 0-.094.515q0 .303.092.517.093.213.261.325.17.11.4.11a.8.8 0 0 0 .356-.073.53.53 0 0 0 .227-.212.63.63 0 0 0 .08-.322l.125.018h-.75v-.463h1.217v.367q0 .383-.162.659a1.1 1.1 0 0 1-.446.423 1.4 1.4 0 0 1-.65.148q-.41 0-.719-.18a1.25 1.25 0 0 1-.483-.516 1.7 1.7 0 0 1-.172-.796q0-.354.102-.63.105-.279.29-.472.186-.194.433-.294.248-.1.536-.1.247 0 .46.072.213.07.378.201a1.1 1.1 0 0 1 .406.706z"></path></svg><div class="grow">Download .SVG logos</div><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 25 26" class="shrink-0 size-4"><path fill="currentColor" fill-rule="evenodd" d="M10.274 7.024a.937.937 0 0 1 1.325 0l5.313 5.313a.94.94 0 0 1 0 1.325l-5.313 5.312a.937.937 0 0 1-1.325-1.325l4.65-4.65-4.65-4.65a.937.937 0 0 1 0-1.325" clip-rule="evenodd"></path></svg></a><a href="/downloads/incidentio-brand-guidelines.pdf?v=20260826" class="p-2 flex gap-2 items-center group rounded hover:bg-surface-secondary transition"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="shrink-0 size-4 text-alarmalade-500"><path fill="currentColor" d="m16.487 6.427-3.914-3.914A1.74 1.74 0 0 0 11.336 2H5.75A2.75 2.75 0 0 0 3 4.75v10.5A2.75 2.75 0 0 0 5.75 18h8.5A2.75 2.75 0 0 0 17 15.25V7.664c0-.467-.182-.907-.513-1.237M6.75 7h2a.75.75 0 0 1 0 1.5h-2a.75.75 0 0 1 0-1.5m6.5 7.5h-6.5a.75.75 0 0 1 0-1.5h6.5a.75.75 0 0 1 0 1.5m0-3h-6.5a.75.75 0 0 1 0-1.5h6.5a.75.75 0 0 1 0 1.5m2.182-4H12.5c-.55 0-1-.45-1-1V3.579l.013-.005 3.922 3.921z"></path></svg><div class="grow">Download Brand Guidelines</div><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 25 26" class="shrink-0 size-4"><path fill="currentColor" fill-rule="evenodd" d="M10.274 7.024a.937.937 0 0 1 1.325 0l5.313 5.313a.94.94 0 0 1 0 1.325l-5.313 5.312a.937.937 0 0 1-1.325-1.325l4.65-4.65-4.65-4.65a.937.937 0 0 1 0-1.325" clip-rule="evenodd"></path></svg></a><a href="/brand" class="p-2 flex gap-2 items-center group rounded hover:bg-surface-secondary transition"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 21" class="shrink-0 size-4 text-content-primary"><path fill="currentColor" d="M14.25 3.5h-8.5A2.75 2.75 0 0 0 3 6.25v6.568c0 1.336 1.616 2.006 2.56 1.06L10.44 9H7.75a.75.75 0 0 1 0-1.5h4.5a.75.75 0 0 1 .75.75v4.5a.75.75 0 0 1-1.5 0V10.06L6.622 14.94c-.945.945-.276 2.56 1.06 2.56h6.568A2.75 2.75 0 0 0 17 14.75v-8.5a2.75 2.75 0 0 0-2.75-2.75"></path></svg><div class="grow">Visit brand center</div><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 25 26" class="shrink-0 size-4"><path fill="currentColor" fill-rule="evenodd" d="M10.274 7.024a.937.937 0 0 1 1.325 0l5.313 5.313a.94.94 0 0 1 0 1.325l-5.313 5.312a.937.937 0 0 1-1.325-1.325l4.65-4.65-4.65-4.65a.937.937 0 0 1 0-1.325" clip-rule="evenodd"></path></svg></a></div></div></div><nav aria-label="Main" data-orientation="horizontal" dir="ltr" class="hidden flex-1 lg:flex items-stretch justify-center h-full grow absolute inset-x-0"><div style="position:relative"><ul data-orientation="horizontal" class="h-full flex items-stretch justify-center gap-0" dir="ltr"><li><button id="radix-_R_16aav5ubsraivb_-trigger-products" data-state="closed" aria-expanded="false" class="relative font-medium border-transparent inline-flex items-center leading-none whitespace-nowrap transition-colors outline-0 dark:text-white focus-visible:text-orange-600 hover:text-orange-600 data-[state=open]:text-orange-600 text-gray-900 px-3 py-2 h-full group" data-radix-collection-item="">Products</button></li><li><button id="radix-_R_16aav5ubsraivb_-trigger-solutions" data-state="closed" aria-expanded="false" class="relative font-medium border-transparent inline-flex items-center leading-none whitespace-nowrap transition-colors outline-0 dark:text-white focus-visible:text-orange-600 hover:text-orange-600 data-[state=open]:text-orange-600 text-gray-900 px-3 py-2 h-full group" data-radix-collection-item="">Solutions</button></li><li><button id="radix-_R_16aav5ubsraivb_-trigger-resources" data-state="closed" aria-expanded="false" class="relative font-medium border-transparent inline-flex items-center leading-none whitespace-nowrap transition-colors outline-0 dark:text-white focus-visible:text-orange-600 hover:text-orange-600 data-[state=open]:text-orange-600 text-gray-900 px-3 py-2 h-full group" data-radix-collection-item="">Resources</button></li><li><button id="radix-_R_16aav5ubsraivb_-trigger-customers" data-state="closed" aria-expanded="false" class="relative font-medium border-transparent inline-flex items-center leading-none whitespace-nowrap transition-colors outline-0 dark:text-white focus-visible:text-orange-600 hover:text-orange-600 data-[state=open]:text-orange-600 text-gray-900 px-3 py-2 h-full group" data-radix-collection-item="">Customers</button></li><li><a class="relative font-medium border-transparent inline-flex items-center leading-none whitespace-nowrap transition-colors outline-0 dark:text-white focus-visible:text-orange-600 hover:text-orange-600 data-[state=open]:text-orange-600 text-gray-900 px-3 py-2 h-full" data-radix-collection-item="" href="/pricing">Pricing</a></li><li><a class="relative font-medium border-transparent inline-flex items-center leading-none whitespace-nowrap transition-colors outline-0 dark:text-white focus-visible:text-orange-600 hover:text-orange-600 data-[state=open]:text-orange-600 text-gray-900 px-3 py-2 h-full" data-radix-collection-item="" href="/careers">Careers</a></li></ul></div></nav><div class="h-full hidden lg:flex flex-shrink-0 items-center relative z-[1]"><div class="ml-auto flex flex-col lg:flex-row gap-2 lg:gap-2 flex-shrink-0 transition duration-500 text-gray-600"><a class="relative inline-flex gap-2 items-center justify-center rounded-full focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:opacity-60 group/button cursor-pointer bg-alarmalade-500 text-content-invert hover:bg-alarmalade-600 active:bg-alarmalade-700 text-body-sm-emphasis h-[38px] px-4 py-2 transition-all duration-500 opacity-100 pointer-events-auto" id="nav-bar-request-a-demo" href="/demo">Get a demo</a><a href="https://app.incident.io/login" target="_blank" class="relative inline-flex gap-2 items-center justify-center rounded-full focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:opacity-60 group/button cursor-pointer transition-[color,background-color,border-color,opacity,transform] bg-surface-secondary hover:bg-surface-tertiary active:bg-surface-quaternary text-content-primary text-body-sm-emphasis h-[38px] px-4 py-2">Login</a></div></div></div><div class="absolute inset-y-0 right-5 flex lg:hidden items-center lg:static lg:inset-auto lg:ml-6 lg:pr-0"><div class="relative h-10"><button class="inline-flex items-center justify-center p-2 -mr-2 rounded-md text-gray-500 hover:text-gray-600 hover:bg-gray-100 focus:outline-none focus:ring-2 focus:ring-inset focus:ring-indigo-500 h-10 dark:text-white"><span class="sr-only">Open main menu</span><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" aria-hidden="true" data-slot="icon" class="block h-6 w-6"><path stroke-linecap="round" stroke-linejoin="round" d="M3.75 6.75h16.5M3.75 12h16.5m-16.5 5.25h16.5"></path></svg></button></div></div></div></div></div><div style="top:0" class="fixed inset-0 bg-black/25 backdrop-blur-sm h-screen transition-opacity delay-100 pointer-events-none opacity-0"></div></nav><div class="relative mx-auto max-w-[1192px] max-xl:px-4 flex flex-col gap-12 md:gap-20 lg:pt-20"><section class="pb-10 md:pb-20 flex flex-col lg:flex-row gap-4 relative border-b border-stroke-primary" style="view-transition-name:none"><div class="max-w-[388px] pr-6 flex flex-col gap-6 grow lg:sticky-sidebar"><div class="flex flex-col gap-3"><button class="relative inline-flex gap-2 items-center justify-center rounded-full focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:opacity-60 group/button cursor-pointer transition-[color,background-color,border-color,opacity,transform] bg-surface-secondary hover:bg-surface-tertiary active:bg-surface-quaternary text-content-primary text-body-sm-emphasis h-[38px] px-4 py-2 self-start"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="none" class="transition ease-in-out inline-block rotate-180 size-4"><path d="M12.8 8.59965H3.00002C2.66882 8.59965 2.40002 8.33085 2.40002 7.99965C2.40002 7.66845 2.66882 7.39965 3.00002 7.39965H12.8C13.1312 7.39965 13.4 7.66845 13.4 7.99965C13.4 8.33085 13.1312 8.59965 12.8 8.59965Z" fill="currentColor"></path><path d="M9.60002 11.9996C9.53122 11.9996 9.46082 11.9876 9.39202 11.9628C9.08082 11.8476 8.92242 11.5028 9.03682 11.1924C9.66242 9.49645 10.7328 8.53005 11.5816 8.00045C10.7328 7.47085 9.66322 6.50445 9.03682 4.80845C8.92162 4.49725 9.08082 4.15245 9.39202 4.03805C9.70322 3.92125 10.048 4.08285 10.1624 4.39325C11.0656 6.84205 13.0656 7.39805 13.1496 7.42045C13.4144 7.49085 13.5992 7.73085 13.5976 8.00365C13.596 8.27725 13.412 8.51565 13.1464 8.58285C13.0648 8.60445 11.0656 9.16045 10.1624 11.6092C10.0728 11.8516 9.84402 12.0012 9.59922 12.0012L9.60002 11.9996Z" fill="currentColor"></path></svg><div class="hidden md:flex absolute whitespace-nowrap items-center gap-2 top-0 text-xs text-white px-3 py-2 bg-surface-invert rounded-lg pointer-events-none translate-y-[-40px] transition opacity-0 group-hover/button:opacity-100">All posts<kbd class="flex items-center justify-center uppercase hs-5 text-[10px] px-1 rounded border border-stroke-invert text-content-secondary-invert">Esc</kbd></div></button></div><div class="flex flex-col gap-8 max-lg:hidden"><ul class="flex flex-col text-body-sm"><li class="flex flex-col gap-3 relative group"><a class="block flex items-center transition text-pretty py-2 hover:text-alarmalade-500" href="#content-top">Intermittent downtime from repeated crashes</a></li><li class="flex flex-col gap-3 relative group"><a class="flex items-center transition text-pretty py-2 hover:text-alarmalade-500" href="#background">Background</a></li><li class="flex flex-col gap-3 relative group"><a class="flex items-center transition text-pretty py-2 hover:text-alarmalade-500" href="#initial-crash">Initial crash</a></li><li class="flex flex-col gap-3 relative group"><a class="flex items-center transition text-pretty py-2 hover:text-alarmalade-500" href="#subsequent-crashes">Subsequent crashes</a></li><li class="flex flex-col gap-3 relative group"><a class="flex items-center transition text-pretty py-2 hover:text-alarmalade-500" href="#regroup">Regroup</a></li><li class="flex flex-col gap-3 relative group"><a class="flex items-center transition text-pretty py-2 hover:text-alarmalade-500" href="#mitigation-1-ensuring-we-handle-message-panics">Mitigation 1: Ensuring we handle message panics</a></li><li class="flex flex-col gap-3 relative group"><a class="flex items-center transition text-pretty py-2 hover:text-alarmalade-500" href="#mitigation-2-separate-our-app-by-work-type">Mitigation 2: Separate our app by work type</a></li><li class="flex flex-col gap-3 relative group"><a class="flex items-center transition text-pretty py-2 hover:text-alarmalade-500" href="#conclusion">Conclusion</a></li></ul></div></div><div class="lg:max-w-[788px] grow flex flex-col gap-4 lg:gap-10" id="content-top"><div class="flex flex-col gap-4"><div class="flex flex-col gap-2"><h1 class="text-h3 text-balance">Intermittent downtime from repeated crashes</h1></div><div class="flex flex-col gap-2 md:flex-row justify-between mt-auto"><div class="flex items-center gap-2"><span class="flex items-center gap-2"><div class="size-[22px] rounded-full overflow-hidden shrink-0"></div><a class="text-body-sm-emphasis leading-[1.1] w-min sm:w-auto transition hover:text-alarmalade-500" href="/blog/author/lawrence">Lawrence Jones</a></span></div><div class="text-body-sm text-content-secondary">November 30, 2022<span> — <!-- -->14 min read</span></div></div></div><div class="relative"><span class="relative overflow-hidden block align-top transition-opacity opacity-0 w-full rounded-lg"><img alt="" width="4800" height="1874" decoding="async" data-nimg="1" class="!m-0 w-full h-full object-cover" style="color:transparent;view-transition-name:blog-intermittent-downtime" sizes="100vw" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=60&amp;q=75 60w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=64&amp;q=75 64w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=96&amp;q=75 96w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=128&amp;q=75 128w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=256&amp;q=75 256w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=384&amp;q=75 384w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=640&amp;q=75 640w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=750&amp;q=75 750w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=828&amp;q=75 828w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=1080&amp;q=75 1080w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=1200&amp;q=75 1200w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=1920&amp;q=75 1920w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=2048&amp;q=75 2048w, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=3840&amp;q=75 3840w" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=3840&amp;q=75"/></span></div><div class="prose prose-orange text-body max-w-none mb-6"><p class="">On Friday 18th November 2022 we experienced 13 minutes of downtime over a period of 32 minutes from 15:40 to 16:12 GMT.</p><p class="">At incident.io, our customers depend on us to be available even when everything else is down, so we take incidents like this seriously. We also sincerely believe in the power of transparency, and want to be as open as possible in these situations, sharing what details we can.</p><p class="">That’s why we’re sharing this technical write-up, explaining what happened and also to share the measures we’ve since taken to improve our reliability.</p><p class="">Here&#x27;s a summary timeline with links, if you&#x27;d like to jump through:</p><div><div class="rawhtml"><table>
<thead>
<tr>
<th>Time</th>
<th>Event</th>
</tr>
</thead>
<tbody>
<tr>
<td>15:40</td>
<td>
<a href="#initial-crash">Initial crash</a>
</td>
</tr>
<tr>
<td>15:42</td>
<td>
<a href="#initial-crash">Restarted dynos, back up</a>
</td>
</tr>
<tr>
<td>15:47</td>
<td>
<a href="#subsequent-crashes">Subsequent crashes</a>
</td>
</tr>
<tr>
<td>16:00</td>
<td>
<a href="#subsequent-crashes">Begin purging GCP subscriptions</a>
</td>
</tr>
<tr>
<td>16:12</td>
<td>
<a href="#subsequent-crashes">App stops crashing</a>
</td>
</tr>
<tr>
<td>16:13</td>
<td>
<a href="#regroup">Regroup, start on mitigations</a>
</td>
</tr>
<tr>
<td>16:48</td>
<td>
<a href="#mitigation-1-ensuring-we-handle-message-panics">Deploy fix to handle panics</a>
</td>
</tr>
<tr>
<td>17:24</td>
<td>
<a href="#mitigation-2-separate-our-app-by-work-type">Deploy split of web, worker and cron into different dynos</a>
</td>
</tr>
</tbody>
</table></div></div><h2 id="background" class=""><a class="heading-link" href="#background"><span class="icon icon-link"></span></a>Background</h2><ul><li>We run our app as a Go monolith in Heroku deployed using container buildpack, Heroku Postgres as a database, and GCP Pub/Sub as an async message queue.</li><li>Our application runs as a single binary running web, worker and cron threads.</li><li>For async work, we have separate Pub/Sub subscriptions that are served by message handlers within our app. If the message handler returns an error, we rely on Pub/Sub to perform back-off on message redelivery.</li><li>We take care to ensure all entrypoints to the app – incoming web requests, Pub/Sub messages in the event queue, cron jobs – are wrapped in panic handlers to prevent an unhandled panic from taking the app down.</li></ul><h2 id="initial-crash" class=""><a class="heading-link" href="#initial-crash"><span class="icon icon-link"></span></a>Initial crash</h2><p class="">Twenty minutes before our end-of-week team debrief, our on-call engineer was paged due to an &quot;<a href="https://devcenter.heroku.com/articles/error-codes#h10-app-crashed" target="_blank" rel="noopener noreferrer" class="">H10 - App crashed</a>&quot; error code from Heroku, which means the app has terminated.</p><p class="">This type of crash happens when a Heroku dyno totally dies, which can happen in response to many events:</p><ol><li>An unhandled panic in the Go runtime has caused the app to die.</li><li>Something has caused the application to exceed its memory limit and we’ve been OOM (out-of-memory) killed.</li><li>Heroku has a fault, or some infrastructure error has caused the process to die.</li></ol><p class="">Regardless of the cause, Heroku responds to this event by following its <a href="https://devcenter.heroku.com/articles/dynos#dyno-crash-restart-policy" target="_blank" rel="noopener noreferrer" class="">Dyno crash restart policy</a>, which can be summarised as:</p><ul><li>The first time a dyno crashes, it will be restarted immediately</li><li>If the dyno crashes again, it will be subject to a cool-off period (up-to 20m) before a restart is attempted.</li></ul><p class="">Subsequent crashes increase the ‘cool off’ period, but it matters little to us: waiting 20m to restart our application would be an unacceptable amount of downtime.</p><p class="">From previous incidents, we’ve learned that manually restarting dynos via the Heroku console is a sensible response whenever an H10 occurs that we don’t immediately recover from. We could see from our monitoring that we hadn’t immediately recovered, suggesting we’d crashed quickly after the first restart, so we triggered a manual reboot at 15:42.</p><p class="">The manual reboot brought the app back up, which meant our PagerDuty automation activated (see our <a href="https://incident.io/blog/realtime-response" target="_blank" rel="noopener noreferrer" class="">response flow</a>) and we get an <a href="https://incident.io" target="_blank" rel="noopener noreferrer" class="">incident.io</a> incident channel for this incident. The on-call and response team join the channel and start examining logs.</p><h2 id="subsequent-crashes" class=""><a class="heading-link" href="#subsequent-crashes"><span class="icon icon-link"></span></a>Subsequent crashes</h2><p class="">Not yet knowing what caused the crash, the team start collating possible causes. Prior incident experience means our guess is a bad Pub/Sub event being consumed that triggers one of the H10 failure cases whenever it is processed, a situation often called a “poison pill”.</p><p class="">The immediate questions are:</p><ul><li>Can we determine it was an event that caused this?</li><li>If so, which event is it?</li></ul><p class="">As we begin to investigate, we get paged again: the app is down due to – presumably – the same error, and we assign an engineer to watch for subsequent crashes so we can quickly trigger a manual start, which we know brings the app back up.</p><p class="">It’s now 15:47, and we allocate people onto several streams of work:</p><ul><li>Someone to lead the incident, which means coordinating our response.</li><li>As mentioned, one person watching for crashes and manually restarting Dynos.</li><li>Someone is assigned to Communications, which involves updating our public status page.</li></ul><p class="">And a few engineers start debugging using a variety of tools, from logs, metrics and traces to try and figure out what’s going on.</p><blockquote>See how we use our tools in <a href="https://incident.io/blog/tracing" target="_blank" rel="noopener noreferrer" class="">Logs and tracing: not just for production</a>.</blockquote><p class="">This investigation continues for some time, due to several roadblocks that slowed us.</p><p class="">Firstly, while we have Heroku capturing logs from each Dyno stdout/stderr into Corologix, when a Go app panics it will print the stacktrace of every goroutine in the application (for those unfamiliar, a goroutine is similar to a thread).</p><p class="">As our application runs with ~300 goroutines, printing the stacktrace of each goroutine caused Heroku to buffer and drop these log lines, preventing us from seeing the original crashing stacktrace.</p><p class="">Then due to Go immediately exiting upon any unhandled panic, even the parts of our app with panic handlers – such as our Sentry error reporting – were terminated, meaning we couldn’t inspect the crash in our exception tracker.</p><p class="">Finally, Heroku’s rate limiting of the logs was even impacting the logs that came just prior to a crash, so we couldn’t even see when we began to process the problematic event.</p><p class="">After 5m of trying and failing to retrieve usable logs, we change strategy.</p><p class="">Knowing that this crash happened about 15m away from the last deploy, we know it’s unlikely to be a frequently used event that’s causing this, and rolling back is therefore unlikely to fix our issue, since we don’t yet know what introduced the bug.</p><p class="">We conclude the original crash was probably the first time we processed this bad event, and start looking at all subscriptions with unacked messages from around the time we first went down. For all subscriptions that match this criteria, we purge the subscription, removing all the events from the queue. This is safe as we treat async events as best-effort, meaning dropping events will only degrade service, which is preferable if it means we come back up.</p><p class="">We also turn off several non-critical parts of the app that trigger async work, prioritising core incident functionality over features such as incident nudges (”It’s been a while since you last updated the incident, would you like to make an update?”).</p><p class="">At 16:12, after purging several subscriptions that matched our criteria, the app stopped crashing and remained stable, without any further intervention.</p><h2 id="regroup" class=""><a class="heading-link" href="#regroup"><span class="icon icon-link"></span></a>Regroup</h2><p class="">The subscription responsible was used for generating a welcome message whenever people first enter an incident channel. From looking at the code, we determined that for a very small subset of our customers we&#x27;d do this in a way which would cause the code to fail.</p><p class="">This meant it was unlikely we’d produce another message like this for a while, and we knew which subscription to purge if we did. So with urgency reduced, we called a pause so we could regroup.</p><p class="">For us, that meant:</p><ul><li>Have the incident lead run a stand-up with all responders, checking what work remains and confirming everyone has what they need (information, support, clarity on role but also &quot;something to eat and drink&quot;).</li><li>Confirm our collective understanding of the incident, summarise it into a written update.</li><li>Produce an update we could share with our customers, focused on the impact they’d feel from the incident.</li><li>Decide next steps and allocate work.</li></ul><p class="">Technically, we had a couple of things we wanted to try putting in place to quickly improve our position before closing this incident.</p><h2 id="mitigation-1-ensuring-we-handle-message-panics" class=""><a class="heading-link" href="#mitigation-1-ensuring-we-handle-message-panics"><span class="icon icon-link"></span></a>Mitigation 1: Ensuring we handle message panics</h2><p class="">One rule you learn quickly when running Go apps is to catch your panics: this means ensuring any goroutine (thread) that might panic – segfault, or crashing via an explicit <code>panic</code> – has added a <code>recover()</code>.</p><p class="">As a basic example:</p><pre class="refractor language-go"><code class="language-go"><span class="token keyword">func</span> <span class="token function">main</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
<span class="token keyword">defer</span> <span class="token keyword">func</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
<span class="token keyword">if</span> err <span class="token operator">:=</span> <span class="token function">recover</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span> err <span class="token operator">!=</span> <span class="token boolean">nil</span> <span class="token punctuation">{</span>
fmt<span class="token punctuation">.</span><span class="token function">Println</span><span class="token punctuation">(</span><span class="token string">&quot;caught error: &quot;</span><span class="token punctuation">,</span> err<span class="token punctuation">)</span>
<span class="token punctuation">}</span>
<span class="token punctuation">}</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
<span class="token comment">// Try to access a blank user pointer.</span>
<span class="token keyword">var</span> blankPointer <span class="token operator">*</span>User
fmt<span class="token punctuation">.</span><span class="token function">Println</span><span class="token punctuation">(</span>blankPointer<span class="token punctuation">.</span>Name<span class="token punctuation">)</span> <span class="token comment">// uh-oh, segfault!</span>
<span class="token punctuation">}</span>
</code></pre><p class="">This main function tries printing the <code>Name</code> field on <code>blankPointer</code>, which is a zero-value pointer to a User struct. That will fail because we’ll try dereferencing a zero-value, causing a segmentation fault where the kernel refuses to allow our application to read memory outside its virtual address range, which does not contain 0.</p><p class="">Thankfully, <code>main</code> has deferred a function that calls <code>recover()</code>, which is a special function that will catch and turn any unhandled panics into a runtime error.</p><p class="">This means that instead of our app crashing with a panic, it will print the error and continue:</p><pre class="refractor language-go"><code class="language-go">$ <span class="token keyword">go</span> run main<span class="token punctuation">.</span><span class="token keyword">go</span>
caught <span class="token builtin">error</span><span class="token punctuation">:</span> runtime <span class="token builtin">error</span><span class="token punctuation">:</span> invalid memory address or <span class="token boolean">nil</span> pointer dereference
</code></pre><p class="">We add these panic handlers all over our app, so we shouldn’t have hit this issue. But clearly we did, given our app has just crashed repeatedly.</p><p class="">So what gives? Well, turns out there’s an edge case that is easy to miss, and it existed in the generic event consumer for our async workers:</p><pre class="refractor language-go"><code class="language-go"><span class="token keyword">func</span> <span class="token function">receive</span><span class="token punctuation">(</span>ctx context<span class="token punctuation">.</span>Context<span class="token punctuation">,</span> client <span class="token operator">*</span>pubsub<span class="token punctuation">.</span>Client<span class="token punctuation">,</span> subscriptionID <span class="token builtin">string</span><span class="token punctuation">)</span> <span class="token builtin">error</span> <span class="token punctuation">{</span>
<span class="token keyword">defer</span> <span class="token keyword">func</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
<span class="token comment">// recover</span>
<span class="token punctuation">}</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
sub <span class="token operator">:=</span> client<span class="token punctuation">.</span><span class="token function">Subscription</span><span class="token punctuation">(</span>subscriptionID<span class="token punctuation">)</span>
<span class="token keyword">return</span> sub<span class="token punctuation">.</span><span class="token function">Receive</span><span class="token punctuation">(</span>ctx<span class="token punctuation">,</span> <span class="token keyword">func</span><span class="token punctuation">(</span>ctx context<span class="token punctuation">.</span>Context<span class="token punctuation">,</span> msg <span class="token operator">*</span>pubsub<span class="token punctuation">.</span>Message<span class="token punctuation">)</span> <span class="token punctuation">{</span>
consumer<span class="token punctuation">.</span><span class="token function">handle</span><span class="token punctuation">(</span>ctx<span class="token punctuation">,</span> msg<span class="token punctuation">)</span> <span class="token comment">// call the event consumer</span>
<span class="token punctuation">}</span><span class="token punctuation">)</span>
<span class="token punctuation">}</span>
</code></pre><p class="">The generic receiver did apply a recover, but we’d missed that the implementation of <code>sub.Receive</code> – which is in the Google Cloud Pub/Sub client – starts several new goroutines to handle the anonymous function we provide to the <code>Receive</code> method.</p><p class="">In Go, goroutines have no meaningful links back to their parent. If your child goroutine panics, it won’t benefit from the parent goroutine having applied a recover: it’s an unhandled panic, and your whole app would crash.</p><p class="">So our first mitigation was to apply a recover to this handler, which we deployed at 16:48.</p><p class="">Since this is a common piece of code used by all our event handlers, this mitigates the issue which caused this incident but it also removes this entire problem class for all other current and future subscriptions too.</p><h2 id="mitigation-2-separate-our-app-by-work-type" class=""><a class="heading-link" href="#mitigation-2-separate-our-app-by-work-type"><span class="icon icon-link"></span></a>Mitigation 2: Separate our app by work type</h2><p class="">It’s considered best practice to run different work types – say web requests, async workers, and long-running cron jobs – separately from one another.</p><p class="">That’s because:</p><ol><li>Different work often demands different resources, like CPU or memory. Running things separately allows right-sizing the machine that hosts that class of work.</li><li>Physically separating work improves reliability, as it prevents competition for shared resources like CPU, avoiding one thread consuming most of your processing ability and everything else slow/stopping.</li><li>If one type of work behaves badly, such as panicking, it can’t adversely impact everything else.</li></ol><p class="">In our case (3) is what we care about, as had we separated async work from web traffic then even if workers had died, we would continue to serve requests for Slack interactions and our web dashboard. It’s likely no one would even notice the workers were down, provided we manually restarted them quickly.</p><p class="">We had begun the process of splitting these work types before this incident, and the app already had a clear logical separation between web, async workers and crons. Not only was the code separate, but each component used separate database connection pools – preventing expensive workers from stealing all web database connections – and the entrypoint to the app allowed toggling components on-or-off via command line flags.</p><p class="">The reason we hadn&#x27;t completed the split was that we&#x27;d already got most of the value from logically separating, and prefer running as simple an app as possible. Adding another dyno would also increase our CI times slightly, so we&#x27;d been waiting for a clear push before making this change: obviously an incident like this is exactly that trigger.</p><p class="">So after confirming we were happy with the approach, we implemented the physical split by adding a new ‘worker’ Dyno tier to Heroku and tweaking each entrypoint:</p><p class=""></p><figure class=""><span class="relative overflow-hidden block align-top"><img alt="" data-zoom-src="https://cdn.sanity.io/images/oqy5aexb/production/6d63615fb4a2893b1f17860dea5dfbf8884e75f8-1726x566.png?q=85&amp;auto=format" loading="lazy" width="1726" height="566" decoding="async" data-nimg="1" class="!m-0 w-full h-full object-cover" style="color:transparent" src="https://cdn.sanity.io/images/oqy5aexb/production/6d63615fb4a2893b1f17860dea5dfbf8884e75f8-1726x566.png?w=2000&amp;q=80&amp;fit=max&amp;auto=format"/></span></figure><p class="">By 17:24, this change was running in production.</p><h2 id="conclusion" class=""><a class="heading-link" href="#conclusion"><span class="icon icon-link"></span></a>Conclusion</h2><p class="">While you never want to hit a serious incident, we’re serious about making the most of the opportunity when you do.</p><p class="">In this instance, we took advantage of the opportunity to learn and socialise knowledge internally about:</p><ul><li>Heroku Dyno restart policies, and how to manually reboot if they go wrong, for those outside of the primary on-call team who hadn’t yet encountered this.</li><li>An easy-to-miss edge-case for our static tooling where a third-party library could run our code in an uncaught goroutine.</li><li>Best-practice incident comms for our current organisation, given we’ve grown significantly since our last major incident.</li></ul><p class="">And after implementing incident follow-ups, we:</p><ul><li>Physically split our application by work-type, which will increase reliability and performance.</li><li>Improved our event system by ensuring we catch panics.</li></ul><p class="">In addition, aside from eveything else we learned, we&#x27;re happy that:</p><ul><li>Outside of Heroku monitoring gaps, we found existing dashboards around Pub/Sub subscription and worker performance to be extremely valuable.</li><li>Limited downtime to 13m, when it could have been much longer.</li><li>Kept our customers in the loop as the incident progressed, which we care a lot about.</li></ul></div><div class="flex justify-between"><div class="flex gap-3"></div><div class="flex gap-6"><button title="Bluesky" class="relative"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" class="shrink-0 size-6 text-content-primary transition-colors hover:text-alarmalade-500"><path fill="currentColor" d="M407.8 294.7c-3.3-.4-6.7-.8-10-1.3 3.4.4 6.7.9 10 1.3M288 227.1c-26.1-50.7-97.1-145.2-163.1-191.8C61.6-9.4 37.5-1.7 21.6 5.5 3.3 13.8 0 41.9 0 58.4S9.1 194 15 213.9c19.5 65.7 89.1 87.9 153.2 80.7 3.3-.5 6.6-.9 10-1.4-3.3.5-6.6 1-10 1.4-93.9 14-177.3 48.2-67.9 169.9C220.6 589.1 265.1 437.8 288 361.1c22.9 76.7 49.2 222.5 185.6 103.4 102.4-103.4 28.1-156-65.8-169.9-3.3-.4-6.7-.8-10-1.3 3.4.4 6.7.9 10 1.3 64.1 7.1 133.6-15.1 153.2-80.7C566.9 194 576 75 576 58.4s-3.3-44.7-21.6-52.9c-15.8-7.1-40-14.9-103.2 29.8C385.1 81.9 314.1 176.4 288 227.1"></path></svg></button><button title="LinkedIn" class="relative"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="shrink-0 size-6 text-content-primary transition-colors hover:text-alarmalade-500"><path fill="currentColor" d="M0 1.43C0 .648.664 0 1.484 0h17.032C19.336 0 20 .64 20 1.43v17.14c0 .782-.664 1.43-1.484 1.43H1.484C.664 20 0 19.36 0 18.57z"></path><path fill="#fff" d="M6.078 16.734v-9H3.086v9zM4.586 6.5c1.04 0 1.688-.687 1.688-1.555-.016-.883-.649-1.554-1.664-1.554-1.024 0-1.696.672-1.696 1.562 0 .86.649 1.555 1.656 1.555h.016zm3.149 10.234h2.992V11.72c0-.274.023-.547.101-.735.211-.53.703-1.093 1.532-1.093 1.085 0 1.515.828 1.515 2.03v4.821h3v-5.156c0-2.766-1.476-4.063-3.437-4.063a2.98 2.98 0 0 0-2.72 1.524h.017V7.734h-3c.039.844 0 9 0 9"></path></svg></button><button title="X" class="relative"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 20 20" class="shrink-0 size-6 text-content-primary transition-colors hover:text-alarmalade-500"><path fill="currentColor" d="M15.751.729h3.067l-6.7 7.766L20 19.062h-6.172l-4.833-6.409-5.531 6.41H.394l7.167-8.307L0 .729h6.328l4.37 5.858zm-1.076 16.472h1.7L5.404 2.493H3.58z"></path></svg></button><button title="Reddit" class="relative"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512" class="shrink-0 size-6 text-content-primary transition-colors hover:text-alarmalade-500"><path fill="currentColor" d="M0 256C0 114.6 114.6 0 256 0s256 114.6 256 256-114.6 256-256 256H37.1c-13.7 0-20.5-16.5-10.9-26.2L75 437C28.7 390.7 0 326.7 0 256m349.6-102.4c23.6 0 42.7-19.1 42.7-42.7s-19.1-42.7-42.7-42.7c-20.6 0-37.8 14.6-41.8 34-34.5 3.7-61.4 33-61.4 68.4v.2c-37.5 1.6-71.8 12.3-99 29.1-10.1-7.8-22.8-12.5-36.5-12.5-33 0-59.8 26.8-59.8 59.8 0 24 14.1 44.6 34.4 54.1 2 69.4 77.6 125.2 170.6 125.2s168.7-55.9 170.6-125.3c20.2-9.6 34.1-30.2 34.1-54 0-33-26.8-59.8-59.8-59.8-13.7 0-26.3 4.6-36.4 12.4-27.4-17-62.1-27.7-100-29.1v-.2c0-25.4 18.9-46.5 43.4-49.9 4.4 18.8 21.3 32.8 41.5 32.8zm-172.5 93.3c16.7 0 29.5 17.6 28.5 39.3s-13.5 29.6-30.3 29.6-31.4-8.8-30.4-30.5S160.3 247 177 247zm190.1 38.3c1 21.7-13.7 30.5-30.4 30.5s-29.3-7.9-30.3-29.6 11.8-39.3 28.5-39.3 31.2 16.6 32.1 38.3zm-48.1 56.7c-10.3 24.6-34.6 41.9-63 41.9s-52.7-17.3-63-41.9c-1.2-2.9.8-6.2 3.9-6.5 18.4-1.9 38.3-2.9 59.1-2.9s40.7 1 59.1 2.9c3.1.3 5.1 3.6 3.9 6.5"></path></svg></button></div></div><div class="p-8 bg-white border border-stroke-primary rounded-lg flex flex-col gap-4 flame-background lg:flex-row"><div class="flex-1 flex flex-col gap-4 justify-between flex-grow items-center lg:flex-row"><div class="flex gap-4 items-center"><span class="relative overflow-hidden block align-top w-25 h-25 rounded-full object-cover ring-1 ring-inset ring-black/10"><img alt="Picture of Lawrence Jones" loading="lazy" width="250" height="250" decoding="async" data-nimg="1" class="!m-0 w-full h-full object-cover" style="color:transparent" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fe7b906adbcc6184c09d6abda07181ee946f50ab5-800x800.png%3Fw%3D250%26h%3D250%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=256&amp;q=75 1x, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fe7b906adbcc6184c09d6abda07181ee946f50ab5-800x800.png%3Fw%3D250%26h%3D250%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=640&amp;q=75 2x" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Fe7b906adbcc6184c09d6abda07181ee946f50ab5-800x800.png%3Fw%3D250%26h%3D250%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=640&amp;q=75"/></span><div class="text-pretty"><div class="text-content-primary text-body-emphasis">Lawrence Jones</div><div class="text-alarmalade-600 text-body-xs">Product Engineer</div></div></div><a class="relative inline-flex gap-2 items-center justify-center rounded-full focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:opacity-60 group/button cursor-pointer transition-[color,background-color,border-color,opacity,transform] border border-stroke-primary hover:border-stroke-hover text-content-primary dark:text-white text-body-emphasis h-[41px] px-4 py-2 max-sm:w-full" href="/blog/author/lawrence">View more<!-- --> <span class="hidden sm:inline">from <!-- -->Lawrence</span><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="none" class="transition ease-in-out inline-block size-5 group-hover/button:translate-x-1"><path d="M12.8 8.59965H3.00002C2.66882 8.59965 2.40002 8.33085 2.40002 7.99965C2.40002 7.66845 2.66882 7.39965 3.00002 7.39965H12.8C13.1312 7.39965 13.4 7.66845 13.4 7.99965C13.4 8.33085 13.1312 8.59965 12.8 8.59965Z" fill="currentColor"></path><path d="M9.60002 11.9996C9.53122 11.9996 9.46082 11.9876 9.39202 11.9628C9.08082 11.8476 8.92242 11.5028 9.03682 11.1924C9.66242 9.49645 10.7328 8.53005 11.5816 8.00045C10.7328 7.47085 9.66322 6.50445 9.03682 4.80845C8.92162 4.49725 9.08082 4.15245 9.39202 4.03805C9.70322 3.92125 10.048 4.08285 10.1624 4.39325C11.0656 6.84205 13.0656 7.39805 13.1496 7.42045C13.4144 7.49085 13.5992 7.73085 13.5976 8.00365C13.596 8.27725 13.412 8.51565 13.1464 8.58285C13.0648 8.60445 11.0656 9.16045 10.1624 11.6092C10.0728 11.8516 9.84402 12.0012 9.59922 12.0012L9.60002 11.9996Z" fill="currentColor"></path></svg></a></div></div></div></section><section class="flex flex-col gap-6"><div class="flex flex-col md:flex-row justify-between gap-4"><h2 class="text-h3">See related articles</h2><a class="relative gap-2 items-center justify-center rounded-full focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:opacity-60 group/button cursor-pointer transition-[color,background-color,border-color,opacity,transform] bg-surface-secondary hover:bg-surface-tertiary active:bg-surface-quaternary text-content-primary text-body-emphasis h-[41px] px-4 py-2 hidden md:block" href="/blog">View all<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="none" class="transition ease-in-out inline-block size-5 group-hover/button:translate-x-1"><path d="M12.8 8.59965H3.00002C2.66882 8.59965 2.40002 8.33085 2.40002 7.99965C2.40002 7.66845 2.66882 7.39965 3.00002 7.39965H12.8C13.1312 7.39965 13.4 7.66845 13.4 7.99965C13.4 8.33085 13.1312 8.59965 12.8 8.59965Z" fill="currentColor"></path><path d="M9.60002 11.9996C9.53122 11.9996 9.46082 11.9876 9.39202 11.9628C9.08082 11.8476 8.92242 11.5028 9.03682 11.1924C9.66242 9.49645 10.7328 8.53005 11.5816 8.00045C10.7328 7.47085 9.66322 6.50445 9.03682 4.80845C8.92162 4.49725 9.08082 4.15245 9.39202 4.03805C9.70322 3.92125 10.048 4.08285 10.1624 4.39325C11.0656 6.84205 13.0656 7.39805 13.1496 7.42045C13.4144 7.49085 13.5992 7.73085 13.5976 8.00365C13.596 8.27725 13.412 8.51565 13.1464 8.58285C13.0648 8.60445 11.0656 9.16045 10.1624 11.6092C10.0728 11.8516 9.84402 12.0012 9.59922 12.0012L9.60002 11.9996Z" fill="currentColor"></path></svg></a></div><div class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-8 md:gap-4"><a id="59bb806c-b2d4-4e43-ac1a-ce346a36d81d" class="group flex flex-col" href="/blog/interning-at-incident-io-rate-limiting-resiliently"><div class="bg-surface-tertiary rounded-lg mb-3 md:mb-6 p-0 relative aspect-video"><span class="block align-top w-full overflow-hidden transition-all ease-in-out duration-300 mx-auto absolute inset-0 group-hover:w-[calc(100%-16px)] group-hover:py-2 [&amp;_img]:transition-all [&amp;_img]:rounded-lg group-hover:[&amp;_img]:rounded"><img alt="Interning at incident.io: rate limiting, resiliently" loading="lazy" width="2400" height="1285" decoding="async" data-nimg="1" class="!m-0 w-full h-full object-cover" style="color:transparent" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F4208ab8bacb5b3cde27465ad8764fa7cd5e30895-2400x1285.jpg%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=3840&amp;q=75 1x" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F4208ab8bacb5b3cde27465ad8764fa7cd5e30895-2400x1285.jpg%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=3840&amp;q=75"/></span></div><div class="max-w-[546px] grow flex flex-col gap-2"><h3 class="text-h4 text-pretty">Interning at incident.io: rate limiting, resiliently</h3><p class="text-body-sm line-clamp-2">Our rate limiter depends on Valkey. If Valkey goes down we fail open and stop limiting which isn&#x27;t good enough for our platform. As an intern, I built per-pod in-memory top-k buffers so we keep rate limiting even with the backing store gone.</p><div class="flex flex-wrap items-center gap-x-4 gap-y-1"><div class="flex items-center gap-2 flex-wrap"><span class="flex items-center gap-2"><div class="size-[22px] rounded-full overflow-hidden shrink-0"><span class="relative overflow-hidden block align-top"><img alt="Anthony Oparaocha" loading="lazy" width="100" height="100" decoding="async" data-nimg="1" class="!m-0 w-full h-full object-cover" style="color:transparent" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F1fa50382fde41588c782aac0b86b8a96df63e21b-4672x7008.jpg%3Frect%3D650%2C357%2C3335%2C3335%26w%3D100%26h%3D100%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=128&amp;q=75 1x, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F1fa50382fde41588c782aac0b86b8a96df63e21b-4672x7008.jpg%3Frect%3D650%2C357%2C3335%2C3335%26w%3D100%26h%3D100%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=256&amp;q=75 2x" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F1fa50382fde41588c782aac0b86b8a96df63e21b-4672x7008.jpg%3Frect%3D650%2C357%2C3335%2C3335%26w%3D100%26h%3D100%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=256&amp;q=75"/></span></div><span class="text-body-sm-emphasis leading-[1.1]">Anthony Oparaocha</span></span></div><div class="text-body-sm text-content-secondary">September 2, 2026</div></div></div></a><a id="f3ee3736-294b-4170-a14c-04abb899cc18" class="group flex flex-col" href="/blog/we-turned-off-pub-sub-and-nobody-noticed"><div class="bg-surface-tertiary rounded-lg mb-3 md:mb-6 p-0 relative aspect-video"><span class="block align-top w-full overflow-hidden transition-all ease-in-out duration-300 mx-auto absolute inset-0 group-hover:w-[calc(100%-16px)] group-hover:py-2 [&amp;_img]:transition-all [&amp;_img]:rounded-lg group-hover:[&amp;_img]:rounded"><img alt="We turned off Pub/Sub and nobody noticed" loading="lazy" width="2400" height="1285" decoding="async" data-nimg="1" class="!m-0 w-full h-full object-cover" style="color:transparent" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F02b1cabbed39c62eed6a6372f44b9b11b416c4f4-2400x1285.jpg%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=3840&amp;q=75 1x" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F02b1cabbed39c62eed6a6372f44b9b11b416c4f4-2400x1285.jpg%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=3840&amp;q=75"/></span></div><div class="max-w-[546px] grow flex flex-col gap-2"><h3 class="text-h4 text-pretty">We turned off Pub/Sub and nobody noticed</h3><p class="text-body-sm line-clamp-2">Our entire event-driven platform ran through a single message broker, which made it a single point of failure. So we added a second one.
This is the story of building an event load balancer, the queuing theory behind it, and the final chaos test where we turned off Pub/Sub in production and nobody noticed.</p><div class="flex flex-wrap items-center gap-x-4 gap-y-1"><div class="flex items-center gap-2 flex-wrap"><span class="flex items-center gap-2"><div class="size-[22px] rounded-full overflow-hidden shrink-0"><span class="relative overflow-hidden block align-top"><img alt="Patrick Hamann" loading="lazy" width="100" height="100" decoding="async" data-nimg="1" class="!m-0 w-full h-full object-cover" style="color:transparent" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F8dbc717422b0562befb8ccc5c9399be570787e9f-1369x1369.png%3Fw%3D100%26h%3D100%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=128&amp;q=75 1x, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F8dbc717422b0562befb8ccc5c9399be570787e9f-1369x1369.png%3Fw%3D100%26h%3D100%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=256&amp;q=75 2x" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F8dbc717422b0562befb8ccc5c9399be570787e9f-1369x1369.png%3Fw%3D100%26h%3D100%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=256&amp;q=75"/></span></div><span class="text-body-sm-emphasis leading-[1.1]">Patrick Hamann</span><div class="text-body-sm-emphasis text-content-tertiary grow">+</div></span><span class="flex items-center gap-2"><div class="size-[22px] rounded-full overflow-hidden shrink-0"><span class="relative overflow-hidden block align-top"><img alt="Mike Fisher" loading="lazy" width="100" height="100" decoding="async" data-nimg="1" class="!m-0 w-full h-full object-cover" style="color:transparent" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F03534500cc455a88eb475ec447557953e2db55c7-512x512.jpg%3Fw%3D100%26h%3D100%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=128&amp;q=75 1x, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F03534500cc455a88eb475ec447557953e2db55c7-512x512.jpg%3Fw%3D100%26h%3D100%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=256&amp;q=75 2x" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F03534500cc455a88eb475ec447557953e2db55c7-512x512.jpg%3Fw%3D100%26h%3D100%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=256&amp;q=75"/></span></div><span class="text-body-sm-emphasis leading-[1.1]">Mike Fisher</span></span></div><div class="text-body-sm text-content-secondary">August 11, 2026</div></div></div></a><a id="c2f628cf-31ed-4847-9b2e-35efdafa9f58" class="group flex flex-col" href="/blog/dont-add-a-read-replica-until-youve-read-this"><div class="bg-surface-tertiary rounded-lg mb-3 md:mb-6 p-0 relative aspect-video"><span class="block align-top w-full overflow-hidden transition-all ease-in-out duration-300 mx-auto absolute inset-0 group-hover:w-[calc(100%-16px)] group-hover:py-2 [&amp;_img]:transition-all [&amp;_img]:rounded-lg group-hover:[&amp;_img]:rounded"><img alt="Don&#x27;t add a read replica until you&#x27;ve read this" loading="lazy" width="2400" height="1285" decoding="async" data-nimg="1" class="!m-0 w-full h-full object-cover" style="color:transparent" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F77427469f4daa51f1385e3d50b13a277e1fe26b9-2400x1285.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=3840&amp;q=75 1x" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F77427469f4daa51f1385e3d50b13a277e1fe26b9-2400x1285.png%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=3840&amp;q=75"/></span></div><div class="max-w-[546px] grow flex flex-col gap-2"><h3 class="text-h4 text-pretty">Don&#x27;t add a read replica until you&#x27;ve read this</h3><p class="text-body-sm line-clamp-2">Our learnings from implementing a product-wide read replica migrations, including some useful patterns for routing queries to replica and primary</p><div class="flex flex-wrap items-center gap-x-4 gap-y-1"><div class="flex items-center gap-2 flex-wrap"><span class="flex items-center gap-2"><div class="size-[22px] rounded-full overflow-hidden shrink-0"><span class="relative overflow-hidden block align-top"><img alt="Johanna Larsson" loading="lazy" width="100" height="100" decoding="async" data-nimg="1" class="!m-0 w-full h-full object-cover" style="color:transparent" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Ff709cea28f7dc0c0a60e669c17c07edf7f72143f-2435x3328.jpg%3Frect%3D0%2C447%2C2435%2C2435%26w%3D100%26h%3D100%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=128&amp;q=75 1x, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Ff709cea28f7dc0c0a60e669c17c07edf7f72143f-2435x3328.jpg%3Frect%3D0%2C447%2C2435%2C2435%26w%3D100%26h%3D100%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=256&amp;q=75 2x" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2Ff709cea28f7dc0c0a60e669c17c07edf7f72143f-2435x3328.jpg%3Frect%3D0%2C447%2C2435%2C2435%26w%3D100%26h%3D100%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=256&amp;q=75"/></span></div><span class="text-body-sm-emphasis leading-[1.1]">Johanna Larsson</span></span></div><div class="text-body-sm text-content-secondary">July 21, 2026</div></div></div></a></div><a class="relative inline-flex gap-2 items-center justify-center rounded-full focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:opacity-60 group/button cursor-pointer transition-[color,background-color,border-color,opacity,transform] bg-surface-secondary hover:bg-surface-tertiary active:bg-surface-quaternary text-content-primary text-body-emphasis h-[41px] px-4 py-2 md:hidden" href="/blog">View all<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="none" class="transition ease-in-out inline-block size-5 group-hover/button:translate-x-1"><path d="M12.8 8.59965H3.00002C2.66882 8.59965 2.40002 8.33085 2.40002 7.99965C2.40002 7.66845 2.66882 7.39965 3.00002 7.39965H12.8C13.1312 7.39965 13.4 7.66845 13.4 7.99965C13.4 8.33085 13.1312 8.59965 12.8 8.59965Z" fill="currentColor"></path><path d="M9.60002 11.9996C9.53122 11.9996 9.46082 11.9876 9.39202 11.9628C9.08082 11.8476 8.92242 11.5028 9.03682 11.1924C9.66242 9.49645 10.7328 8.53005 11.5816 8.00045C10.7328 7.47085 9.66322 6.50445 9.03682 4.80845C8.92162 4.49725 9.08082 4.15245 9.39202 4.03805C9.70322 3.92125 10.048 4.08285 10.1624 4.39325C11.0656 6.84205 13.0656 7.39805 13.1496 7.42045C13.4144 7.49085 13.5992 7.73085 13.5976 8.00365C13.596 8.27725 13.412 8.51565 13.1464 8.58285C13.0648 8.60445 11.0656 9.16045 10.1624 11.6092C10.0728 11.8516 9.84402 12.0012 9.59922 12.0012L9.60002 11.9996Z" fill="currentColor"></path></svg></a></section><hr class="h-px w-full border-stroke-primary dark:border-stroke-invert max-w-[1192px] mx-auto"/><section class=""><div class="mx-auto flex flex-col gap-3 items-center text-center mb-10"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 246 151" class="w-[96px] sm:w-[246px]"><g fill="#E4D9C8" clip-path="url(#server-racks_svg__a)"><path d="M64.585 70.707c0-2.056-.082-3.562.024-5.051.103-1.436-.6-1.805-1.861-1.941a65 65 0 0 1-7.613-1.077c-3.706-.858-5.644-2.95-5.676-6.699-.058-6.805.239-13.615.449-20.422.058-1.908.982-3.522 2.753-4.233 2.345-.941 4.79-2.126 7.238-2.24 18.487-.86 36.98-1.648 55.481-2.117 11.857-.3 23.731-.09 35.595.054a57 57 0 0 1 9.753 1.07c4.438.833 6.258 2.847 6.862 7.275.842 6.162 1.19 12.354.082 18.52-1.066 5.937-3.995 8.578-10.004 9.166-1.849.18-3.713.194-5.57.283-2.641.128-2.717.233-2.193 3.277.811.066 1.674.146 2.538.205 1.855.127 3.719.177 5.566.373 7.408.788 9.507 2.64 10.162 10.009.495 5.104.571 10.24.228 15.357-.499 6.596-3.609 9.422-10.143 10.138-3.423.375-6.867.55-10.527.834l.252 4.773c.904.089 1.755.16 2.605.257 3.426.404 6.88.656 10.267 1.276 3.231.59 5.469 2.576 6.018 5.942 1.088 6.676 1.473 13.391.219 20.086-1.048 5.591-4.045 8.412-9.78 8.719-7.337.394-14.688.47-22.035.565-20.19.259-40.38.518-60.57.638-5.216.031-10.436-.425-15.652-.715a22.4 22.4 0 0 1-3.668-.575c-4.525-1.023-6.462-3.047-6.5-7.718-.053-6.454.272-12.911.478-19.365.068-2.122 1.05-3.869 3.03-4.626 2.92-1.121 5.95-1.953 9.486-3.083l.563-4.066c-2.14-.331-4.057-.545-5.937-.936-4.235-.88-6.258-2.95-6.376-7.261-.179-6.54-.028-13.09.036-19.637.02-2.116.97-3.888 2.945-4.679a57 57 0 0 1 7.496-2.328c1.145-.282 2.418-.048 3.98-.048m44.498 26.886v-.361c3.006 0 6.014.073 9.018-.013q18.975-.54 37.945-1.17c3.85-.129 4.805-.875 4.939-4.7.17-4.846-.075-9.708-.159-14.562-.03-1.658-.782-2.612-2.57-2.684-1.676-.068-3.343-.377-5.019-.44-20.993-.78-41.932.383-62.865 1.777q-16.68 1.11-33.377 1.992c-1.774.096-2.039 1-1.95 2.406.344 5.476.702 10.946 1.01 16.421.107 1.895 1.158 2.659 2.918 2.678 7.868.085 15.744.4 23.606.176 8.842-.255 17.67-.993 26.504-1.52m-3.848 17.2-.011.352c-6.008.319-12.016.691-18.03.941q-15.382.639-30.772 1.117c-1.843.053-2.273.911-2.207 2.482.212 5.126.382 10.256.603 15.382.13 3.064.388 3.552 3.583 3.706 7.596.368 15.206.751 22.809.65 15.132-.2 30.259-.767 45.39-1.095 9.469-.205 18.943-.217 28.412-.386 3.548-.064 4.589-.837 4.793-4.285.292-4.929.178-9.883.21-14.826.012-1.662-.765-2.628-2.522-2.769-1.939-.155-3.866-.581-5.804-.613-15.485-.257-30.969-.447-46.454-.656m.038-81.623-.009.412c-5.127.256-10.253.547-15.382.76-10.875.452-21.75.908-32.63 1.244-1.821.056-2.617.56-2.533 2.416.229 5.038.39 10.079.598 15.118.143 3.469.346 3.918 3.783 4.084 7.51.362 15.035.718 22.548.616 14.691-.2 29.376-.763 44.066-1.083 9.912-.217 19.828-.248 29.74-.416 3.677-.062 4.643-.783 4.884-4.472.305-4.664.291-9.357.207-14.036-.053-2.83-.368-3.002-3.272-3.453a31 31 0 0 0-4.219-.46q-23.889-.397-47.781-.73m34.918 70.226-71.115 2.452.699 4.303 70.15-2.374zM71.847 70.004l69.494-3.832q.182-.503.26-1.032-.016-.9-.156-1.79l-70.394.867c.192 1.914.334 3.403.5 4.891q.106.462.294.896z"></path><path d="M86.987 89.137c-4.978.235-11.428.551-17.88.834-1.144.05-2.415.254-3.406-.151-.893-.366-1.921-1.399-2.067-2.275-.115-.69.869-2.023 1.64-2.298a17.7 17.7 0 0 1 5.194-.918q19.227-.75 38.46-1.35a21.3 21.3 0 0 1 3.698.25c1.445.208 2.544 1.01 2.609 2.555s-1.004 2.499-2.399 2.76c-1.821.35-3.669.54-5.524.567-6.282.067-12.565.026-20.325.026M153.008 88.199a3.404 3.404 0 0 1-3.32-3.416c.074-2.035 1.458-3.543 3.218-3.509a3.72 3.72 0 0 1 3.509 3.606 3.373 3.373 0 0 1-3.407 3.319M141.657 88.559a3.414 3.414 0 0 1-3.403-3.347c-.019-1.875 1.177-2.952 3.26-2.938 2.153.013 3.504 1.246 3.436 3.136a3.315 3.315 0 0 1-3.293 3.148M128.268 82.723a3.27 3.27 0 0 1 3.137 2.968 2.86 2.86 0 0 1-.907 2.118 2.86 2.86 0 0 1-2.176.758 2.95 2.95 0 0 1-2.801-2.94 2.74 2.74 0 0 1 .749-2.043 2.75 2.75 0 0 1 1.998-.861M87.676 130.021h-19.35q-1.591.05-3.179-.069c-1.452-.156-2.566-.86-2.632-2.465-.06-1.486.95-2.276 2.254-2.539a26 26 0 0 1 4.727-.599q19.49-.294 38.979-.466c1.14-.011 2.427-.064 3.382.422.962.489 2.188 1.612 2.206 2.481.019.879-1.14 2.071-2.079 2.6-.926.522-2.222.51-3.357.517-6.984.041-13.967.02-20.948.02zM151.787 123.145a3.65 3.65 0 0 1 3.615 3.514 3.375 3.375 0 0 1-3.115 3.372 3.38 3.38 0 0 1-3.551-3.2c-.06-1.981 1.311-3.637 3.051-3.686M137.269 126.814c-.026-1.853 1.215-2.999 3.224-2.977 2.127.025 3.489 1.271 3.435 3.146a3.283 3.283 0 0 1-3.26 3.148 3.4 3.4 0 0 1-3.399-3.317M127.531 129.834a2.91 2.91 0 0 1-3.004-2.959 2.8 2.8 0 0 1 .192-1.087 2.802 2.802 0 0 1 2.587-1.784 3.305 3.305 0 0 1 3.067 3.025 2.77 2.77 0 0 1-.828 2.003 2.776 2.776 0 0 1-2.014.802"></path></g><g fill="#E4D9C8" clip-path="url(#server-racks_svg__b)"><path d="M155.218 63.963c1.78-1.028 3.043-1.853 4.386-2.505 1.294-.629 1.263-1.422.75-2.582a65 65 0 0 1-2.874-7.132c-1.109-3.639-.267-6.362 2.964-8.265 5.864-3.453 11.911-6.6 17.91-9.822 1.681-.903 3.541-.91 5.042.268 1.988 1.56 4.237 3.085 5.56 5.148 9.987 15.58 19.917 31.202 29.573 46.99 6.188 10.118 11.944 20.506 17.752 30.853a57 57 0 0 1 3.948 8.981c1.498 4.26.664 6.843-2.868 9.58-4.916 3.81-10.104 7.208-15.998 9.331-5.674 2.045-9.426.829-12.94-4.081-1.08-1.51-2.024-3.118-3.031-4.682-1.431-2.223-1.56-2.236-3.933-.261.348.736.71 1.523 1.091 2.301.818 1.67 1.706 3.309 2.461 5.007 3.02 6.809 2.465 9.553-3.588 13.804a94 94 0 0 1-13.186 7.877c-5.961 2.865-9.964 1.586-13.851-3.715-2.036-2.777-3.91-5.672-5.985-8.7-1.337.868-2.558 1.66-4.008 2.605.375.827.739 1.599 1.079 2.385 1.363 3.168 2.872 6.286 4.029 9.529 1.105 3.092.504 6.024-2.137 8.182-5.237 4.281-10.86 7.972-17.285 10.233-5.366 1.888-9.308.703-12.442-4.11-4.009-6.157-7.751-12.485-11.506-18.801-10.32-17.354-20.639-34.71-30.838-52.136-2.634-4.501-4.85-9.25-7.206-13.912a22.3 22.3 0 0 1-1.336-3.465c-1.377-4.43-.592-7.119 3.434-9.488 5.562-3.272 11.317-6.22 17.01-9.268 1.871-1.002 3.875-1.025 5.521.312 2.43 1.968 4.666 4.176 7.413 6.673l3.802-1.546c-.784-2.019-1.556-3.786-2.158-5.61-1.355-4.107-.574-6.894 3.1-9.152 5.575-3.424 11.324-6.57 17.024-9.787 1.843-1.04 3.853-1.105 5.525.211a57 57 0 0 1 5.764 5.328c.817.85 1.251 2.07 2.032 3.422m-1.036 51.979.313-.181c1.503 2.604 2.944 5.245 4.52 7.804a7868 7868 0 0 0 19.986 32.277c2.036 3.269 3.161 3.723 6.54 1.927 4.282-2.276 8.37-4.919 12.531-7.419 1.421-.855 1.872-1.983 1.04-3.568-.779-1.485-1.345-3.084-2.129-4.567-9.82-18.571-21.297-36.122-32.971-53.554a2405 2405 0 0 1-18.413-27.91c-.971-1.488-1.886-1.265-3.059-.485-4.571 3.036-9.129 6.081-13.716 9.086-1.588 1.04-1.724 2.331-.86 3.865 3.86 6.857 7.525 13.835 11.65 20.532 4.641 7.53 9.694 14.806 14.568 22.193m-16.819 5.268-.311.166c-3.28-5.043-6.606-10.06-9.83-15.144a2962 2962 0 0 1-16.352-26.09c-.968-1.57-1.926-1.513-3.254-.671-4.333 2.747-8.69 5.459-13.02 8.213-2.588 1.646-2.882 2.112-1.418 4.956 3.48 6.762 6.953 13.545 10.842 20.078 7.739 13.005 15.794 25.822 23.643 38.762 4.913 8.098 9.659 16.296 14.541 24.412 1.829 3.041 3.019 3.556 6.107 2.009 4.415-2.212 8.648-4.788 12.945-7.231 1.445-.821 1.893-1.977 1.137-3.569-.835-1.757-1.43-3.639-2.371-5.333-7.52-13.539-15.098-27.044-22.659-40.559zm70.707-40.779-.362.198c-2.785-4.312-5.6-8.605-8.349-12.941-5.829-9.193-11.661-18.383-17.392-27.636-.96-1.55-1.794-1.987-3.359-.986-4.249 2.717-8.534 5.377-12.794 8.077-2.932 1.858-3.22 2.259-1.645 5.318 3.442 6.685 6.896 13.38 10.741 19.835 7.518 12.623 15.349 25.06 22.971 37.621 5.143 8.476 10.128 17.047 15.23 25.548 1.892 3.153 3 3.629 6.315 1.993 4.191-2.068 8.249-4.427 12.259-6.839 2.424-1.461 2.415-1.819 1.354-4.56a31 31 0 0 0-1.711-3.883 18099 18099 0 0 0-23.258-41.745m-43.359 65.353-37.682-60.362-3.376 2.757 37.131 59.565zM159.457 69.9l38.066 58.268q.526-.094 1.023-.291a13 13 0 0 0 1.472-1.03l-35.947-60.53c-1.561 1.124-2.78 1.99-3.986 2.879q-.347.322-.628.703"></path><path d="M150.457 92.578c-2.692-4.194-6.191-9.622-9.662-15.068-.615-.966-1.428-1.964-1.573-3.025-.129-.957.251-2.363.937-2.927.541-.445 2.187-.26 2.811.271a17.7 17.7 0 0 1 3.391 4.039A4854 4854 0 0 1 166.76 108.5a21.3 21.3 0 0 1 1.634 3.328c.542 1.355.397 2.707-.909 3.536s-2.666.381-3.591-.696a32 32 0 0 1-3.252-4.501c-3.199-5.407-6.305-10.869-10.185-17.59M184.28 149.285a3.4 3.4 0 0 1 1.298-4.584c1.799-.952 3.797-.508 4.648 1.033a3.717 3.717 0 0 1-1.369 4.842 3.37 3.37 0 0 1-4.577-1.291M178.294 139.634a3.417 3.417 0 0 1 1.196-4.62c1.614-.953 3.145-.457 4.175 1.355 1.065 1.871.673 3.657-.999 4.543a3.31 3.31 0 0 1-4.372-1.278M176.653 125.122a3.27 3.27 0 0 1-1.002 4.201 2.85 2.85 0 0 1-2.288.273 2.86 2.86 0 0 1-1.744-1.506 2.95 2.95 0 0 1 1.145-3.896 2.76 2.76 0 0 1 3.15.098c.3.224.552.506.739.83M186.448 73.394l-9.544-16.53c-.532-.921-1.097-1.819-1.591-2.756-.721-1.368-.93-2.801.481-3.807 1.576-1.122 2.881-.221 3.899 1.011a32 32 0 0 1 2.794 3.866 6645 6645 0 0 1 18.701 31.457c.85 1.44 1.751 2.946 2.124 4.54a4.12 4.12 0 0 1-.831 3.278c-1.052 1.098-2.52.558-3.536-.47a14 14 0 0 1-2.086-2.716c-3.482-5.951-6.912-11.932-10.36-17.903zM218.587 129.118a3.33 3.33 0 0 1 1.119-4.637c1.726-1.001 3.831-.586 4.695.927a3.713 3.713 0 0 1-1.16 4.722 3.33 3.33 0 0 1-3.748-.054 3.3 3.3 0 0 1-.906-.958M212.767 119.233a3.34 3.34 0 0 1-.409-2.529 3.34 3.34 0 0 1 1.498-2.077c1.601-.985 3.196-.519 4.197 1.232 1.063 1.859.697 3.653-.93 4.565a3.235 3.235 0 0 1-4.356-1.191M206.259 107.429a2.89 2.89 0 0 1 1.472-3.95c1.466-.588 2.78-.347 3.604 1.132a3.167 3.167 0 0 1-1.339 4.267 2.836 2.836 0 0 1-3.737-1.449"></path></g><path fill="#E4D9C8" d="M153.387 37.918c-.642.435-1.268.644-2.099.39-2.077-.636-4.188-1.16-6.288-1.721-.362-.097-.733-.152-1.24-.254.017.398.018.702.046 1.003q.151.977.186 1.965c-.034 1.143-.887 1.686-1.964 1.291a6.6 6.6 0 0 1-1.13-.59 66 66 0 0 1-9.969-7.292c-.604-.544-1.137-1.133-1.196-1.976.684-1.128 1.694-1.179 2.767-.912 1.184.295 2.352.657 3.669 1.028-.194-1.042-.355-2.004-.558-2.957-.122-.57-.289-1.165.244-1.576.593-.458 1.26-.33 1.87.029 1.309.766 2.609 1.547 3.91 2.326 2.6 1.56 5.208 3.105 7.792 4.69q1.576.952 3.028 2.082c.41.324.634.883.941 1.335zM168.257 2.815c.757.17 1.338.483 1.721 1.263.958 1.95 2.021 3.847 3.05 5.762.177.33.392.638.664 1.078.278-.285.499-.494.699-.72q.608-.78 1.305-1.48c.858-.759 1.837-.509 2.286.547q.232.598.343 1.228a66 66 0 0 1 1.509 12.259c.016.812-.048 1.604-.622 2.224-1.29.273-2.019-.428-2.559-1.393-.596-1.065-1.132-2.164-1.763-3.378-.626.854-1.217 1.63-1.773 2.43-.332.479-.651 1.009-1.315.902-.741-.12-1.103-.694-1.26-1.383-.337-1.48-.658-2.961-.98-4.444-.643-2.962-1.302-5.921-1.915-8.89a31 31 0 0 1-.555-3.632c-.045-.521.21-1.067.329-1.6z"></path><defs><clipPath id="server-racks_svg__a"><path fill="#fff" d="M48.644 83.479c0-9.395 7.616-17.012 17.01-17.012h102.878v79.386H48.644z"></path></clipPath><clipPath id="server-racks_svg__b"><path fill="#fff" d="m185.411 28.395 59.945 103.826-32.271 18.632L153.14 47.026z"></path></clipPath></defs></svg><h2 class="text-h3 md:text-h2 text-balance max-w-[588px]">So good, you’ll break things on purpose</h2><p class="text-body md:text-body-lg max-w-[588px]">Ready for modern incident management? Book a call with one of our experts today.</p></div><div class="flex gap-4 flex-col lg:flex-row lg:gap-4"><div class="basis-1/2"><img alt="Signup image" loading="lazy" width="1176" height="641" decoding="async" data-nimg="1" class="w-full h-full object-cover rounded-lg" style="color:transparent" srcSet="/_next/image?url=%2F_next%2Fstatic%2Fimmutable%2Fmedia%2Fsignup-image.1f5nktjcty_re.png&amp;w=1200&amp;q=75 1x, /_next/image?url=%2F_next%2Fstatic%2Fimmutable%2Fmedia%2Fsignup-image.1f5nktjcty_re.png&amp;w=3840&amp;q=75 2x" src="/_next/image?url=%2F_next%2Fstatic%2Fimmutable%2Fmedia%2Fsignup-image.1f5nktjcty_re.png&amp;w=3840&amp;q=75"/></div><div class="basis-1/2 flex flex-col gap-6 bg-surface-secondary rounded-lg p-6"><div class="grow"><h3 class="text-body-emphasis mb-2">We’d love to talk to you about</h3><ul class="flex flex-col gap-2 text-body-serif grow"><li class="flex gap-1"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="21" viewBox="0 0 20 21" fill="none" class="mt-0.5"><path d="M9.14598 14.4365C10.6142 11.6377 12.4682 9.39312 14.6552 7.7673C15.0469 7.47672 15.1281 6.92263 14.8375 6.53205C14.547 6.1403 13.994 6.0603 13.6035 6.34971C11.5753 7.85671 9.81654 9.83428 8.35895 12.2401C7.8025 11.3648 7.19429 10.5978 6.5202 9.92016C6.1755 9.57311 5.61552 9.57312 5.27201 9.91663C4.92614 10.2601 4.92496 10.8201 5.26848 11.1648C6.14727 12.0483 6.90371 13.1165 7.58016 14.4306C7.73191 14.7247 8.03896 14.9129 8.36483 14.9094C8.69306 14.9094 8.99422 14.727 9.14598 14.4365Z" fill="#F25533"></path></svg>All-in-one incident management</li><li class="flex gap-1"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="21" viewBox="0 0 20 21" fill="none" class="mt-0.5"><path d="M9.14598 14.4365C10.6142 11.6377 12.4682 9.39312 14.6552 7.7673C15.0469 7.47672 15.1281 6.92263 14.8375 6.53205C14.547 6.1403 13.994 6.0603 13.6035 6.34971C11.5753 7.85671 9.81654 9.83428 8.35895 12.2401C7.8025 11.3648 7.19429 10.5978 6.5202 9.92016C6.1755 9.57311 5.61552 9.57312 5.27201 9.91663C4.92614 10.2601 4.92496 10.8201 5.26848 11.1648C6.14727 12.0483 6.90371 13.1165 7.58016 14.4306C7.73191 14.7247 8.03896 14.9129 8.36483 14.9094C8.69306 14.9094 8.99422 14.727 9.14598 14.4365Z" fill="#F25533"></path></svg>Our unmatched speed of deployment</li><li class="flex gap-1"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="21" viewBox="0 0 20 21" fill="none" class="mt-0.5"><path d="M9.14598 14.4365C10.6142 11.6377 12.4682 9.39312 14.6552 7.7673C15.0469 7.47672 15.1281 6.92263 14.8375 6.53205C14.547 6.1403 13.994 6.0603 13.6035 6.34971C11.5753 7.85671 9.81654 9.83428 8.35895 12.2401C7.8025 11.3648 7.19429 10.5978 6.5202 9.92016C6.1755 9.57311 5.61552 9.57312 5.27201 9.91663C4.92614 10.2601 4.92496 10.8201 5.26848 11.1648C6.14727 12.0483 6.90371 13.1165 7.58016 14.4306C7.73191 14.7247 8.03896 14.9129 8.36483 14.9094C8.69306 14.9094 8.99422 14.727 9.14598 14.4365Z" fill="#F25533"></path></svg>Why we’re loved by users and easily adopted</li><li class="flex gap-1"><svg xmlns="http://www.w3.org/2000/svg" width="20" height="21" viewBox="0 0 20 21" fill="none" class="mt-0.5"><path d="M9.14598 14.4365C10.6142 11.6377 12.4682 9.39312 14.6552 7.7673C15.0469 7.47672 15.1281 6.92263 14.8375 6.53205C14.547 6.1403 13.994 6.0603 13.6035 6.34971C11.5753 7.85671 9.81654 9.83428 8.35895 12.2401C7.8025 11.3648 7.19429 10.5978 6.5202 9.92016C6.1755 9.57311 5.61552 9.57312 5.27201 9.91663C4.92614 10.2601 4.92496 10.8201 5.26848 11.1648C6.14727 12.0483 6.90371 13.1165 7.58016 14.4306C7.73191 14.7247 8.03896 14.9129 8.36483 14.9094C8.69306 14.9094 8.99422 14.727 9.14598 14.4365Z" fill="#F25533"></path></svg>How we work for the whole organization</li></ul></div><div class="flex justify-between items-center"><div class="w-full flex gap-4 flex-col sm:flex-row whitespace-nowrap"><a class="relative inline-flex gap-2 items-center justify-center rounded-full focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:opacity-60 group/button cursor-pointer transition-[color,background-color,border-color,opacity,transform] bg-alarmalade-500 text-content-invert hover:bg-alarmalade-600 active:bg-alarmalade-700 text-body-emphasis h-[49px] px-5 py-3" id="homepage-signup-card-request-a-demo" href="/demo">Get a demo</a><a class="relative inline-flex gap-2 items-center justify-center rounded-full focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-offset-2 disabled:opacity-60 group/button cursor-pointer transition-[color,background-color,border-color,opacity,transform] border border-stroke-primary hover:border-stroke-hover text-content-primary dark:text-white text-body-emphasis h-[49px] px-5 py-3" id="homepage-signup-card-sign-up" href="/free-trial">Get started for free</a></div><div class="-space-x-1 hidden md:flex"><span class="relative overflow-hidden block align-top size-10 rounded-full border-2 border-surface-secondary"><img alt="Rory Malcolm" loading="lazy" width="3024" height="4032" decoding="async" data-nimg="1" class="!m-0 w-full h-full object-cover" style="color:transparent" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F1e715111dcf04d620a13db5d6b70d9a19b74033f-3024x4032.jpg%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=3840&amp;q=75 1x" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F1e715111dcf04d620a13db5d6b70d9a19b74033f-3024x4032.jpg%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=3840&amp;q=75"/></span><span class="relative overflow-hidden block align-top size-10 rounded-full border-2 border-surface-secondary"><img alt="Lucy Jennings" loading="lazy" width="800" height="800" decoding="async" data-nimg="1" class="!m-0 w-full h-full object-cover" style="color:transparent" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F4a76d0ae3b8eef5d16f525ac375b3570563346a3-800x800.jpg%3Frect%3D263%2C175%2C351%2C341%26w%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=828&amp;q=75 1x, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F4a76d0ae3b8eef5d16f525ac375b3570563346a3-800x800.jpg%3Frect%3D263%2C175%2C351%2C341%26w%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=1920&amp;q=75 2x" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F4a76d0ae3b8eef5d16f525ac375b3570563346a3-800x800.jpg%3Frect%3D263%2C175%2C351%2C341%26w%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=1920&amp;q=75"/></span><span class="relative overflow-hidden block align-top size-10 rounded-full border-2 border-surface-secondary"><img alt="Eryn Carman" loading="lazy" width="943" height="943" decoding="async" data-nimg="1" class="!m-0 w-full h-full object-cover" style="color:transparent" srcSet="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F13c3df986e33da4b270a0ede8c6859982f9f87f9-943x943.jpg%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=1080&amp;q=75 1x, /_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F13c3df986e33da4b270a0ede8c6859982f9f87f9-943x943.jpg%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=1920&amp;q=75 2x" src="/_next/image?url=https%3A%2F%2Fcdn.sanity.io%2Fimages%2Foqy5aexb%2Fproduction%2F13c3df986e33da4b270a0ede8c6859982f9f87f9-943x943.jpg%3Fw%3D2000%26q%3D80%26fit%3Dmax%26auto%3Dformat&amp;w=1920&amp;q=75"/></span></div></div></div></div></section><footer class="grid grid-cols-2 gap-x-4 gap-y-8 md:grid-cols-4 pb-6 lg:pb-14 lg:pt-20 text-content-primary"><div class="flex flex-col justify-between gap-3 order-last md:order-first"><a class="grow transition hover:opacity-90" href="/"><span class="sr-only">incident.io</span><span class="flex size-9 items-center justify-center rounded-lg bg-alarmalade-500"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 32 32" class="shrink-0 size-8 text-white"><path fill="currentColor" d="M14.957 4.18a.728.728 0 0 1 1.199-.248c.907 1.004 2.024 3.196 2.39 5.682.234 1.593.035 2.266.57 2.266.573 0 .948-.87 1.03-1.77.053-.575.623-.768 1.017-.575 1.892.927 3.823 5.34 4.29 7.506.146.68.242 1.381.242 2.105 0 4.413-3.111 8.113-7.307 9.13v-3.512h-4.624v3.518c-4.21-1.006-7.334-4.714-7.334-9.136 0-2.55 1.077-4.573 2.725-6.558 1.366-1.645 4.74-6.003 5.802-8.408m1.159 12.272a.3.3 0 0 0-.177.075c-.054.052-.068.113-.097.233-.346 1.457-2.458 1.92-2.459 4.06 0 1.403 1.2 2.54 2.679 2.54 1.48 0 2.68-1.136 2.68-2.54 0-2.236-1.647-3.835-2.468-4.32-.055-.033-.083-.05-.158-.049"></path></svg></span></a><div class="flex flex-col gap-3"><div class="flex items-center gap-3"><a href="https://bsky.app/profile/incident.io" class="transition-colors text-content-primary hover:text-alarmalade-500 dark:hover:text-content-invert" target="_blank" rel="noopener noreferrer"><span class="sr-only">BlueSky</span><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512" class="size-4 md:size-5" aria-hidden="true"><path fill="currentColor" d="M407.8 294.7c-3.3-.4-6.7-.8-10-1.3 3.4.4 6.7.9 10 1.3M288 227.1c-26.1-50.7-97.1-145.2-163.1-191.8C61.6-9.4 37.5-1.7 21.6 5.5 3.3 13.8 0 41.9 0 58.4S9.1 194 15 213.9c19.5 65.7 89.1 87.9 153.2 80.7 3.3-.5 6.6-.9 10-1.4-3.3.5-6.6 1-10 1.4-93.9 14-177.3 48.2-67.9 169.9C220.6 589.1 265.1 437.8 288 361.1c22.9 76.7 49.2 222.5 185.6 103.4 102.4-103.4 28.1-156-65.8-169.9-3.3-.4-6.7-.8-10-1.3 3.4.4 6.7.9 10 1.3 64.1 7.1 133.6-15.1 153.2-80.7C566.9 194 576 75 576 58.4s-3.3-44.7-21.6-52.9c-15.8-7.1-40-14.9-103.2 29.8C385.1 81.9 314.1 176.4 288 227.1"></path></svg></a><a href="https://www.linkedin.com/company/incident-io/" class="transition-colors text-content-primary hover:text-alarmalade-500 dark:hover:text-content-invert" target="_blank" rel="noopener noreferrer"><span class="sr-only">LinkedIn</span><svg viewBox="0 0 256 256" class="size-4 md:size-5" aria-hidden="true"><g fill="none"><path fill="currentColor" d="M0 18.3C0 8.3 8.5 0 19 0h218c10.5 0 19 8.2 19 18.3v219.4c0 10-8.5 18.3-19 18.3H19c-10.5 0-19-8.2-19-18.3z"></path><path fill="#fff" d="M77.8 214.2V99H39.5v115.2zm-19.1-131c13.3 0 21.6-8.8 21.6-19.9C80.1 52 72 43.4 59 43.4c-13.1 0-21.7 8.6-21.7 20 0 11 8.3 19.9 21.2 19.9h.2zm40.3 131h38.3V150c0-3.5.3-7 1.3-9.4 2.7-6.8 9-14 19.6-14 13.9 0 19.4 10.6 19.4 26v61.7H216v-66c0-35.4-18.9-52-44-52a38.2 38.2 0 0 0-34.8 19.5h.2V99H99c.5 10.8 0 115.2 0 115.2"></path></g></svg></a><a href="https://x.com/incident_io" class="transition-colors text-content-primary hover:text-alarmalade-500 dark:hover:text-content-invert" target="_blank" rel="noopener noreferrer"><span class="sr-only">X</span><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 25" class="size-4 md:size-5" aria-hidden="true"><path fill="currentColor" d="M18.901 1.875h3.68l-8.04 9.319L24 23.875h-7.406l-5.8-7.691-6.638 7.691H.474l8.6-9.968L0 1.875h7.594l5.243 7.03zM17.61 21.641h2.039L6.486 3.991H4.298z"></path></svg></a><a href="https://www.reddit.com/user/incident-io/" class="transition-colors text-content-primary hover:text-alarmalade-500 dark:hover:text-content-invert" target="_blank" rel="noopener noreferrer"><span class="sr-only">Reddit</span><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512" class="size-4 md:size-5" aria-hidden="true"><path fill="currentColor" d="M0 256C0 114.6 114.6 0 256 0s256 114.6 256 256-114.6 256-256 256H37.1c-13.7 0-20.5-16.5-10.9-26.2L75 437C28.7 390.7 0 326.7 0 256m349.6-102.4c23.6 0 42.7-19.1 42.7-42.7s-19.1-42.7-42.7-42.7c-20.6 0-37.8 14.6-41.8 34-34.5 3.7-61.4 33-61.4 68.4v.2c-37.5 1.6-71.8 12.3-99 29.1-10.1-7.8-22.8-12.5-36.5-12.5-33 0-59.8 26.8-59.8 59.8 0 24 14.1 44.6 34.4 54.1 2 69.4 77.6 125.2 170.6 125.2s168.7-55.9 170.6-125.3c20.2-9.6 34.1-30.2 34.1-54 0-33-26.8-59.8-59.8-59.8-13.7 0-26.3 4.6-36.4 12.4-27.4-17-62.1-27.7-100-29.1v-.2c0-25.4 18.9-46.5 43.4-49.9 4.4 18.8 21.3 32.8 41.5 32.8zm-172.5 93.3c16.7 0 29.5 17.6 28.5 39.3s-13.5 29.6-30.3 29.6-31.4-8.8-30.4-30.5S160.3 247 177 247zm190.1 38.3c1 21.7-13.7 30.5-30.4 30.5s-29.3-7.9-30.3-29.6 11.8-39.3 28.5-39.3 31.2 16.6 32.1 38.3zm-48.1 56.7c-10.3 24.6-34.6 41.9-63 41.9s-52.7-17.3-63-41.9c-1.2-2.9.8-6.2 3.9-6.5 18.4-1.9 38.3-2.9 59.1-2.9s40.7 1 59.1 2.9c3.1.3 5.1 3.6 3.9 6.5"></path></svg></a><a href="https://www.facebook.com/incident.io/" class="transition-colors text-content-primary hover:text-alarmalade-500 dark:hover:text-content-invert" target="_blank" rel="noopener noreferrer"><span class="sr-only">Facebook</span><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1365.33 1365.33" class="size-4 md:size-5" aria-hidden="true"><path fill="currentColor" d="M1365.333 682.667C1365.333 305.64 1059.693 0 682.667 0S0 305.64 0 682.667c0 340.738 249.641 623.16 576 674.373V880H402.667V682.667H576v-150.4c0-171.094 101.917-265.6 257.853-265.6 74.69 0 152.814 13.333 152.814 13.333v168h-86.083c-84.804 0-111.25 52.623-111.25 106.61v128.057h189.333L948.4 880H789.333v477.04c326.359-51.213 576-333.635 576-674.373"></path><path fill="transparent" d="m948.4 880 30.267-197.333H789.333V554.609C789.333 500.623 815.78 448 900.584 448h86.083V280s-78.124-13.333-152.814-13.333c-155.936 0-257.853 94.506-257.853 265.6v150.4H402.667V880H576v477.04a688 688 0 0 0 106.667 8.293c36.288 0 71.91-2.84 106.666-8.293V880z"></path></svg></a><a href="https://www.youtube.com/@incident-io" class="transition-colors text-content-primary hover:text-alarmalade-500 dark:hover:text-content-invert" target="_blank" rel="noopener noreferrer"><span class="sr-only">Youtube</span><svg class="size-4 md:size-5" xmlns="http://www.w3.org/2000/svg" viewBox="14.9 64 546.2 384"><path d="M549.7 124.1c-6.3-23.7-24.8-42.3-48.3-48.6C458.8 64 288 64 288 64S117.2 64 74.6 75.5c-23.5 6.3-42 24.9-48.3 48.6-11.4 42.9-11.4 132.3-11.4 132.3s0 89.4 11.4 132.3c6.3 23.7 24.8 41.5 48.3 47.8C117.2 448 288 448 288 448s170.8 0 213.4-11.5c23.5-6.3 42-24.2 48.3-47.8 11.4-42.9 11.4-132.3 11.4-132.3s0-89.4-11.4-132.3zm-317.5 213.5V175.2l142.7 81.2-142.7 81.2z" fill="currentColor"></path></svg></a><a href="/community" class="transition-colors text-content-primary hover:text-alarmalade-500 dark:hover:text-content-invert" target="_blank" rel="noopener noreferrer"><span class="sr-only">Slack Community</span><svg fill="currentColor" aria-hidden="true" viewBox="0 0 128 128" class="size-4 md:size-5"><path d="M27.255 80.719c0 7.33-5.978 13.317-13.309 13.317S.63 88.049.63 80.719s5.987-13.317 13.317-13.317h13.309zm6.709 0c0-7.33 5.987-13.317 13.317-13.317s13.317 5.986 13.317 13.317v33.335c0 7.33-5.986 13.317-13.317 13.317-7.33 0-13.317-5.987-13.317-13.317zM47.281 27.255c-7.33 0-13.317-5.978-13.317-13.309S39.951.63 47.281.63s13.317 5.987 13.317 13.317v13.309zm0 6.709c7.33 0 13.317 5.987 13.317 13.317s-5.986 13.317-13.317 13.317H13.946C6.616 60.598.63 54.612.63 47.281c0-7.33 5.987-13.317 13.317-13.317zM100.745 47.281c0-7.33 5.978-13.317 13.309-13.317s13.317 5.987 13.317 13.317-5.987 13.317-13.317 13.317h-13.309zm-6.709 0c0 7.33-5.987 13.317-13.317 13.317s-13.317-5.986-13.317-13.317V13.946C67.402 6.616 73.388.63 80.719.63c7.33 0 13.317 5.987 13.317 13.317zM80.719 100.745c7.33 0 13.317 5.978 13.317 13.309s-5.987 13.317-13.317 13.317-13.317-5.987-13.317-13.317v-13.309zm0-6.709c-7.33 0-13.317-5.987-13.317-13.317s5.986-13.317 13.317-13.317h33.335c7.33 0 13.317 5.986 13.317 13.317 0 7.33-5.987 13.317-13.317 13.317zm0 0"></path></svg></a></div><div class="text-body-xs dark:text-content-invert"><p>© <!-- -->2026<!-- --> Pineapple Technology Ltd.</p><p>All rights reserved.</p></div></div></div><div class="flex flex-col gap-y-3 md:pl-4 md:border-l md:border-alarmalade-200" style="order:1"><div class="text-body-serif-xs md:text-body-serif-sm text-alarmalade-500 dark:text-content-invert">Product</div><a class="block transition text-body-xs-emphasis md:text-body-sm hover:text-alarmalade-500 dark:text-content-invert" href="/on-call">On-call</a><a class="block transition text-body-xs-emphasis md:text-body-sm hover:text-alarmalade-500 dark:text-content-invert" href="/investigations">Investigations</a><a class="block transition text-body-xs-emphasis md:text-body-sm hover:text-alarmalade-500 dark:text-content-invert" href="/respond">Incident Response</a><a class="block transition text-body-xs-emphasis md:text-body-sm hover:text-alarmalade-500 dark:text-content-invert" href="/status-pages">Status Pages</a><a class="block transition text-body-xs-emphasis md:text-body-sm hover:text-alarmalade-500 dark:text-content-invert" href="/changelog">Changelog</a></div><div class="flex flex-col gap-y-3 md:pl-4 md:border-l md:border-alarmalade-200" style="order:2"><div class="text-body-serif-xs md:text-body-serif-sm text-alarmalade-500 dark:text-content-invert">Learn</div><a class="block transition text-body-xs-emphasis md:text-body-sm hover:text-alarmalade-500 dark:text-content-invert" href="/blog">Blog</a><a class="block transition text-body-xs-emphasis md:text-body-sm hover:text-alarmalade-500 dark:text-content-invert" href="/customers">Customer Stories</a><a class="block transition text-body-xs-emphasis md:text-body-sm hover:text-alarmalade-500 dark:text-content-invert" href="https://docs.incident.io">Documentation</a><a class="block transition text-body-xs-emphasis md:text-body-sm hover:text-alarmalade-500 dark:text-content-invert" href="/alternatives">Alternatives</a><a class="block transition text-body-xs-emphasis md:text-body-sm hover:text-alarmalade-500 dark:text-content-invert" href="/community">Community</a></div><div class="flex flex-col gap-y-3 md:pl-4 md:border-l md:border-alarmalade-200" style="order:3"><div class="text-body-serif-xs md:text-body-serif-sm text-alarmalade-500 dark:text-content-invert">Company</div><a class="block transition text-body-xs-emphasis md:text-body-sm hover:text-alarmalade-500 dark:text-content-invert" href="/legal">Legal</a><a class="block transition text-body-xs-emphasis md:text-body-sm hover:text-alarmalade-500 dark:text-content-invert" href="#">Privacy Choices</a><a class="block transition text-body-xs-emphasis md:text-body-sm hover:text-alarmalade-500 dark:text-content-invert" href="/security">Security and Compliance</a><a class="block transition text-body-xs-emphasis md:text-body-sm hover:text-alarmalade-500 dark:text-content-invert" href="/careers">Careers</a><a class="block transition text-body-xs-emphasis md:text-body-sm hover:text-alarmalade-500 dark:text-content-invert" href="https://status.incident.io">Status</a></div></footer></div><!--$--><!--/$--><noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-W4ZF4L4" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript><script src="/_next/static/immutable/chunks/1f8tzf816i32j.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n2:I[339756,[\"/_next/static/immutable/chunks/0esgme6slw2os.js\"],\"default\"]\n3:I[837457,[\"/_next/static/immutable/chunks/0esgme6slw2os.js\"],\"default\"]\n12:I[168027,[],\"default\",1]\n:HL[\"/_next/static/immutable/chunks/1wus27ic5fsbj.css\",\"style\"]\n:HL[\"/_next/static/immutable/chunks/0ok2xz0peykn3.css\",\"style\"]\n:HL[\"/_next/static/immutable/chunks/3_pcvzakx73f_.css\",\"style\"]\n:HL[\"/_next/static/immutable/media/stkbureau_serif_book-s.p.0_js8gy1zifgl.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n10:X\n0:{\"P\":null,\"c\":[\"\",\"blog\",\"intermittent-downtime\"],\"q\":\"\",\"i\":false,\"f\":[[[\"\",{\"children\":[\"(website)\",{\"children\":[\"blog\",{\"children\":[[\"slug\",\"intermittent-downtime\",\"oc\",[\"author\"]],{\"children\":[\"__PAGE__\",{},\"$undefined\",\"$undefined\",4256]},\"$undefined\",\"$undefined\",4192]},\"$undefined\",\"$undefined\",4160]},\"$undefined\",\"$undefined\",4144]},\"$undefined\",\"$undefined\",4176],[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L2\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L3\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":404}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/1wus27ic5fsbj.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"1\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/0ok2xz0peykn3.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"2\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/3_pcvzakx73f_.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-1\",{\"src\":\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-2\",{\"src\":\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-3\",{\"src\":\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-4\",{\"src\":\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-5\",{\"src\":\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-6\",{\"src\":\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-7\",{\"src\":\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-8\",{\"src\":\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-9\",{\"src\":\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-10\",{\"src\":\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-11\",{\"src\":\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-12\",{\"src\":\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-13\",{\"src\":\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\",\"async\":true,\"nonce\":\"$undefined\"}]],\"$L4\"]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L2\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L3\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L2\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L3\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[\"$L5\",[[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/immutable/chunks/39pfb5kk8her1.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-1\",{\"src\":\"/_next/static/immutable/chunks/3qz9rwp_7jbnb.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-2\",{\"src\":\"/_next/static/immutable/chunks/1my_9vnng6gxj.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-3\",{\"src\":\"/_next/static/immutable/chunks/040yti440pn1b.js\",\"async\":true,\"nonce\":\"$undefined\"}],\"$L6\",\"$L7\",\"$L8\",\"$L9\",\"$La\",\"$Lb\",\"$Lc\",\"$Ld\",\"$Le\"],\"$Lf\"]}],{},null,false,null]},null,false,\"$10\"]},null,false,\"$10\"]},null,false,null]},null,false,\"$10\"],\"$L11\",false]],\"m\":\"$undefined\",\"G\":[\"$12\",[]],\"S\":true,\"h\":null,\"r\":\"$undefined\",\"s\":\"$undefined\",\"a\":\"$undefined\",\"l\":\"$undefined\",\"p\":\"$undefined\",\"d\":\"$undefined\"}\n13:I[897367,[\"/_next/static/immutable/chunks/0esgme6slw2os.js\"],\"OutletBoundary\"]\n14:\"$Sreact.suspense\"\n16:I[897367,[\"/_next/static/immutable/chunks/0esgme6slw2os.js\"],\"ViewportBoundary\"]\n18:I[897367,[\"/_next/static/immutable/chunks/0esgme6slw2os.js\"],\"MetadataBoundary\"]\n6:[\"$\",\"script\",\"script-4\",{\"src\":\"/_next/static/immutable/chunks/0v4j5g1ppvtg_.js\",\"async\":true,\"nonce\":\"$undefined\"}]\n7:[\"$\",\"script\",\"script-5\",{\"src\":\"/_next/static/immutable/chunks/3buml8s4fdaii.js\",\"async\":true,\"nonce\":\"$undefined\"}]\n8:[\"$\",\"script\",\"script-6\",{\"src\":\"/_next/static/immutable/chunks/2_frah5pspzdw.js\",\"async\":true,\"nonce\":\"$undefined\"}]\n9:[\"$\",\"script\",\"script-7\",{\"src\":\"/_next/static/immutable/chunks/1mcb8cvy5_n5b.js\",\"async\":true,\"nonce\":\"$undefined\"}]\na:[\"$\",\"script\",\"script-8\",{\"src\":\"/_next/static/immutable/chunks/132szer1wwfxa.js\",\"async\":true,\"nonce\":\"$undefined\"}]\nb:[\"$\",\"script\",\"script-9\",{\"src\":\"/_next/static/immutable/chunks/2jjhnej_crwty.js\",\"async\":true,\"nonce\":\"$undefined\"}]\nc:[\"$\",\"script\",\"script-10\",{\"src\":\"/_next/static/immutable/chunks/19t-07qzgwjic.js\",\"async\":true,\"nonce\":\"$undefined\"}]\nd:[\"$\",\"script\",\"script-11\",{\"src\":\"/_next/static/immutable/chunks/1wpznbhwql3dd.js\",\"async\":true,\"nonce\":\"$undefined\"}]\ne:[\"$\",\"script\",\"script-12\",{\"src\":\"/_next/static/immutable/chunks/20izjjpsu0ag8.js\",\"async\":true,\"nonce\":\"$undefined\"}]\nf:[\"$\",\"$L13\",null,{\"children\":[\"$\",\"$14\",null,{\"name\":\"Next.MetadataOutlet\",\"children\":\"$@15\"}]}]\n11:[\"$\",\"$1\",\"h\",{\"children\":[null,[\"$\",\"$L16\",null,{\"children\":\"$L17\"}],[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$L18\",null,{\"children\":[\"$\",\"$14\",null,{\"name\":\"Next.Metadata\",\"children\":\"$L19\"}]}]}],[\"$\",\"meta\",null,{\"name\":\"next-size-adjust\",\"content\":\"\"}]]}]\n10:C\n17:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}]]\n"])</script><script>self.__next_f.push([1,"1a:I[456989,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\"],\"CookieConsentProvider\"]\n1b:I[926127,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\"],\"SignupFlowModalProvider\"]\n1c:I[378756,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\"],\"HubspotFormModalHandler\"]\n1d:I[12985,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\"],\"NuqsAdapter\"]\n4:[\"$\",\"html\",null,{\"lang\":\"en\",\"data-scroll-behavior\":\"smooth\",\"className\":\"geist_mono_54069fc3-module__PcBEcq__variable kalam_48bfd9a2-module__JSifUG__variable stkbureausans_1e54e897-module__zxjcxa__variable stkbureauserif_2525014a-module__zkXytW__variable stkbureauserifdeferred_742a2474-module__ukGqna__variable pixelify_sans_81b7d097-module__1Rm0Lq__variable scroll-smooth\",\"children\":[[\"$\",\"head\",null,{\"children\":[[\"$\",\"meta\",null,{\"httpEquiv\":\"x-ua-compatible\",\"content\":\"ie=edge\"}],[\"$\",\"link\",null,{\"rel\":\"alternate\",\"type\":\"application/rss+xml\",\"title\":\"Blog\",\"href\":\"https://incident.io/blog.xml\"}],[\"$\",\"link\",null,{\"rel\":\"alternate\",\"type\":\"application/rss+xml\",\"title\":\"Changelog\",\"href\":\"https://incident.io/changelog.xml\"}],[\"$\",\"link\",null,{\"rel\":\"apple-touch-icon\",\"sizes\":\"180x180\",\"href\":\"/apple-touch-icon.png\"}],[\"$\",\"link\",null,{\"rel\":\"icon\",\"type\":\"image/png\",\"sizes\":\"32x32\",\"href\":\"/favicon-32x32.png\"}],[\"$\",\"link\",null,{\"rel\":\"icon\",\"type\":\"image/png\",\"sizes\":\"16x16\",\"href\":\"/favicon-16x16.png\"}]]}],[\"$\",\"body\",null,{\"children\":[[[\"$\",\"script\",\"organization-jsonld-Organization\",{\"type\":\"application/ld+json\",\"id\":\"organization-jsonld-Organization\",\"data-testid\":\"$undefined\",\"dangerouslySetInnerHTML\":{\"__html\":\"{\\\"@context\\\":\\\"https://schema.org\\\",\\\"@type\\\":\\\"Organization\\\",\\\"name\\\":\\\"incident.io\\\",\\\"url\\\":\\\"https://incident.io\\\",\\\"logo\\\":\\\"https://incident.io/icons/icon-512x512.png\\\",\\\"sameAs\\\":[\\\"https://x.com/incident_io\\\",\\\"https://www.linkedin.com/company/incident-io\\\",\\\"https://www.youtube.com/@incident-io\\\",\\\"https://bsky.app/profile/incident.io\\\"]}\"}}]],[\"$\",\"$L1a\",null,{\"children\":[\"$\",\"$L1b\",null,{\"children\":[[\"$\",\"$L1c\",null,{}],false,[\"$\",\"$L1d\",null,{\"children\":[\"$\",\"$L2\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L3\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[[\"$\",\"title\",null,{\"children\":\"Page not found | incident.io\"}],\"$L1e\",[\"$\",\"div\",null,{\"className\":\"relative mx-auto max-w-[1192px] max-xl:px-4 flex flex-col gap-12 md:gap-20 lg:pt-20\",\"style\":\"$undefined\",\"children\":[false,[[\"$\",\"section\",null,{\"className\":\"min-h-[518px] bg-surface-secondary rounded-lg text-center flex flex-col items-center justify-center gap-10 p-4\",\"children\":[[\"$\",\"svg\",null,{\"xmlns\":\"http://www.w3.org/2000/svg\",\"fill\":\"none\",\"viewBox\":\"0 0 202 146\",\"className\":\"w-[202px]\",\"children\":[[\"$\",\"path\",null,{\"fill\":\"#F1EBE2\",\"d\":\"M33.494 4.164A26.4 26.4 0 0 1 47.724 0H133.6a6.4 6.4 0 0 1 6.4 6.4V100l58.998 36.16a4.8 4.8 0 0 1 2.148 5.256L200 146H6.4a6.4 6.4 0 0 1-6.4-6.4V29.103a6.4 6.4 0 0 1 2.95-5.391z\"}],[\"$\",\"path\",null,{\"fill\":\"#E4D9C8\",\"d\":\"M50.6 13.2a3.2 3.2 0 0 1 3.2-3.2h73.6a3.2 3.2 0 0 1 3.2 3.2v49.6a3.2 3.2 0 0 1-3.2 3.2H53.8a3.2 3.2 0 0 1-3.2-3.2z\"}],[\"$\",\"path\",null,{\"fill\":\"#161618\",\"fillOpacity\":0.4,\"d\":\"M102.6 46.95a1 1 0 0 1 0 2h-24a1 1 0 1 1 0-2zM81.579 28.706a1 1 0 0 1 1.414 1.414l-2.83 2.828 2.83 2.83a1 1 0 0 1-1.414 1.414l-2.83-2.83-2.828 2.83a1 1 0 0 1-1.414-1.414l2.828-2.83-2.828-2.828a1 1 0 1 1 1.414-1.414l2.828 2.828zm23.899 0a1 1 0 0 1 1.414 1.414l-2.829 2.828 2.829 2.83a1 1 0 0 1-1.414 1.414l-2.829-2.83-2.828 2.83a1 1 0 0 1-1.414-1.414l2.828-2.83-2.828-2.828a1 1 0 0 1 1.414-1.414l2.828 2.828z\"}],[\"$\",\"path\",null,{\"fill\":\"#DD340E\",\"d\":\"M103.4 75.2a3.2 3.2 0 0 1 3.2-3.2h17.6a3.2 3.2 0 0 1 3.2 3.2v9.6a3.2 3.2 0 0 1-3.2 3.2h-17.6a3.2 3.2 0 0 1-3.2-3.2z\"}],[\"$\",\"path\",null,{\"fill\":\"#F25533\",\"d\":\"M105.8 75.2A3.2 3.2 0 0 1 109 72h17.6a3.2 3.2 0 0 1 3.2 3.2v9.6a3.2 3.2 0 0 1-3.2 3.2H109a3.2 3.2 0 0 1-3.2-3.2z\"}],[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M113.546 76.9v4.73h.75v1.03h-.75V84h-1.1v-1.34h-3.59v-1.09l3.43-4.67zm-3.39 4.73h2.29v-3.1zm7.446 2.53c-1.59 0-2.56-1.46-2.56-3.7 0-2.25.97-3.72 2.56-3.72s2.56 1.47 2.56 3.72c0 2.24-.97 3.7-2.56 3.7m-1.4-3.7c0 .56.06 1.04.17 1.44l2.01-3.78c-.21-.23-.48-.35-.78-.35-.87 0-1.4 1-1.4 2.69m.6 2.3c.22.24.49.37.8.37.87 0 1.4-1 1.4-2.67q0-.885-.18-1.5zm8.736-5.86v4.73h.75v1.03h-.75V84h-1.1v-1.34h-3.59v-1.09l3.43-4.67zm-3.39 4.73h2.29v-3.1z\"}],\"$L1f\",\"$L20\",\"$L21\"]}],\"$L22\"]}],\"$L23\",\"$L24\",\"$L25\"]]}]],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]}],\"$L26\",false]}]}]]}]]}]\n"])</script><script>self.__next_f.push([1,"27:I[710007,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\",\"/_next/static/immutable/chunks/39pfb5kk8her1.js\",\"/_next/static/immutable/chunks/3qz9rwp_7jbnb.js\",\"/_next/static/immutable/chunks/1my_9vnng6gxj.js\",\"/_next/static/immutable/chunks/040yti440pn1b.js\",\"/_next/static/immutable/chunks/0v4j5g1ppvtg_.js\",\"/_next/static/immutable/chunks/3buml8s4fdaii.js\",\"/_next/static/immutable/chunks/2_frah5pspzdw.js\",\"/_next/static/immutable/chunks/1mcb8cvy5_n5b.js\",\"/_next/static/immutable/chunks/132szer1wwfxa.js\",\"/_next/static/immutable/chunks/2jjhnej_crwty.js\",\"/_next/static/immutable/chunks/19t-07qzgwjic.js\",\"/_next/static/immutable/chunks/1wpznbhwql3dd.js\",\"/_next/static/immutable/chunks/20izjjpsu0ag8.js\"],\"Button\"]\n28:I[13642,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\",\"/_next/static/immutable/chunks/39pfb5kk8her1.js\",\"/_next/static/immutable/chunks/3qz9rwp_7jbnb.js\",\"/_next/static/immutable/chunks/1my_9vnng6gxj.js\",\"/_next/static/immutable/chunks/040yti440pn1b.js\",\"/_next/static/immutable/chunks/0v4j5g1ppvtg_.js\",\"/_next/static/immutable/chunks/3buml8s4fdaii.js\",\"/_next/static/immutable/chunks/2_frah5pspzdw.js\",\"/_next/static/immutable/chunks/1mcb8cvy5_n5b.js\",\"/_next/static/immutable/chunks/132szer1wwfxa.js\",\"/_next/static/immutable/chunks/2jjhnej_crwty.js\",\"/_next/static/immutable/chunks/19t-07qzgwjic.js\",\"/_next/static/immutable/chunks/1wpznbhwql3dd.js\",\"/_next/static/immutable/chunks/20izjjpsu0ag8.js\"],\"Footer\"]\n29:I[240073,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\"],\"AnalyticsTags\"]\n2a:I[399095,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\",\"/_next/static/immutable/chunks/39pfb5kk8her1.js\",\"/_next/static/immutable/chunks/3qz9rwp_7jbnb.js\",\"/_next/static/immutable/chunks/1my_9vnng6gxj.js\",\"/_next/static/immutable/chunks/040yti440pn1b.js\",\"/_next/static/immutable/chunks/0v4j5g1ppvtg_.js\",\"/_next/static/immutable/chunks/3buml8s4fdaii.js\",\"/_next/static/immutable/chunks/2_frah5pspzdw.js\",\"/_next/static/immutable/chunks/1mcb8cvy5_n5b.js\",\"/_next/static/immutable/chunks/132szer1wwfxa.js\",\"/_next/static/immutable/chunks/2jjhnej_crwty.js\",\"/_next/static/immutable/chunks/19t-07qzgwjic.js\",\"/_next/static/immutable/chunks/1wpznbhwql3dd.js\",\"/_next/static/immutable/chunks/20izjjpsu0ag8.js\"],\"NavigationClientV2\"]\n1f:[\"$\",\"path\",null,{\"fill\":\"url(#sad-404-computer_svg__a)\",\"d\":\"M0 29.102a6.4 6.4 0 0 1 2.614-5.16l.336-.23L33.494 4.164A26.4 26.4 0 0 1 40 1.156V100l58.997 36.16a4.8 4.8 0 0 1 2.149 5.257L100 146H6.4a6.4 6.4 0 0 1-6.4-6.4z\"}]\n20:[\"$\",\"path\",null,{\"fill\":\"#E4D9C8\",\"d\":\"M140 100H24.2v-6H140z\"}]\n21:[\"$\",\"defs\",null,{\"children\":[\"$\",\"linearGradient\",null,{\"id\":\"sad-404-computer_svg__a\",\"x1\":50.645,\"x2\":50.645,\"y1\":1.156,\"y2\":87.3,\"gradientUnits\":\"userSpaceOnUse\",\"children\":[[\"$\",\"stop\",null,{\"stopColor\":\"#E4D9C8\",\"stopOpacity\":0}],[\"$\",\"stop\",null,{\"offset\":1,\"stopColor\":\"#E4D9C8\"}]]}]}]\n22:[\"$\",\"div\",null,{\"className\":\"flex flex-col gap-5\",\"children\":[[\"$\",\"h2\",null,{\"className\":\"text-h2 md:text-h1-60\",\"children\":\"Something went sideways\"}],[\"$\",\"p\",null,{\"className\":\"text-body text-content-secondary\",\"children\":\"No need to page anyone. Let’s get you back on track.\"}],[\"$\",\"$L27\",null,{\"href\":\"/\",\"showArrow\":true,\"theme\":\"primary\",\"className\":\"self-center\",\"children\":\"Go back home\"}]]}]\n24:[\"$\",\"hr\",null,{\"className\":\"h-px w-full border-stroke-primary max-w-[1192px] mx-auto\"}]\n25:[\"$\",\"$L28\",null,{\"useLegacyWidth\":false}]\n26:[\"$\",\"$L29\",null,{\"isDraftModeEnabled\":false}]\n1e:[\"$\",\"$L2a\",null,{\"className\":\"$undefined\",\"theme\":\"light\",\"stickyNavTheme\":\"dark\",\"stickyNavClassName\":\"$undefined\",\"stickyNavAfterSlot\":\"$undefined\",\"hasBlurredBackground\":\"$undefined\",\"showSignupButtonOnInitialLoad\":\"$undefined\",\"showSignupButtonInStickyNav\":\"$undefined\",\"showNavContent\":\"$undefined\",\"showCtas\":[\"request-demo\",\"login\"],\"children\":\"$undefined\",\"mobileChildren\":\"$undefined\",\"initiallyTransparent\":\"$undefined\",\"switchToLightOnScroll\":\"$undefined\",\"customCtas\":\"$undefined\",\"demoHref\":\"$undefined\"}]\n"])</script><script>self.__next_f.push([1,"2c:I[898118,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\",\"/_next/static/immutable/chunks/39pfb5kk8her1.js\",\"/_next/static/immutable/chunks/3qz9rwp_7jbnb.js\",\"/_next/static/immutable/chunks/1my_9vnng6gxj.js\",\"/_next/static/immutable/chunks/040yti440pn1b.js\",\"/_next/static/immutable/chunks/0v4j5g1ppvtg_.js\",\"/_next/static/immutable/chunks/3buml8s4fdaii.js\",\"/_next/static/immutable/chunks/2_frah5pspzdw.js\",\"/_next/static/immutable/chunks/1mcb8cvy5_n5b.js\",\"/_next/static/immutable/chunks/132szer1wwfxa.js\",\"/_next/static/immutable/chunks/2jjhnej_crwty.js\",\"/_next/static/immutable/chunks/19t-07qzgwjic.js\",\"/_next/static/immutable/chunks/1wpznbhwql3dd.js\",\"/_next/static/immutable/chunks/20izjjpsu0ag8.js\"],\"ContentPostTemplate\"]\n32:I[27201,[\"/_next/static/immutable/chunks/0esgme6slw2os.js\"],\"IconMark\"]\n2d:T4be,\u003ctable\u003e\n \u003cthead\u003e\n \u003ctr\u003e\n \u003cth\u003eTime\u003c/th\u003e\n \u003cth\u003eEvent\u003c/th\u003e\n \u003c/tr\u003e\n \u003c/thead\u003e\n \u003ctbody\u003e\n \u003ctr\u003e\n \u003ctd\u003e15:40\u003c/td\u003e\n \u003ctd\u003e\n \u003ca href=\"#initial-crash\"\u003eInitial crash\u003c/a\u003e\n \u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd\u003e15:42\u003c/td\u003e\n \u003ctd\u003e\n \u003ca href=\"#initial-crash\"\u003eRestarted dynos, back up\u003c/a\u003e\n \u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd\u003e15:47\u003c/td\u003e\n \u003ctd\u003e\n \u003ca href=\"#subsequent-crashes\"\u003eSubsequent crashes\u003c/a\u003e\n \u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd\u003e16:00\u003c/td\u003e\n \u003ctd\u003e\n \u003ca href=\"#subsequent-crashes\"\u003eBegin purging GCP subscriptions\u003c/a\u003e\n \u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd\u003e16:12\u003c/td\u003e\n \u003ctd\u003e\n \u003ca href=\"#subsequent-crashes\"\u003eApp stops crashing\u003c/a\u003e\n \u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd\u003e16:13\u003c/td\u003e\n \u003ctd\u003e\n \u003ca href=\"#regroup\"\u003eRegroup, start on mitigations\u003c/a\u003e\n \u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd\u003e16:48\u003c/td\u003e\n \u003ctd\u003e\n \u003ca href=\"#mitigation-1-ensuring-we-handle-message-panics\"\u003eDeploy fix to handle panics\u003c/a\u003e\n \u003c/td\u003e\n \u003c/tr\u003e\n \u003ctr\u003e\n \u003ctd\u003e17:24\u003c/td\u003e\n \u003ctd\u003e\n \u003ca href=\"#mitigation-2-separate-our-app-by-work-type\"\u003eDeploy split of web, worker and cron into different dynos\u003c/a\u003e\n \u003c/td\u003e\n \u003c/tr\u003e\n \u003c/tbody\u003e\n\u003c/table\u003e5:[false,[[[[\"$\",\"script\",\"article-jsonld-BlogPosting\",{\"type\":\"application/ld+json\",\"id\":\"article-jsonld-BlogPosting\",\"data-testid\":\"$undefined\",\"dangerouslySetInnerHTML\":{\"__html\":\"{\\\"@context\\\":\\\"https://schema.org\\\",\\\"@type\\\":\\\"BlogPosting\\\",\\\"headline\\\":\\\"Intermittent downtime from repeated crashes\\\",\\\"url\\\":\\\"https://incident.io/blog/intermittent-downtime\\\",\\\"author\\\":[{\\\"@type\\\":\\\"Person\\\",\\\"name\\\":\\\"Lawrence Jones\\\"}],\\\"datePublished\\\":\\\"2022-11-30T00:00:00.000Z\\\",\\\"dateModified\\\":\\\"2022-11-30T00:00:00.000Z\\\",\\\"image\\\":[\\\"https://cdn.sanity.io/images/oqy5aexb/production/ee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png?w=2000\u0026fit=max\u0026auto=format\\\"],\\\"publisher\\\":{\\\"@type\\\":\\\"Organization\\\",\\\"name\\\":\\\"incident.io\\\",\\\"url\\\":\\\"https://incident.io\\\"},\\\"description\\\":\\\"This is a technical write-up of an incident on Friday 18th November 2022 where we experienced 13 minutes of downtime from intermittent crashes.\\\"}\"}}]],null],\"$L2b\",[\"$\",\"div\",null,{\"className\":\"relative mx-auto max-w-[1192px] max-xl:px-4 flex flex-col gap-12 md:gap-20 lg:pt-20\",\"style\":\"$undefined\",\"children\":[false,[[\"$\",\"$L2c\",null,{\"allPostsPath\":\"/blog\",\"post\":{\"_id\":\"imported-blog-2022-11-30-intermitten-downtime-index-md\",\"_type\":\"blog\",\"authors\":[{\"avatar\":{\"_type\":\"image\",\"asset\":{\"_ref\":\"image-e7b906adbcc6184c09d6abda07181ee946f50ab5-800x800-png\",\"_type\":\"reference\"}},\"bio\":null,\"id\":\"lawrence\",\"name\":\"Lawrence Jones\",\"title\":\"Product Engineer\"}],\"body\":[{\"_key\":\"5f77ccf8a9fc\",\"_type\":\"block\",\"children\":[{\"_key\":\"5f77ccf8a9fc0\",\"_type\":\"span\",\"marks\":[],\"text\":\"On Friday 18th November 2022 we experienced 13 minutes of downtime over a period of 32 minutes from 15:40 to 16:12 GMT.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"c53bd000b229\",\"_type\":\"block\",\"children\":[{\"_key\":\"c53bd000b2290\",\"_type\":\"span\",\"marks\":[],\"text\":\"At incident.io, our customers depend on us to be available even when everything else is down, so we take incidents like this seriously. We also sincerely believe in the power of transparency, and want to be as open as possible in these situations, sharing what details we can.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"a94f2447706d\",\"_type\":\"block\",\"children\":[{\"_key\":\"a94f2447706d0\",\"_type\":\"span\",\"marks\":[],\"text\":\"That’s why we’re sharing this technical write-up, explaining what happened and also to share the measures we’ve since taken to improve our reliability.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"3d46beeb7ee8\",\"_type\":\"block\",\"children\":[{\"_key\":\"3d46beeb7ee80\",\"_type\":\"span\",\"marks\":[],\"text\":\"Here's a summary timeline with links, if you'd like to jump through:\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"68b12959ec13\",\"_type\":\"rawhtml\",\"html\":\"$2d\",\"markDefs\":null},{\"_key\":\"ed3d30a1267f\",\"_type\":\"block\",\"children\":[{\"_key\":\"ed3d30a1267f0\",\"_type\":\"span\",\"marks\":[],\"text\":\"Background\"}],\"markDefs\":[{\"_type\":\"slug\",\"_key\":\"slug-background\",\"slug\":\"background\"}],\"style\":\"h2\"},{\"_key\":\"4d03781651c6\",\"_type\":\"block\",\"children\":[{\"_key\":\"4d03781651c60\",\"_type\":\"span\",\"marks\":[],\"text\":\"We run our app as a Go monolith in Heroku deployed using container buildpack, Heroku Postgres as a database, and GCP Pub/Sub as an async message queue.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"725bd0e97316\",\"_type\":\"block\",\"children\":[{\"_key\":\"725bd0e973160\",\"_type\":\"span\",\"marks\":[],\"text\":\"Our application runs as a single binary running web, worker and cron threads.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"75d04c5f75a2\",\"_type\":\"block\",\"children\":[{\"_key\":\"75d04c5f75a20\",\"_type\":\"span\",\"marks\":[],\"text\":\"For async work, we have separate Pub/Sub subscriptions that are served by message handlers within our app. If the message handler returns an error, we rely on Pub/Sub to perform back-off on message redelivery.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"d536cf1820ca\",\"_type\":\"block\",\"children\":[{\"_key\":\"d536cf1820ca0\",\"_type\":\"span\",\"marks\":[],\"text\":\"We take care to ensure all entrypoints to the app – incoming web requests, Pub/Sub messages in the event queue, cron jobs – are wrapped in panic handlers to prevent an unhandled panic from taking the app down.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"929140ca20f1\",\"_type\":\"block\",\"children\":[{\"_key\":\"929140ca20f10\",\"_type\":\"span\",\"marks\":[],\"text\":\"Initial crash\"}],\"markDefs\":[{\"_type\":\"slug\",\"_key\":\"slug-initial-crash\",\"slug\":\"initial-crash\"}],\"style\":\"h2\"},{\"_key\":\"0ad1ee1973bc\",\"_type\":\"block\",\"children\":[{\"_key\":\"0ad1ee1973bc0\",\"_type\":\"span\",\"marks\":[],\"text\":\"Twenty minutes before our end-of-week team debrief, our on-call engineer was paged due to an \\\"\"},{\"_key\":\"0ad1ee1973bc1\",\"_type\":\"span\",\"marks\":[\"bd77973e53e5\"],\"text\":\"H10 - App crashed\"},{\"_key\":\"0ad1ee1973bc2\",\"_type\":\"span\",\"marks\":[],\"text\":\"\\\" error code from Heroku, which means the app has terminated.\"}],\"markDefs\":[{\"_key\":\"bd77973e53e5\",\"_type\":\"link\",\"href\":\"https://devcenter.heroku.com/articles/error-codes#h10-app-crashed\"}],\"style\":\"normal\"},{\"_key\":\"716e927387ce\",\"_type\":\"block\",\"children\":[{\"_key\":\"716e927387ce0\",\"_type\":\"span\",\"marks\":[],\"text\":\"This type of crash happens when a Heroku dyno totally dies, which can happen in response to many events:\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"4d72e0bc57e8\",\"_type\":\"block\",\"children\":[{\"_key\":\"4d72e0bc57e80\",\"_type\":\"span\",\"marks\":[],\"text\":\"An unhandled panic in the Go runtime has caused the app to die.\"}],\"level\":1,\"listItem\":\"number\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"6f102b3738b9\",\"_type\":\"block\",\"children\":[{\"_key\":\"6f102b3738b90\",\"_type\":\"span\",\"marks\":[],\"text\":\"Something has caused the application to exceed its memory limit and we’ve been OOM (out-of-memory) killed.\"}],\"level\":1,\"listItem\":\"number\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"c6f7c94959cb\",\"_type\":\"block\",\"children\":[{\"_key\":\"c6f7c94959cb0\",\"_type\":\"span\",\"marks\":[],\"text\":\"Heroku has a fault, or some infrastructure error has caused the process to die.\"}],\"level\":1,\"listItem\":\"number\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"3976b7affac3\",\"_type\":\"block\",\"children\":[{\"_key\":\"3976b7affac30\",\"_type\":\"span\",\"marks\":[],\"text\":\"Regardless of the cause, Heroku responds to this event by following its \"},{\"_key\":\"3976b7affac31\",\"_type\":\"span\",\"marks\":[\"59738dc95d24\"],\"text\":\"Dyno crash restart policy\"},{\"_key\":\"3976b7affac32\",\"_type\":\"span\",\"marks\":[],\"text\":\", which can be summarised as:\"}],\"markDefs\":[{\"_key\":\"59738dc95d24\",\"_type\":\"link\",\"href\":\"https://devcenter.heroku.com/articles/dynos#dyno-crash-restart-policy\"}],\"style\":\"normal\"},{\"_key\":\"342801522eeb\",\"_type\":\"block\",\"children\":[{\"_key\":\"342801522eeb0\",\"_type\":\"span\",\"marks\":[],\"text\":\"The first time a dyno crashes, it will be restarted immediately\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"7db52c5bb415\",\"_type\":\"block\",\"children\":[{\"_key\":\"7db52c5bb4150\",\"_type\":\"span\",\"marks\":[],\"text\":\"If the dyno crashes again, it will be subject to a cool-off period (up-to 20m) before a restart is attempted.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"12d79c324091\",\"_type\":\"block\",\"children\":[{\"_key\":\"12d79c3240910\",\"_type\":\"span\",\"marks\":[],\"text\":\"Subsequent crashes increase the ‘cool off’ period, but it matters little to us: waiting 20m to restart our application would be an unacceptable amount of downtime.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"40fdcde08a0a\",\"_type\":\"block\",\"children\":[{\"_key\":\"40fdcde08a0a0\",\"_type\":\"span\",\"marks\":[],\"text\":\"From previous incidents, we’ve learned that manually restarting dynos via the Heroku console is a sensible response whenever an H10 occurs that we don’t immediately recover from. We could see from our monitoring that we hadn’t immediately recovered, suggesting we’d crashed quickly after the first restart, so we triggered a manual reboot at 15:42.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"7d19c5bff734\",\"_type\":\"block\",\"children\":[{\"_key\":\"7d19c5bff7340\",\"_type\":\"span\",\"marks\":[],\"text\":\"The manual reboot brought the app back up, which meant our PagerDuty automation activated (see our \"},{\"_key\":\"7d19c5bff7341\",\"_type\":\"span\",\"marks\":[\"6cee688abe60\"],\"text\":\"response flow\"},{\"_key\":\"7d19c5bff7342\",\"_type\":\"span\",\"marks\":[],\"text\":\") and we get an \"},{\"_key\":\"7d19c5bff7343\",\"_type\":\"span\",\"marks\":[\"3da1317a5ffe\"],\"text\":\"incident.io\"},{\"_key\":\"7d19c5bff7344\",\"_type\":\"span\",\"marks\":[],\"text\":\" incident channel for this incident. The on-call and response team join the channel and start examining logs.\"}],\"markDefs\":[{\"_key\":\"6cee688abe60\",\"_type\":\"link\",\"href\":\"https://incident.io/blog/realtime-response\"},{\"_key\":\"3da1317a5ffe\",\"_type\":\"link\",\"href\":\"http://incident.io\"}],\"style\":\"normal\"},{\"_key\":\"b4a439ad54c8\",\"_type\":\"block\",\"children\":[{\"_key\":\"b4a439ad54c80\",\"_type\":\"span\",\"marks\":[],\"text\":\"Subsequent crashes\"}],\"markDefs\":[{\"_type\":\"slug\",\"_key\":\"slug-subsequent-crashes\",\"slug\":\"subsequent-crashes\"}],\"style\":\"h2\"},{\"_key\":\"5d2954020a31\",\"_type\":\"block\",\"children\":[{\"_key\":\"5d2954020a310\",\"_type\":\"span\",\"marks\":[],\"text\":\"Not yet knowing what caused the crash, the team start collating possible causes. Prior incident experience means our guess is a bad Pub/Sub event being consumed that triggers one of the H10 failure cases whenever it is processed, a situation often called a “poison pill”.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"889338908f44\",\"_type\":\"block\",\"children\":[{\"_key\":\"889338908f440\",\"_type\":\"span\",\"marks\":[],\"text\":\"The immediate questions are:\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"ce70e3eb9f8a\",\"_type\":\"block\",\"children\":[{\"_key\":\"ce70e3eb9f8a0\",\"_type\":\"span\",\"marks\":[],\"text\":\"Can we determine it was an event that caused this?\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"7b7bdd0678dd\",\"_type\":\"block\",\"children\":[{\"_key\":\"7b7bdd0678dd0\",\"_type\":\"span\",\"marks\":[],\"text\":\"If so, which event is it?\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"459fe24069f8\",\"_type\":\"block\",\"children\":[{\"_key\":\"459fe24069f80\",\"_type\":\"span\",\"marks\":[],\"text\":\"As we begin to investigate, we get paged again: the app is down due to – presumably – the same error, and we assign an engineer to watch for subsequent crashes so we can quickly trigger a manual start, which we know brings the app back up.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"8f9beeaddc64\",\"_type\":\"block\",\"children\":[{\"_key\":\"8f9beeaddc640\",\"_type\":\"span\",\"marks\":[],\"text\":\"It’s now 15:47, and we allocate people onto several streams of work:\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"4219ec187238\",\"_type\":\"block\",\"children\":[{\"_key\":\"4219ec1872380\",\"_type\":\"span\",\"marks\":[],\"text\":\"Someone to lead the incident, which means coordinating our response.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"8000fbda980c\",\"_type\":\"block\",\"children\":[{\"_key\":\"8000fbda980c0\",\"_type\":\"span\",\"marks\":[],\"text\":\"As mentioned, one person watching for crashes and manually restarting Dynos.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"504c5b1e1887\",\"_type\":\"block\",\"children\":[{\"_key\":\"504c5b1e18870\",\"_type\":\"span\",\"marks\":[],\"text\":\"Someone is assigned to Communications, which involves updating our public status page.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"95f7708462b3\",\"_type\":\"block\",\"children\":[{\"_key\":\"95f7708462b30\",\"_type\":\"span\",\"marks\":[],\"text\":\"And a few engineers start debugging using a variety of tools, from logs, metrics and traces to try and figure out what’s going on.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"103263b6ffd0\",\"_type\":\"block\",\"children\":[{\"_key\":\"103263b6ffd00\",\"_type\":\"span\",\"marks\":[],\"text\":\"See how we use our tools in \"},{\"_key\":\"103263b6ffd01\",\"_type\":\"span\",\"marks\":[\"13c5a9aaffa5\"],\"text\":\"Logs and tracing: not just for production\"},{\"_key\":\"103263b6ffd02\",\"_type\":\"span\",\"marks\":[],\"text\":\".\"}],\"markDefs\":[{\"_key\":\"13c5a9aaffa5\",\"_type\":\"link\",\"href\":\"https://incident.io/blog/tracing\"}],\"style\":\"blockquote\"},{\"_key\":\"709d5885a18d\",\"_type\":\"block\",\"children\":[{\"_key\":\"709d5885a18d0\",\"_type\":\"span\",\"marks\":[],\"text\":\"This investigation continues for some time, due to several roadblocks that slowed us.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"f0a2cbee0bd5\",\"_type\":\"block\",\"children\":[{\"_key\":\"f0a2cbee0bd50\",\"_type\":\"span\",\"marks\":[],\"text\":\"Firstly, while we have Heroku capturing logs from each Dyno stdout/stderr into Corologix, when a Go app panics it will print the stacktrace of every goroutine in the application (for those unfamiliar, a goroutine is similar to a thread).\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"6ff587593b99\",\"_type\":\"block\",\"children\":[{\"_key\":\"6ff587593b990\",\"_type\":\"span\",\"marks\":[],\"text\":\"As our application runs with ~300 goroutines, printing the stacktrace of each goroutine caused Heroku to buffer and drop these log lines, preventing us from seeing the original crashing stacktrace.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"f9b48686e2e3\",\"_type\":\"block\",\"children\":[{\"_key\":\"f9b48686e2e30\",\"_type\":\"span\",\"marks\":[],\"text\":\"Then due to Go immediately exiting upon any unhandled panic, even the parts of our app with panic handlers – such as our Sentry error reporting – were terminated, meaning we couldn’t inspect the crash in our exception tracker.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"0bd0130fd43b\",\"_type\":\"block\",\"children\":[{\"_key\":\"0bd0130fd43b0\",\"_type\":\"span\",\"marks\":[],\"text\":\"Finally, Heroku’s rate limiting of the logs was even impacting the logs that came just prior to a crash, so we couldn’t even see when we began to process the problematic event.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"8415e7420497\",\"_type\":\"block\",\"children\":[{\"_key\":\"8415e74204970\",\"_type\":\"span\",\"marks\":[],\"text\":\"After 5m of trying and failing to retrieve usable logs, we change strategy.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"3ac5921437a1\",\"_type\":\"block\",\"children\":[{\"_key\":\"3ac5921437a10\",\"_type\":\"span\",\"marks\":[],\"text\":\"Knowing that this crash happened about 15m away from the last deploy, we know it’s unlikely to be a frequently used event that’s causing this, and rolling back is therefore unlikely to fix our issue, since we don’t yet know what introduced the bug.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"09d081ab412a\",\"_type\":\"block\",\"children\":[{\"_key\":\"09d081ab412a0\",\"_type\":\"span\",\"marks\":[],\"text\":\"We conclude the original crash was probably the first time we processed this bad event, and start looking at all subscriptions with unacked messages from around the time we first went down. For all subscriptions that match this criteria, we purge the subscription, removing all the events from the queue. This is safe as we treat async events as best-effort, meaning dropping events will only degrade service, which is preferable if it means we come back up.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"a654bc874f7e\",\"_type\":\"block\",\"children\":[{\"_key\":\"a654bc874f7e0\",\"_type\":\"span\",\"marks\":[],\"text\":\"We also turn off several non-critical parts of the app that trigger async work, prioritising core incident functionality over features such as incident nudges (”It’s been a while since you last updated the incident, would you like to make an update?”).\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"c9b49bddc461\",\"_type\":\"block\",\"children\":[{\"_key\":\"c9b49bddc4610\",\"_type\":\"span\",\"marks\":[],\"text\":\"At 16:12, after purging several subscriptions that matched our criteria, the app stopped crashing and remained stable, without any further intervention.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"67ec3aeaa9b3\",\"_type\":\"block\",\"children\":[{\"_key\":\"67ec3aeaa9b30\",\"_type\":\"span\",\"marks\":[],\"text\":\"Regroup\"}],\"markDefs\":[{\"_type\":\"slug\",\"_key\":\"slug-regroup\",\"slug\":\"regroup\"}],\"style\":\"h2\"},{\"_key\":\"22b0b13056d3\",\"_type\":\"block\",\"children\":[{\"_key\":\"22b0b13056d30\",\"_type\":\"span\",\"marks\":[],\"text\":\"The subscription responsible was used for generating a welcome message whenever people first enter an incident channel. From looking at the code, we determined that for a very small subset of our customers we'd do this in a way which would cause the code to fail.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"f9510f431eb5\",\"_type\":\"block\",\"children\":[{\"_key\":\"f9510f431eb50\",\"_type\":\"span\",\"marks\":[],\"text\":\"This meant it was unlikely we’d produce another message like this for a while, and we knew which subscription to purge if we did. So with urgency reduced, we called a pause so we could regroup.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"b755596db66d\",\"_type\":\"block\",\"children\":[{\"_key\":\"b755596db66d0\",\"_type\":\"span\",\"marks\":[],\"text\":\"For us, that meant:\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"c7a2b4b82c26\",\"_type\":\"block\",\"children\":[{\"_key\":\"c7a2b4b82c260\",\"_type\":\"span\",\"marks\":[],\"text\":\"Have the incident lead run a stand-up with all responders, checking what work remains and confirming everyone has what they need (information, support, clarity on role but also \\\"something to eat and drink\\\").\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"4b4524754f3b\",\"_type\":\"block\",\"children\":[{\"_key\":\"4b4524754f3b0\",\"_type\":\"span\",\"marks\":[],\"text\":\"Confirm our collective understanding of the incident, summarise it into a written update.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"0e490ec381ef\",\"_type\":\"block\",\"children\":[{\"_key\":\"0e490ec381ef0\",\"_type\":\"span\",\"marks\":[],\"text\":\"Produce an update we could share with our customers, focused on the impact they’d feel from the incident.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"c3de4243bf0f\",\"_type\":\"block\",\"children\":[{\"_key\":\"c3de4243bf0f0\",\"_type\":\"span\",\"marks\":[],\"text\":\"Decide next steps and allocate work.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"f40a84244e80\",\"_type\":\"block\",\"children\":[{\"_key\":\"f40a84244e800\",\"_type\":\"span\",\"marks\":[],\"text\":\"Technically, we had a couple of things we wanted to try putting in place to quickly improve our position before closing this incident.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"d080a065f99f\",\"_type\":\"block\",\"children\":[{\"_key\":\"d080a065f99f0\",\"_type\":\"span\",\"marks\":[],\"text\":\"Mitigation 1: Ensuring we handle message panics\"}],\"markDefs\":[{\"_type\":\"slug\",\"_key\":\"slug-mitigation-1-ensuring-we-handle-message-panics\",\"slug\":\"mitigation-1-ensuring-we-handle-message-panics\"}],\"style\":\"h2\"},{\"_key\":\"90dd05966f67\",\"_type\":\"block\",\"children\":[{\"_key\":\"90dd05966f670\",\"_type\":\"span\",\"marks\":[],\"text\":\"One rule you learn quickly when running Go apps is to catch your panics: this means ensuring any goroutine (thread) that might panic – segfault, or crashing via an explicit \"},{\"_key\":\"90dd05966f671\",\"_type\":\"span\",\"marks\":[\"code\"],\"text\":\"panic\"},{\"_key\":\"90dd05966f672\",\"_type\":\"span\",\"marks\":[],\"text\":\" – has added a \"},{\"_key\":\"90dd05966f673\",\"_type\":\"span\",\"marks\":[\"code\"],\"text\":\"recover()\"},{\"_key\":\"90dd05966f674\",\"_type\":\"span\",\"marks\":[],\"text\":\".\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"1e22fd3a9ebe\",\"_type\":\"block\",\"children\":[{\"_key\":\"1e22fd3a9ebe0\",\"_type\":\"span\",\"marks\":[],\"text\":\"As a basic example:\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"9a3818bf9e14\",\"_type\":\"code\",\"code\":\"func main() {\\n defer func() {\\n if err := recover(); err != nil {\\n fmt.Println(\\\"caught error: \\\", err)\\n }\\n }()\\n\\n // Try to access a blank user pointer.\\n var blankPointer *User\\n fmt.Println(blankPointer.Name) // uh-oh, segfault!\\n}\\n\",\"language\":\"go\",\"markDefs\":null},{\"_key\":\"0198b398f3e5\",\"_type\":\"block\",\"children\":[{\"_key\":\"0198b398f3e50\",\"_type\":\"span\",\"marks\":[],\"text\":\"This main function tries printing the \"},{\"_key\":\"0198b398f3e51\",\"_type\":\"span\",\"marks\":[\"code\"],\"text\":\"Name\"},{\"_key\":\"0198b398f3e52\",\"_type\":\"span\",\"marks\":[],\"text\":\" field on \"},{\"_key\":\"0198b398f3e53\",\"_type\":\"span\",\"marks\":[\"code\"],\"text\":\"blankPointer\"},{\"_key\":\"0198b398f3e54\",\"_type\":\"span\",\"marks\":[],\"text\":\", which is a zero-value pointer to a User struct. That will fail because we’ll try dereferencing a zero-value, causing a segmentation fault where the kernel refuses to allow our application to read memory outside its virtual address range, which does not contain 0.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"39a387cffbc5\",\"_type\":\"block\",\"children\":[{\"_key\":\"39a387cffbc50\",\"_type\":\"span\",\"marks\":[],\"text\":\"Thankfully, \"},{\"_key\":\"39a387cffbc51\",\"_type\":\"span\",\"marks\":[\"code\"],\"text\":\"main\"},{\"_key\":\"39a387cffbc52\",\"_type\":\"span\",\"marks\":[],\"text\":\" has deferred a function that calls \"},{\"_key\":\"39a387cffbc53\",\"_type\":\"span\",\"marks\":[\"code\"],\"text\":\"recover()\"},{\"_key\":\"39a387cffbc54\",\"_type\":\"span\",\"marks\":[],\"text\":\", which is a special function that will catch and turn any unhandled panics into a runtime error.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"5b6ccf0c1ece\",\"_type\":\"block\",\"children\":[{\"_key\":\"5b6ccf0c1ece0\",\"_type\":\"span\",\"marks\":[],\"text\":\"This means that instead of our app crashing with a panic, it will print the error and continue:\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"2b4b484850ce\",\"_type\":\"code\",\"code\":\"$$ go run main.go\\ncaught error: runtime error: invalid memory address or nil pointer dereference\\n\",\"language\":\"go\",\"markDefs\":null},{\"_key\":\"7b2ef0ea5417\",\"_type\":\"block\",\"children\":[{\"_key\":\"7b2ef0ea54170\",\"_type\":\"span\",\"marks\":[],\"text\":\"We add these panic handlers all over our app, so we shouldn’t have hit this issue. But clearly we did, given our app has just crashed repeatedly.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"81c841a98432\",\"_type\":\"block\",\"children\":[{\"_key\":\"81c841a984320\",\"_type\":\"span\",\"marks\":[],\"text\":\"So what gives? Well, turns out there’s an edge case that is easy to miss, and it existed in the generic event consumer for our async workers:\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"99779f6f8381\",\"_type\":\"code\",\"code\":\"func receive(ctx context.Context, client *pubsub.Client, subscriptionID string) error {\\n defer func() {\\n // recover\\n }()\\n\\n sub := client.Subscription(subscriptionID)\\n return sub.Receive(ctx, func(ctx context.Context, msg *pubsub.Message) {\\n consumer.handle(ctx, msg) // call the event consumer\\n })\\n}\\n\",\"language\":\"go\",\"markDefs\":null},{\"_key\":\"83fe68f35a44\",\"_type\":\"block\",\"children\":[{\"_key\":\"83fe68f35a440\",\"_type\":\"span\",\"marks\":[],\"text\":\"The generic receiver did apply a recover, but we’d missed that the implementation of \"},{\"_key\":\"83fe68f35a441\",\"_type\":\"span\",\"marks\":[\"code\"],\"text\":\"sub.Receive\"},{\"_key\":\"83fe68f35a442\",\"_type\":\"span\",\"marks\":[],\"text\":\" – which is in the Google Cloud Pub/Sub client – starts several new goroutines to handle the anonymous function we provide to the \"},{\"_key\":\"83fe68f35a443\",\"_type\":\"span\",\"marks\":[\"code\"],\"text\":\"Receive\"},{\"_key\":\"83fe68f35a444\",\"_type\":\"span\",\"marks\":[],\"text\":\" method.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"75ec618a83cc\",\"_type\":\"block\",\"children\":[{\"_key\":\"75ec618a83cc0\",\"_type\":\"span\",\"marks\":[],\"text\":\"In Go, goroutines have no meaningful links back to their parent. If your child goroutine panics, it won’t benefit from the parent goroutine having applied a recover: it’s an unhandled panic, and your whole app would crash.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"b88690c9e764\",\"_type\":\"block\",\"children\":[{\"_key\":\"b88690c9e7640\",\"_type\":\"span\",\"marks\":[],\"text\":\"So our first mitigation was to apply a recover to this handler, which we deployed at 16:48.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"d7cdabeabe32\",\"_type\":\"block\",\"children\":[{\"_key\":\"d7cdabeabe320\",\"_type\":\"span\",\"marks\":[],\"text\":\"Since this is a common piece of code used by all our event handlers, this mitigates the issue which caused this incident but it also removes this entire problem class for all other current and future subscriptions too.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"c7c52792f6ca\",\"_type\":\"block\",\"children\":[{\"_key\":\"c7c52792f6ca0\",\"_type\":\"span\",\"marks\":[],\"text\":\"Mitigation 2: Separate our app by work type\"}],\"markDefs\":[{\"_type\":\"slug\",\"_key\":\"slug-mitigation-2-separate-our-app-by-work-type\",\"slug\":\"mitigation-2-separate-our-app-by-work-type\"}],\"style\":\"h2\"},{\"_key\":\"e8cf758e78ce\",\"_type\":\"block\",\"children\":[{\"_key\":\"e8cf758e78ce0\",\"_type\":\"span\",\"marks\":[],\"text\":\"It’s considered best practice to run different work types – say web requests, async workers, and long-running cron jobs – separately from one another.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"6d23c5553e04\",\"_type\":\"block\",\"children\":[{\"_key\":\"6d23c5553e040\",\"_type\":\"span\",\"marks\":[],\"text\":\"That’s because:\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"9174abc6ba95\",\"_type\":\"block\",\"children\":[{\"_key\":\"9174abc6ba950\",\"_type\":\"span\",\"marks\":[],\"text\":\"Different work often demands different resources, like CPU or memory. Running things separately allows right-sizing the machine that hosts that class of work.\"}],\"level\":1,\"listItem\":\"number\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"de41d8c62cdc\",\"_type\":\"block\",\"children\":[{\"_key\":\"de41d8c62cdc0\",\"_type\":\"span\",\"marks\":[],\"text\":\"Physically separating work improves reliability, as it prevents competition for shared resources like CPU, avoiding one thread consuming most of your processing ability and everything else slow/stopping.\"}],\"level\":1,\"listItem\":\"number\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"4e42f9b050ac\",\"_type\":\"block\",\"children\":[{\"_key\":\"4e42f9b050ac0\",\"_type\":\"span\",\"marks\":[],\"text\":\"If one type of work behaves badly, such as panicking, it can’t adversely impact everything else.\"}],\"level\":1,\"listItem\":\"number\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"9a9a5ea749f2\",\"_type\":\"block\",\"children\":[{\"_key\":\"9a9a5ea749f20\",\"_type\":\"span\",\"marks\":[],\"text\":\"In our case (3) is what we care about, as had we separated async work from web traffic then even if workers had died, we would continue to serve requests for Slack interactions and our web dashboard. It’s likely no one would even notice the workers were down, provided we manually restarted them quickly.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"f92a3776e4ff\",\"_type\":\"block\",\"children\":[{\"_key\":\"f92a3776e4ff0\",\"_type\":\"span\",\"marks\":[],\"text\":\"We had begun the process of splitting these work types before this incident, and the app already had a clear logical separation between web, async workers and crons. Not only was the code separate, but each component used separate database connection pools – preventing expensive workers from stealing all web database connections – and the entrypoint to the app allowed toggling components on-or-off via command line flags.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"c2d8e5a3df0d\",\"_type\":\"block\",\"children\":[{\"_key\":\"c2d8e5a3df0d0\",\"_type\":\"span\",\"marks\":[],\"text\":\"The reason we hadn't completed the split was that we'd already got most of the value from logically separating, and prefer running as simple an app as possible. Adding another dyno would also increase our CI times slightly, so we'd been waiting for a clear push before making this change: obviously an incident like this is exactly that trigger.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"302c53c5dc6f\",\"_type\":\"block\",\"children\":[{\"_key\":\"302c53c5dc6f0\",\"_type\":\"span\",\"marks\":[],\"text\":\"So after confirming we were happy with the approach, we implemented the physical split by adding a new ‘worker’ Dyno tier to Heroku and tweaking each entrypoint:\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"d7cb63f1c788\",\"_type\":\"block\",\"children\":[{\"_key\":\"d7cb63f1c7880\",\"_type\":\"span\",\"marks\":[],\"text\":\"\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"87512b63dc08\",\"_type\":\"image\",\"asset\":{\"_ref\":\"image-6d63615fb4a2893b1f17860dea5dfbf8884e75f8-1726x566-png\",\"_type\":\"reference\"},\"description\":\"Heroku dynos now split between web, worker and cron\",\"markDefs\":null},{\"_key\":\"49ec28604720\",\"_type\":\"block\",\"children\":[{\"_key\":\"49ec286047200\",\"_type\":\"span\",\"marks\":[],\"text\":\"By 17:24, this change was running in production.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"e96b632320f2\",\"_type\":\"block\",\"children\":[{\"_key\":\"e96b632320f20\",\"_type\":\"span\",\"marks\":[],\"text\":\"Conclusion\"}],\"markDefs\":[{\"_type\":\"slug\",\"_key\":\"slug-conclusion\",\"slug\":\"conclusion\"}],\"style\":\"h2\"},{\"_key\":\"0c72a937459f\",\"_type\":\"block\",\"children\":[{\"_key\":\"0c72a937459f0\",\"_type\":\"span\",\"marks\":[],\"text\":\"While you never want to hit a serious incident, we’re serious about making the most of the opportunity when you do.\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"8c9d14228b47\",\"_type\":\"block\",\"children\":[{\"_key\":\"8c9d14228b470\",\"_type\":\"span\",\"marks\":[],\"text\":\"In this instance, we took advantage of the opportunity to learn and socialise knowledge internally about:\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"c9e96a4fff17\",\"_type\":\"block\",\"children\":[{\"_key\":\"c9e96a4fff170\",\"_type\":\"span\",\"marks\":[],\"text\":\"Heroku Dyno restart policies, and how to manually reboot if they go wrong, for those outside of the primary on-call team who hadn’t yet encountered this.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"8a2372e1ae6b\",\"_type\":\"block\",\"children\":[{\"_key\":\"8a2372e1ae6b0\",\"_type\":\"span\",\"marks\":[],\"text\":\"An easy-to-miss edge-case for our static tooling where a third-party library could run our code in an uncaught goroutine.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"d2bc5cd4b6ad\",\"_type\":\"block\",\"children\":[{\"_key\":\"d2bc5cd4b6ad0\",\"_type\":\"span\",\"marks\":[],\"text\":\"Best-practice incident comms for our current organisation, given we’ve grown significantly since our last major incident.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"0ed5ef5cb0b7\",\"_type\":\"block\",\"children\":[{\"_key\":\"0ed5ef5cb0b70\",\"_type\":\"span\",\"marks\":[],\"text\":\"And after implementing incident follow-ups, we:\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"45db18ed9e1b\",\"_type\":\"block\",\"children\":[{\"_key\":\"45db18ed9e1b0\",\"_type\":\"span\",\"marks\":[],\"text\":\"Physically split our application by work-type, which will increase reliability and performance.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"1a524d746c48\",\"_type\":\"block\",\"children\":[{\"_key\":\"1a524d746c480\",\"_type\":\"span\",\"marks\":[],\"text\":\"Improved our event system by ensuring we catch panics.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"dd60091cbd40\",\"_type\":\"block\",\"children\":[{\"_key\":\"dd60091cbd400\",\"_type\":\"span\",\"marks\":[],\"text\":\"In addition, aside from eveything else we learned, we're happy that:\"}],\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"5f10d7b53efd\",\"_type\":\"block\",\"children\":[{\"_key\":\"5f10d7b53efd0\",\"_type\":\"span\",\"marks\":[],\"text\":\"Outside of Heroku monitoring gaps, we found existing dashboards around Pub/Sub subscription and worker performance to be extremely valuable.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"d143f2c5a319\",\"_type\":\"block\",\"children\":[{\"_key\":\"d143f2c5a3190\",\"_type\":\"span\",\"marks\":[],\"text\":\"Limited downtime to 13m, when it could have been much longer.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"},{\"_key\":\"1ab3f9f425cb\",\"_type\":\"block\",\"children\":[{\"_key\":\"1ab3f9f425cb0\",\"_type\":\"span\",\"marks\":[],\"text\":\"Kept our customers in the loop as the incident progressed, which we care a lot about.\"}],\"level\":1,\"listItem\":\"bullet\",\"markDefs\":[],\"style\":\"normal\"}],\"category\":{\"id\":\"engineering\",\"name\":\"Engineering\"},\"date\":\"2022-11-30T00:00:00.000Z\",\"description\":\"This is a technical write-up of an incident on Friday 18th November 2022 where we experienced 13 minutes of downtime from intermittent crashes.\",\"faqs\":{\"title\":\"FAQs\",\"items\":[]},\"heroImg\":\"https://cdn.sanity.io/images/oqy5aexb/production/ee5256258d999388726fa8f6dba1dc2092c3d90c-4800x1874.png?w=2000\u0026fit=max\u0026auto=format\",\"heroImgAlt\":null,\"heroImgCreditName\":null,\"heroImgCreditUrl\":\"\",\"isFeatured\":null,\"isHeroImgHidden\":null,\"isHiddenFromLists\":null,\"mediaBox\":\"$undefined\",\"ogCardStyle\":null,\"ogDescription\":null,\"ogImage\":\"\",\"ogTitle\":null,\"priority\":50,\"readingTime\":{\"minutes\":14,\"text\":\"14 min read\"},\"related\":[{\"_id\":\"59bb806c-b2d4-4e43-ac1a-ce346a36d81d\",\"_type\":\"blog\",\"authors\":[{\"avatar\":{\"_type\":\"image\",\"asset\":{\"_ref\":\"image-1fa50382fde41588c782aac0b86b8a96df63e21b-4672x7008-jpg\",\"_type\":\"reference\"},\"crop\":{\"_type\":\"sanity.imageCrop\",\"bottom\":0.42250378030339986,\"left\":0.13922187103602673,\"right\":0.14706922127573965,\"top\":0},\"hotspot\":{\"_type\":\"sanity.imageHotspot\",\"height\":0.5774962196966001,\"width\":0.7137089076882337,\"x\":0.4960763248801435,\"y\":0.28874810984830007}},\"bio\":null,\"id\":\"anthony-oparaocha\",\"name\":\"Anthony Oparaocha\",\"title\":\"Product Engineer Intern\"}],\"category\":{\"id\":\"engineering\",\"name\":\"Engineering\"},\"date\":\"2026-09-02T10:33:13.211Z\",\"description\":\"Our rate limiter depends on Valkey. If Valkey goes down we fail open and stop limiting which isn't good enough for our platform. As an intern, I built per-pod in-memory top-k buffers so we keep rate limiting even with the backing store gone.\",\"heroImg\":\"https://cdn.sanity.io/images/oqy5aexb/production/4208ab8bacb5b3cde27465ad8764fa7cd5e30895-2400x1285.jpg?w=2000\u0026fit=max\u0026auto=format\",\"heroImgAlt\":null,\"heroImgCreditName\":null,\"heroImgCreditUrl\":null,\"isFeatured\":false,\"isHeroImgHidden\":false,\"isHiddenFromLists\":false,\"ogCardStyle\":\"none\",\"ogDescription\":\" Our rate limiter depends on Valkey. If Valkey goes down we fail open and stop limiting which isn't good enough for our platform. As an intern, I built per-pod in-memory top-k buffers so we keep rate limiting even with the backing store gone.\",\"ogImage\":\"https://cdn.sanity.io/images/oqy5aexb/production/4208ab8bacb5b3cde27465ad8764fa7cd5e30895-2400x1285.jpg?w=2000\u0026fit=max\u0026auto=format\",\"ogTitle\":\"Interning at incident.io: Rate Limiting, Resiliently\",\"priority\":50,\"slug\":\"/blog/interning-at-incident-io-rate-limiting-resiliently\",\"thumbImg\":\"https://cdn.sanity.io/images/oqy5aexb/production/4208ab8bacb5b3cde27465ad8764fa7cd5e30895-2400x1285.jpg?w=2000\u0026fit=max\u0026auto=format\",\"thumbImgAlt\":null,\"title\":\"Interning at incident.io: rate limiting, resiliently\",\"updatedAt\":\"2026-09-08T16:58:29Z\",\"url\":\"https://incident.io/blog/interning-at-incident-io-rate-limiting-resiliently\",\"postPath\":\"/blog/interning-at-incident-io-rate-limiting-resiliently\",\"body\":[],\"canonicalSlug\":\"/blog/interning-at-incident-io-rate-limiting-resiliently\",\"headings\":[],\"humanDate\":\"September 2, 2026\",\"machineDate\":\"2026-02-09\",\"readingTime\":{\"minutes\":\"$undefined\",\"text\":\"undefined min read\"},\"faqs\":{\"title\":\"FAQs\",\"items\":[]},\"mediaBox\":\"$undefined\",\"related\":\"$undefined\",\"relatedHub\":\"$undefined\",\"isDebriefPodcastPost\":false},{\"_id\":\"f3ee3736-294b-4170-a14c-04abb899cc18\",\"_type\":\"blog\",\"authors\":[{\"avatar\":{\"_type\":\"image\",\"asset\":{\"_ref\":\"image-8dbc717422b0562befb8ccc5c9399be570787e9f-1369x1369-png\",\"_type\":\"reference\"}},\"bio\":null,\"id\":\"patrick-hamann\",\"name\":\"Patrick Hamann\",\"title\":\"Product Engineer\"},{\"avatar\":{\"_type\":\"image\",\"asset\":{\"_ref\":\"image-03534500cc455a88eb475ec447557953e2db55c7-512x512-jpg\",\"_type\":\"reference\"}},\"bio\":null,\"id\":\"mike-fisher\",\"name\":\"Mike Fisher\",\"title\":\"Product Engineer\"}],\"category\":{\"id\":\"engineering\",\"name\":\"Engineering\"},\"date\":\"2026-08-11T13:56:40.924Z\",\"description\":\"Our entire event-driven platform ran through a single message broker, which made it a single point of failure. So we added a second one. \\n\\nThis is the story of building an event load balancer, the queuing theory behind it, and the final chaos test where we turned off Pub/Sub in production and nobody noticed.\",\"heroImg\":\"https://cdn.sanity.io/images/oqy5aexb/production/02b1cabbed39c62eed6a6372f44b9b11b416c4f4-2400x1285.jpg?w=2000\u0026fit=max\u0026auto=format\",\"heroImgAlt\":\"We turned off Pub/Sub and no one noticed\",\"heroImgCreditName\":null,\"heroImgCreditUrl\":null,\"isFeatured\":false,\"isHeroImgHidden\":false,\"isHiddenFromLists\":false,\"ogCardStyle\":\"none\",\"ogDescription\":\"Our entire event-driven platform ran through a single message broker, which made it a single point of failure. So we added a second one. This is the story of building an event load balancer, the queueing theory behind it, and the chaos test where we turned off Pub/Sub in production and nobody noticed.\",\"ogImage\":\"https://cdn.sanity.io/images/oqy5aexb/production/02b1cabbed39c62eed6a6372f44b9b11b416c4f4-2400x1285.jpg?w=2000\u0026fit=max\u0026auto=format\",\"ogTitle\":\"We turned off Pub/Sub and nobody noticed\",\"priority\":50,\"slug\":\"/blog/we-turned-off-pub-sub-and-nobody-noticed\",\"thumbImg\":\"https://cdn.sanity.io/images/oqy5aexb/production/02b1cabbed39c62eed6a6372f44b9b11b416c4f4-2400x1285.jpg?w=2000\u0026fit=max\u0026auto=format\",\"thumbImgAlt\":null,\"title\":\"We turned off Pub/Sub and nobody noticed\",\"updatedAt\":\"2026-08-13T15:51:13Z\",\"url\":\"https://incident.io/blog/we-turned-off-pub-sub-and-nobody-noticed\",\"postPath\":\"/blog/we-turned-off-pub-sub-and-nobody-noticed\",\"body\":[],\"canonicalSlug\":\"/blog/we-turned-off-pub-sub-and-nobody-noticed\",\"headings\":[],\"humanDate\":\"August 11, 2026\",\"machineDate\":\"2026-11-08\",\"readingTime\":{\"minutes\":\"$undefined\",\"text\":\"undefined min read\"},\"faqs\":{\"title\":\"FAQs\",\"items\":[]},\"mediaBox\":\"$undefined\",\"related\":\"$undefined\",\"relatedHub\":\"$undefined\",\"isDebriefPodcastPost\":false},{\"_id\":\"c2f628cf-31ed-4847-9b2e-35efdafa9f58\",\"_type\":\"blog\",\"authors\":[{\"avatar\":{\"_type\":\"image\",\"asset\":{\"_ref\":\"image-f709cea28f7dc0c0a60e669c17c07edf7f72143f-2435x3328-jpg\",\"_type\":\"reference\"},\"crop\":{\"_type\":\"sanity.imageCrop\",\"bottom\":0,\"left\":0,\"right\":0,\"top\":0},\"hotspot\":{\"_type\":\"sanity.imageHotspot\",\"height\":1,\"width\":1,\"x\":0.5,\"y\":0.5}},\"bio\":null,\"id\":\"johanna-larsson\",\"name\":\"Johanna Larsson\",\"title\":\"Product Engineer\"}],\"category\":{\"id\":\"engineering\",\"name\":\"Engineering\"},\"date\":\"2026-07-21T11:00:45.227Z\",\"description\":\"Our learnings from implementing a product-wide read replica migrations, including some useful patterns for routing queries to replica and primary\",\"heroImg\":\"https://cdn.sanity.io/images/oqy5aexb/production/77427469f4daa51f1385e3d50b13a277e1fe26b9-2400x1285.png?w=2000\u0026fit=max\u0026auto=format\",\"heroImgAlt\":null,\"heroImgCreditName\":null,\"heroImgCreditUrl\":null,\"isFeatured\":false,\"isHeroImgHidden\":false,\"isHiddenFromLists\":false,\"ogCardStyle\":\"none\",\"ogDescription\":null,\"ogImage\":\"\",\"ogTitle\":null,\"priority\":50,\"slug\":\"/blog/dont-add-a-read-replica-until-youve-read-this\",\"thumbImg\":\"https://cdn.sanity.io/images/oqy5aexb/production/77427469f4daa51f1385e3d50b13a277e1fe26b9-2400x1285.png?w=2000\u0026fit=max\u0026auto=format\",\"thumbImgAlt\":null,\"title\":\"Don't add a read replica until you've read this\",\"updatedAt\":\"2026-09-03T14:54:42Z\",\"url\":\"https://incident.io/blog/dont-add-a-read-replica-until-youve-read-this\",\"postPath\":\"/blog/dont-add-a-read-replica-until-youve-read-this\",\"body\":[],\"canonicalSlug\":\"/blog/dont-add-a-read-replica-until-youve-read-this\",\"headings\":[],\"humanDate\":\"July 21, 2026\",\"machineDate\":\"2026-21-07\",\"readingTime\":{\"minutes\":\"$undefined\",\"text\":\"undefined min read\"},\"faqs\":{\"title\":\"FAQs\",\"items\":[]},\"mediaBox\":\"$undefined\",\"related\":\"$undefined\",\"relatedHub\":\"$undefined\",\"isDebriefPodcastPost\":false}],\"relatedHub\":null,\"slug\":\"/blog/intermittent-downtime\",\"thumbImg\":\"https://cdn.sanity.io/images/oqy5aexb/production/156896b74c8d43afc6d4064e63b008ba079bc943-2482x1696.png?w=2000\u0026fit=max\u0026auto=format\",\"thumbImgAlt\":null,\"title\":\"Intermittent downtime from repeated crashes\",\"updatedAt\":\"2023-06-23T08:30:00Z\",\"url\":\"https://incident.io/blog/intermittent-downtime\",\"postPath\":\"/blog/intermittent-downtime\",\"canonicalSlug\":\"/blog/intermittent-downtime\",\"headings\":[{\"id\":\"background\",\"value\":\"Background\",\"depth\":2},{\"id\":\"initial-crash\",\"value\":\"Initial crash\",\"depth\":2},{\"id\":\"subsequent-crashes\",\"value\":\"Subsequent crashes\",\"depth\":2},{\"id\":\"regroup\",\"value\":\"Regroup\",\"depth\":2},{\"id\":\"mitigation-1-ensuring-we-handle-message-panics\",\"value\":\"Mitigation 1: Ensuring we handle message panics\",\"depth\":2},{\"id\":\"mitigation-2-separate-our-app-by-work-type\",\"value\":\"Mitigation 2: Separate our app by work type\",\"depth\":2},{\"id\":\"conclusion\",\"value\":\"Conclusion\",\"depth\":2}],\"humanDate\":\"November 30, 2022\",\"machineDate\":\"2022-30-11\",\"isDebriefPodcastPost\":false},\"otherPosts\":[\"$5:1:2:props:children:1:0:props:post:related:0\",\"$5:1:2:props:children:1:0:props:post:related:1\",\"$5:1:2:props:children:1:0:props:post:related:2\"],\"otherPostsTitle\":\"See related articles\"}],\"$L2e\",\"$L2f\",\"$L30\"]]}],\"$L31\"]]\n15:null\n19:[[\"$\",\"title\",\"0\",{\"children\":\"Intermittent downtime from repeated crashes | Blog | incident.io\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"This is a technical write-up of an incident on Friday 18th November 2022 where we experienced 13 minutes of downtime from intermittent crashes.\"}],[\"$\",\"meta\",\"2\",{\"name\":\"keywords\",\"content\":\"incident,incident management,incident response,post mortem,outage,slack incident,incident channel\"}],[\"$\",\"meta\",\"3\",{\"name\":\"slack-app-id\",\"content\":\"A01DEGPUHHC\"}],[\"$\",\"link\",\"4\",{\"rel\":\"canonical\",\"href\":\"https://incident.io/blog/intermittent-downtime\"}],[\"$\",\"link\",\"5\",{\"rel\":\"alternate\",\"type\":\"text/markdown\",\"href\":\"https://incident.io/blog/intermittent-downtime.md\"}],[\"$\",\"meta\",\"6\",{\"property\":\"og:title\",\"content\":\"Intermittent downtime from repeated crashes | Blog | incident.io\"}],[\"$\",\"meta\",\"7\",{\"property\":\"og:description\",\"content\":\"This is a technical write-up of an incident on Friday 18th November 2022 where we experienced 13 minutes of downtime from intermittent crashes.\"}],[\"$\",\"meta\",\"8\",{\"property\":\"og:image\",\"content\":\"https://incident.io/socialcard/blog/intermittent-downtime?1687509000000\"}],[\"$\",\"meta\",\"9\",{\"property\":\"og:image:width\",\"content\":\"1200\"}],[\"$\",\"meta\",\"10\",{\"property\":\"og:image:height\",\"content\":\"628\"}],[\"$\",\"meta\",\"11\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"12\",{\"name\":\"twitter:site\",\"content\":\"@incident_io\"}],[\"$\",\"meta\",\"13\",{\"name\":\"twitter:creator\",\"content\":\"@incident_io\"}],[\"$\",\"meta\",\"14\",{\"name\":\"twitter:title\",\"content\":\"Intermittent downtime from repeated crashes | Blog\"}],[\"$\",\"meta\",\"15\",{\"name\":\"twitter:description\",\"content\":\"This is a technical write-up of an incident on Friday 18th November 2022 where we experienced 13 minutes of downtime from intermittent crashes.\"}],[\"$\",\"meta\",\"16\",{\"name\":\"twitter:image\",\"content\":\"https://incident.io/socialcard/blog/intermittent-downtime?1687509000000\"}],[\"$\",\"meta\",\"17\",{\"name\":\"twitter:image:width\",\"content\":\"1200\"}],[\"$\",\"meta\",\"18\",{\"name\":\"twitter:image:height\",\"content\":\"628\"}],[\"$\",\"link\",\"19\",{\"rel\":\"icon\",\"href\":\"/icons/icon-48x48.png\"}],[\"$\",\"link\",\"20\",{\"rel\":\"apple-touch-icon\",\"href\":\"/icons/icon-48x48.png\",\"sizes\":\"48x48\"}],[\"$\",\"link\",\"21\",{\"rel\":\"apple-touch-icon\",\"href\":\"/icons/icon-72x72.png\",\"sizes\":\"72x72\"}],[\"$\",\"link\",\"22\",{\"rel\":\"apple-touch-icon\",\"href\":\"/icons/icon-96x96.png\",\"sizes\":\"96x96\"}],[\"$\",\"link\",\"23\",{\"rel\":\"apple-touch-icon\",\"href\":\"/icons/icon-144x144.png\",\"sizes\":\"144x144\"}],[\"$\",\"link\",\"24\",{\"rel\":\"apple-touch-icon\",\"href\":\"/icons/icon-192x192.png\",\"sizes\":\"192x192\"}],[\"$\",\"link\",\"25\",{\"rel\":\"apple-touch-icon\",\"href\":\"/icons/icon-256x256.png\",\"sizes\":\"256x256\"}],[\"$\",\"link\",\"26\",{\"rel\":\"apple-touch-icon\",\"href\":\"/icons/icon-384x384.png\",\"sizes\":\"384x384\"}],[\"$\",\"link\",\"27\",{\"rel\":\"apple-touch-icon\",\"href\":\"/icons/icon-512x512.png\",\"sizes\":\"512x512\"}],[\"$\",\"$L32\",\"28\",{}]]\n"])</script><script>self.__next_f.push([1,"33:I[479520,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\",\"/_next/static/immutable/chunks/39pfb5kk8her1.js\",\"/_next/static/immutable/chunks/3qz9rwp_7jbnb.js\",\"/_next/static/immutable/chunks/1my_9vnng6gxj.js\",\"/_next/static/immutable/chunks/040yti440pn1b.js\",\"/_next/static/immutable/chunks/0v4j5g1ppvtg_.js\",\"/_next/static/immutable/chunks/3buml8s4fdaii.js\",\"/_next/static/immutable/chunks/2_frah5pspzdw.js\",\"/_next/static/immutable/chunks/1mcb8cvy5_n5b.js\",\"/_next/static/immutable/chunks/132szer1wwfxa.js\",\"/_next/static/immutable/chunks/2jjhnej_crwty.js\",\"/_next/static/immutable/chunks/19t-07qzgwjic.js\",\"/_next/static/immutable/chunks/1wpznbhwql3dd.js\",\"/_next/static/immutable/chunks/20izjjpsu0ag8.js\"],\"\"]\n34:I[388027,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\",\"/_next/static/immutable/chunks/39pfb5kk8her1.js\",\"/_next/static/immutable/chunks/3qz9rwp_7jbnb.js\",\"/_next/static/immutable/chunks/1my_9vnng6gxj.js\",\"/_next/static/immutable/chunks/040yti440pn1b.js\",\"/_next/static/immutable/chunks/0v4j5g1ppvtg_.js\",\"/_next/static/immutable/chunks/3buml8s4fdaii.js\",\"/_next/static/immutable/chunks/2_frah5pspzdw.js\",\"/_next/static/immutable/chunks/1mcb8cvy5_n5b.js\",\"/_next/static/immutable/chunks/132szer1wwfxa.js\",\"/_next/static/immutable/chunks/2jjhnej_crwty.js\",\"/_next/static/immutable/chunks/19t-07qzgwjic.js\",\"/_next/static/immutable/chunks/1wpznbhwql3dd.js\",\"/_next/static/immutable/chunks/20izjjpsu0ag8.js\"],\"ReadingProgressBar\"]\n35:I[522016,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\",\"/_next/static/immutable/chunks/39pfb5kk8her1.js\",\"/_next/static/immutable/chunks/3qz9rwp_7jbnb.js\",\"/_next/static/immutable/chunks/1my_9vnng6gxj.js\",\"/_next/static/immutable/chunks/040yti440pn1b.js\",\"/_next/static/immutable/chunks/0v4j5g1ppvtg_.js\",\"/_next/static/immutable/chunks/3buml8s4fdaii.js\",\"/_next/static/immutable/chunks/2_frah5pspzdw.js\",\"/_next/static/immutable/chunks/1mcb8cvy5_n5b.js\",\"/_next/static/immutable/chunks/132szer1wwfxa.js\",\"/_next/static/immutable/chunks/2jjhnej_crwty.js\",\"/_next/static/immutable/chunks/19t-07qzgwjic.js\",\"/_next/static/immutable/chunks/1wpznbhwql3dd.js\",\"/_next/static/immutable/chunks/20izjjpsu0ag8.js\"],\"\"]\n36:I[415697,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\",\"/_next/static/immutable/chunks/39pfb5kk8her1.js\",\"/_next/static/immutable/chunks/3qz9rwp_7jbnb.js\",\"/_next/static/immutable/chunks/1my_9vnng6gxj.js\",\"/_next/static/immutable/chunks/040yti440pn1b.js\",\"/_next/static/immutable/chunks/0v4j5g1ppvtg_.js\",\"/_next/static/immutable/chunks/3buml8s4fdaii.js\",\"/_next/static/immutable/chunks/2_frah5pspzdw.js\",\"/_next/static/immutable/chunks/1mcb8cvy5_n5b.js\",\"/_next/static/immutable/chunks/132szer1wwfxa.js\",\"/_next/static/immutable/chunks/2jjhnej_crwty.js\",\"/_next/static/immutable/chunks/19t-07qzgwjic.js\",\"/_next/static/immutable/chunks/1wpznbhwql3dd.js\",\"/_next/static/immutable/chunks/20izjjpsu0ag8.js\"],\"default\"]\n37:I[466105,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\",\"/_next/static/immutable/chunks/39pfb5kk8her1.js\",\"/_next/static/immutable/chunks/3qz9rwp_7jbnb.js\",\"/_next/static/immutable/chunks/1my_9vnng6gxj.js\",\"/_next/static/immutable/chunks/040yti440pn1b.js\",\"/_next/static/immutable/chunks/0v4j5g1ppvtg_.js\",\"/_next/static/immutable/chunks/3buml8s4fdaii.js\",\"/_next/static/immutable/chunks/2_frah5pspzdw.js\",\"/_next/static/immutable/chunks/1mcb8cvy5_n5b.js\",\"/_next/static/immutable/chunks/132szer1wwfxa.js\",\"/_next/static/immutable/chunks/2jjhnej_crwty.js\",\"/_next/static/immutable/chunks/19t-07qzgwjic.js\",\"/_next/static/immutable/chunks/1wpznbhwql3dd.js\",\"/_next/static/immutable/chunks/20izjjpsu0ag8.js\"],\"AuthorsBar\"]\n2e:[\"$\",\"hr\",null,{\"className\":\"h-px w-full border-stroke-primary dark:border-stroke-invert max-w-[1192px] mx-auto\"}]\n30:[\"$\",\"$L28\",null,{\"useLegacyWidth\":false}]\n31:[\"$\",\"$L33\",null,{\"type\":\"text/javascript\",\"src\":\"https://platform.twitter.com/widgets.js\",\"async\":true,\"strategy\":\"lazyOnload\"}]\n2b:[\"$\",\"$L2a\",null,{\"className\":\"$undefined\",\"theme\":\"light\",\"stickyNavTheme\":\"dark\",\"stickyNavClassName\":\"$undefined\",\"stickyNavAfterSlot\":[\"$\",\"$L34\",null,{\"className\":\"absolute -bottom-2\"}],\"hasBlurredBackground\":\"$undefined\",\"showSignupButtonOnInitialLoad\":\"$undefined\",\"showSignupButtonInStickyNav\":\"$undefined\",\"showNavContent\":\"$undefined\",\"showCtas\":[\"request-demo\",\"login\"],\"children\":\"$undefined\",\"mobileChildren\":\"$undefined\",\"initiallyTransparent\":\"$undefined\",\"switchToLightOnScroll\":\"$undefined\",\"customCtas\":\"$undefined\",\"demoHref\":\"$undefined\"}]\n23:[\"$\",\"section\",null,{\"ref\":\"$undefined\",\"id\":\"you-may-be-interested-in\",\"className\":\"flex flex-col gap-10 [\u0026_h2]:text-h3\",\"children\":[[\"$\",\"header\",null,{\"className\":\"flex flex-col gap-3\",\"children\":[[\"$\",\"h2\",null,{\"className\":\"text-h2 text-balance\",\"children\":\"You may be interested in\"}],\"$undefined\"]}],[\"$\",\"div\",null,{\"className\":\"grid gap-4 grid-cols-1 lg:grid-cols-3\",\"children\":[[\"$\",\"$L35\",\"Interning at incident.io: rate limiting, resiliently\",{\"id\":\"59bb806c-b2d4-4e43-ac1a-ce346a36d81d\",\"href\":\"/blog/interning-at-incident-io-rate-limiting-resiliently\",\"className\":\"group flex flex-col\",\"children\":[[\"$\",\"div\",null,{\"className\":\"bg-surface-tertiary rounded-lg mb-3 md:mb-6 p-0 relative aspect-video\",\"children\":[\"$\",\"$L36\",null,{\"src\":\"https://cdn.sanity.io/images/oqy5aexb/production/4208ab8bacb5b3cde27465ad8764fa7cd5e30895-2400x1285.jpg?w=2000\u0026fit=max\u0026auto=format\",\"alt\":\"Interning at incident.io: rate limiting, resiliently\",\"fit\":\"cover\",\"className\":\"w-full overflow-hidden transition-all ease-in-out duration-300 mx-auto absolute inset-0 group-hover:w-[calc(100%-16px)] group-hover:py-2 [\u0026_img]:transition-all [\u0026_img]:rounded-lg group-hover:[\u0026_img]:rounded\"}]}],[\"$\",\"div\",null,{\"className\":\"max-w-[546px] grow flex flex-col gap-2\",\"children\":[false,false,[\"$\",\"h3\",null,{\"className\":\"text-h4 text-pretty\",\"children\":\"Interning at incident.io: rate limiting, resiliently\"}],false,[\"$\",\"div\",null,{\"className\":\"flex flex-wrap items-center gap-x-4 gap-y-1\",\"children\":[[\"$\",\"$L37\",null,{\"authors\":[{\"avatar\":{\"_type\":\"image\",\"asset\":{\"_ref\":\"image-1fa50382fde41588c782aac0b86b8a96df63e21b-4672x7008-jpg\",\"_type\":\"reference\"},\"crop\":{\"_type\":\"sanity.imageCrop\",\"bottom\":0.42250378030339986,\"left\":0.13922187103602673,\"right\":0.14706922127573965,\"top\":0},\"hotspot\":{\"_type\":\"sanity.imageHotspot\",\"height\":0.5774962196966001,\"width\":0.7137089076882337,\"x\":0.4960763248801435,\"y\":0.28874810984830007}},\"bio\":null,\"id\":\"anthony-oparaocha\",\"name\":\"Anthony Oparaocha\",\"title\":\"Product Engineer Intern\"}],\"showLinks\":false}],[\"$\",\"div\",null,{\"className\":\"text-body-sm text-content-secondary\",\"children\":\"September 2, 2026\"}]]}]]}]]}],[\"$\",\"$L35\",\"We turned off Pub/Sub and nobody noticed\",{\"id\":\"f3ee3736-294b-4170-a14c-04abb899cc18\",\"href\":\"/blog/we-turned-off-pub-sub-and-nobody-noticed\",\"className\":\"group flex flex-col\",\"children\":[[\"$\",\"div\",null,{\"className\":\"bg-surface-tertiary rounded-lg mb-3 md:mb-6 p-0 relative aspect-video\",\"children\":[\"$\",\"$L36\",null,{\"src\":\"https://cdn.sanity.io/images/oqy5aexb/production/02b1cabbed39c62eed6a6372f44b9b11b416c4f4-2400x1285.jpg?w=2000\u0026fit=max\u0026auto=format\",\"alt\":\"We turned off Pub/Sub and nobody noticed\",\"fit\":\"cover\",\"className\":\"w-full overflow-hidden transition-all ease-in-out duration-300 mx-auto absolute inset-0 group-hover:w-[calc(100%-16px)] group-hover:py-2 [\u0026_img]:transition-all [\u0026_img]:rounded-lg group-hover:[\u0026_img]:rounded\"}]}],[\"$\",\"div\",null,{\"className\":\"max-w-[546px] grow flex flex-col gap-2\",\"children\":[false,false,[\"$\",\"h3\",null,{\"className\":\"text-h4 text-pretty\",\"children\":\"We turned off Pub/Sub and nobody noticed\"}],false,[\"$\",\"div\",null,{\"className\":\"flex flex-wrap items-center gap-x-4 gap-y-1\",\"children\":[[\"$\",\"$L37\",null,{\"authors\":[{\"avatar\":{\"_type\":\"image\",\"asset\":{\"_ref\":\"image-8dbc717422b0562befb8ccc5c9399be570787e9f-1369x1369-png\",\"_type\":\"reference\"}},\"bio\":null,\"id\":\"patrick-hamann\",\"name\":\"Patrick Hamann\",\"title\":\"Product Engineer\"},{\"avatar\":{\"_type\":\"image\",\"asset\":{\"_ref\":\"image-03534500cc455a88eb475ec447557953e2db55c7-512x512-jpg\",\"_type\":\"reference\"}},\"bio\":null,\"id\":\"mike-fisher\",\"name\":\"Mike Fisher\",\"title\":\"Product Engineer\"}],\"showLinks\":false}],[\"$\",\"div\",null,{\"className\":\"text-body-sm text-content-secondary\",\"children\":\"August 11, 2026\"}]]}]]}]]}],[\"$\",\"$L35\",\"Introducing Investigations, powered by Nexus.\",{\"id\":\"96c39cdc-e0c8-4ab5-833a-a0b3ddcfb262\",\"href\":\"/blog/introducing-investigations-powered-by-nexus\",\"className\":\"group flex flex-col\",\"children\":[[\"$\",\"div\",null,{\"className\":\"bg-surface-tertiary rounded-lg mb-3 md:mb-6 p-0 relative aspect-video\",\"children\":[\"$\",\"$L36\",null,{\"src\":\"https://cdn.sanity.io/images/oqy5aexb/production/fe5cc8a5e40649335c29265f6d423361338262f8-2400x1285.png?w=2000\u0026fit=max\u0026auto=format\",\"alt\":\"Introducing Investigations, powered by Nexus.\",\"fit\":\"cover\",\"className\":\"w-full overflow-hidden transition-all ease-in-out duration-300 mx-auto absolute inset-0 group-hover:w-[calc(100%-16px)] group-hover:py-2 [\u0026_img]:transition-all [\u0026_img]:rounded-lg group-hover:[\u0026_img]:rounded\"}]}],\"$L38\"]}]]}]]}]\n"])</script><script>self.__next_f.push([1,"38:[\"$\",\"div\",null,{\"className\":\"max-w-[546px] grow flex flex-col gap-2\",\"children\":[false,false,[\"$\",\"h3\",null,{\"className\":\"text-h4 text-pretty\",\"children\":\"Introducing Investigations, powered by Nexus.\"}],false,[\"$\",\"div\",null,{\"className\":\"flex flex-wrap items-center gap-x-4 gap-y-1\",\"children\":[[\"$\",\"$L37\",null,{\"authors\":[{\"avatar\":{\"_type\":\"image\",\"asset\":{\"_ref\":\"image-4884ed999f884514b6fc1571af01407f7c78c286-512x512-png\",\"_type\":\"reference\"}},\"bio\":null,\"id\":\"pete\",\"name\":\"Pete Hamilton\",\"title\":\"Co-Founder \u0026 CTO\"}],\"showLinks\":false}],[\"$\",\"div\",null,{\"className\":\"text-body-sm text-content-secondary\",\"children\":\"August 5, 2026\"}]]}]]}]\n39:Tac7,M64.585 70.707c0-2.056-.082-3.562.024-5.051.103-1.436-.6-1.805-1.861-1.941a65 65 0 0 1-7.613-1.077c-3.706-.858-5.644-2.95-5.676-6.699-.058-6.805.239-13.615.449-20.422.058-1.908.982-3.522 2.753-4.233 2.345-.941 4.79-2.126 7.238-2.24 18.487-.86 36.98-1.648 55.481-2.117 11.857-.3 23.731-.09 35.595.054a57 57 0 0 1 9.753 1.07c4.438.833 6.258 2.847 6.862 7.275.842 6.162 1.19 12.354.082 18.52-1.066 5.937-3.995 8.578-10.004 9.166-1.849.18-3.713.194-5.57.283-2.641.128-2.717.233-2.193 3.277.811.066 1.674.146 2.538.205 1.855.127 3.719.177 5.566.373 7.408.788 9.507 2.64 10.162 10.009.495 5.104.571 10.24.228 15.357-.499 6.596-3.609 9.422-10.143 10.138-3.423.375-6.867.55-10.527.834l.252 4.773c.904.089 1.755.16 2.605.257 3.426.404 6.88.656 10.267 1.276 3.231.59 5.469 2.576 6.018 5.942 1.088 6.676 1.473 13.391.219 20.086-1.048 5.591-4.045 8.412-9.78 8.719-7.337.394-14.688.47-22.035.565-20.19.259-40.38.518-60.57.638-5.216.031-10.436-.425-15.652-.715a22.4 22.4 0 0 1-3.668-.575c-4.525-1.023-6.462-3.047-6.5-7.718-.053-6.454.272-12.911.478-19.365.068-2.122 1.05-3.869 3.03-4.626 2.92-1.121 5.95-1.953 9.486-3.083l.563-4.066c-2.14-.331-4.057-.545-5.937-.936-4.235-.88-6.258-2.95-6.376-7.261-.179-6.54-.028-13.09.036-19.637.02-2.116.97-3.888 2.945-4.679a57 57 0 0 1 7.496-2.328c1.145-.282 2.418-.048 3.98-.048m44.498 26.886v-.361c3.006 0 6.014.073 9.018-.013q18.975-.54 37.945-1.17c3.85-.129 4.805-.875 4.939-4.7.17-4.846-.075-9.708-.159-14.562-.03-1.658-.782-2.612-2.57-2.684-1.676-.068-3.343-.377-5.019-.44-20.993-.78-41.932.383-62.865 1.777q-16.68 1.11-33.377 1.992c-1.774.096-2.039 1-1.95 2.406.344 5.476.702 10.946 1.01 16.421.107 1.895 1.158 2.659 2.918 2.678 7.868.085 15.744.4 23.606.176 8.842-.255 17.67-.993 26.504-1.52m-3.848 17.2-.011.352c-6.008.319-12.016.691-18.03.941q-15.382.639-30.772 1.117c-1.843.053-2.273.911-2.207 2.482.212 5.126.382 10.256.603 15.382.13 3.064.388 3.552 3.583 3.706 7.596.368 15.206.751 22.809.65 15.132-.2 30.259-.767 45.39-1.095 9.469-.205 18.943-.217 28.412-.386 3.548-.064 4.589-.837 4.793-4.285.292-4.929.178-9.883.21-14.826.012-1.662-.765-2.628-2.522-2.769-1.939-.155-3.866-.581-5.804-.613-15.485-.257-30.969-.447-46.454-.656m.038-81.623-.009.412c-5.127.256-10.253.547-15.382.76-10.875.452-21.75.908-32.63 1.244-1.821.056-2.617.56-2.533 2.416.229 5.038.39 10.079.598 15.118.143 3.469.346 3.918 3.783 4.084 7.51.362 15.035.718 22.548.616 14.691-.2 29.376-.763 44.066-1.083 9.912-.217 19.828-.248 29.74-.416 3.677-.062 4.643-.783 4.884-4.472.305-4.664.291-9.357.207-14.036-.053-2.83-.368-3.002-3.272-3.453a31 31 0 0 0-4.219-.46q-23.889-.397-47.781-.73m34.918 70.226-71.115 2.452.699 4.303 70.15-2.374zM71.847 70.004l69.494-3.832q.182-.503.26-1.032-.016-.9-.156-1.79l-70.394.867c.192 1.914.334 3.403.5 4.891q.106.462.294.896z3a:T613,M86.987 89.137c-4.978.235-11.428.551-17.88.834-1.144.05-2.415.254-3.406-.151-.893-.366-1.921-1.399-2.067-2.275-.115-.69.869-2.023 1.64-2.298a17.7 17.7 0 0 1 5.194-.918q19.227-.75 38.46-1.35a21.3 21.3 0 0 1 3.698.25c1.445.208 2.544 1.01 2.609 2.555s-1.004 2.499-2.399 2.76c-1.821.35-3.669.54-5.524.567-6.282.067-12.565.026-20.325.026M153.008 88.199a3.404 3.404 0 0 1-3.32-3.416c.074-2.035 1.458-3.543 3.218-3.509a3.72 3.72 0 0 1 3.509 3.606 3.373 3.373 0 0 1-3.407 3.319M141.657 88.559a3.414 3.414 0 0 1-3.403-3.347c-.019-1.875 1.177-2.952 3.26-2.938 2.153.013 3.504 1.246 3.436 3.136a3.315 3.315 0 0 1-3.293 3.148M128.268 82.723a3.27 3.27 0 0 1 3.137 2.968 2.86 2.86 0 0 1-.907 2.118 2.86 2.86 0 0 1-2.176.758 2.95 2.95 0 0 1-2.801-2.94 2.74 2.74 0 0 1 .749-2.043 2.75 2.75 0 0 1 1.998-.861M87.676 130.021h-19.35q-1.591.05-3.179-.069c-1.452-.156-2.566-.86-2.632-2.465-.06-1.486.95-2.276 2.254-2.539a26 26 0 0 1 4.727-.599q19.49-.294 38.979-.466c1.14-.011 2.427-.064 3.382.422.962.489 2.188 1.612 2.206 2.481.019.879-1.14 2.071-2.079 2.6-.926.522-2.222.51-3.357.517-6.984.041-13.967.02-20.948.02zM151.787 123.145a3.65 3.65 0 0 1 3.615 3.514 3.375 3.375 0 0 1-3.115 3.372 3.38 3.38 0 0 1-3.551-3.2c-.06-1.981 1.311-3.637 3.051-3.686M137.269 126.814c-.026-1.853 1.215-2.999 3.224-2.977 2.127.025 3.489 1.271 3.435 3.146a3.283 3.283 0 0 1-3.26 3.148 3.4 3.4 0 0 1-3.399-3.317M127.531 129.834a2.91 2.91 0 0 1-3.004-2.959 2.8 2.8 0 0 1 .192-1.087 2.802 2.802 0 0 1 2.587-1.784 3.305 3.305 0 0 1 3.067 3.025 2.77 2.77 0 0 1-.828 2.003 2.776 2.776 0 0 1-2.014.8022f:[\"$\",\"section\",null,{\"className\":\"\",\"children\":[[\"$\",\"div\",null,{\"className\":\"mx-auto flex flex-col gap-3 items-center text-center mb-10\",\"children\":[[\"$\",\"svg\",null,{\"xmlns\":\"http://www.w3.org/2000/svg\",\"fill\":\"none\",\"viewBox\":\"0 0 246 151\",\"className\":\"w-[96px] sm:w-[246px]\",\"children\":[[\"$\",\"g\",null,{\"fill\":\"#E4D9C8\",\"clipPath\":\"url(#server-racks_svg__a)\",\"children\":[[\"$\",\"path\",null,{\"d\":\"$39\"}],[\"$\",\"path\",null,{\"d\":\"$3a\"}]]}],\"$L3b\",\"$L3c\",\"$L3d\"]}],\"$L3e\",\"$L3f\"]}],\"$L40\"]}]\n"])</script><script>self.__next_f.push([1,"44:I[605500,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\",\"/_next/static/immutable/chunks/39pfb5kk8her1.js\",\"/_next/static/immutable/chunks/3qz9rwp_7jbnb.js\",\"/_next/static/immutable/chunks/1my_9vnng6gxj.js\",\"/_next/static/immutable/chunks/040yti440pn1b.js\",\"/_next/static/immutable/chunks/0v4j5g1ppvtg_.js\",\"/_next/static/immutable/chunks/3buml8s4fdaii.js\",\"/_next/static/immutable/chunks/2_frah5pspzdw.js\",\"/_next/static/immutable/chunks/1mcb8cvy5_n5b.js\",\"/_next/static/immutable/chunks/132szer1wwfxa.js\",\"/_next/static/immutable/chunks/2jjhnej_crwty.js\",\"/_next/static/immutable/chunks/19t-07qzgwjic.js\",\"/_next/static/immutable/chunks/1wpznbhwql3dd.js\",\"/_next/static/immutable/chunks/20izjjpsu0ag8.js\"],\"Image\"]\n41:Tbe9,M155.218 63.963c1.78-1.028 3.043-1.853 4.386-2.505 1.294-.629 1.263-1.422.75-2.582a65 65 0 0 1-2.874-7.132c-1.109-3.639-.267-6.362 2.964-8.265 5.864-3.453 11.911-6.6 17.91-9.822 1.681-.903 3.541-.91 5.042.268 1.988 1.56 4.237 3.085 5.56 5.148 9.987 15.58 19.917 31.202 29.573 46.99 6.188 10.118 11.944 20.506 17.752 30.853a57 57 0 0 1 3.948 8.981c1.498 4.26.664 6.843-2.868 9.58-4.916 3.81-10.104 7.208-15.998 9.331-5.674 2.045-9.426.829-12.94-4.081-1.08-1.51-2.024-3.118-3.031-4.682-1.431-2.223-1.56-2.236-3.933-.261.348.736.71 1.523 1.091 2.301.818 1.67 1.706 3.309 2.461 5.007 3.02 6.809 2.465 9.553-3.588 13.804a94 94 0 0 1-13.186 7.877c-5.961 2.865-9.964 1.586-13.851-3.715-2.036-2.777-3.91-5.672-5.985-8.7-1.337.868-2.558 1.66-4.008 2.605.375.827.739 1.599 1.079 2.385 1.363 3.168 2.872 6.286 4.029 9.529 1.105 3.092.504 6.024-2.137 8.182-5.237 4.281-10.86 7.972-17.285 10.233-5.366 1.888-9.308.703-12.442-4.11-4.009-6.157-7.751-12.485-11.506-18.801-10.32-17.354-20.639-34.71-30.838-52.136-2.634-4.501-4.85-9.25-7.206-13.912a22.3 22.3 0 0 1-1.336-3.465c-1.377-4.43-.592-7.119 3.434-9.488 5.562-3.272 11.317-6.22 17.01-9.268 1.871-1.002 3.875-1.025 5.521.312 2.43 1.968 4.666 4.176 7.413 6.673l3.802-1.546c-.784-2.019-1.556-3.786-2.158-5.61-1.355-4.107-.574-6.894 3.1-9.152 5.575-3.424 11.324-6.57 17.024-9.787 1.843-1.04 3.853-1.105 5.525.211a57 57 0 0 1 5.764 5.328c.817.85 1.251 2.07 2.032 3.422m-1.036 51.979.313-.181c1.503 2.604 2.944 5.245 4.52 7.804a7868 7868 0 0 0 19.986 32.277c2.036 3.269 3.161 3.723 6.54 1.927 4.282-2.276 8.37-4.919 12.531-7.419 1.421-.855 1.872-1.983 1.04-3.568-.779-1.485-1.345-3.084-2.129-4.567-9.82-18.571-21.297-36.122-32.971-53.554a2405 2405 0 0 1-18.413-27.91c-.971-1.488-1.886-1.265-3.059-.485-4.571 3.036-9.129 6.081-13.716 9.086-1.588 1.04-1.724 2.331-.86 3.865 3.86 6.857 7.525 13.835 11.65 20.532 4.641 7.53 9.694 14.806 14.568 22.193m-16.819 5.268-.311.166c-3.28-5.043-6.606-10.06-9.83-15.144a2962 2962 0 0 1-16.352-26.09c-.968-1.57-1.926-1.513-3.254-.671-4.333 2.747-8.69 5.459-13.02 8.213-2.588 1.646-2.882 2.112-1.418 4.956 3.48 6.762 6.953 13.545 10.842 20.078 7.739 13.005 15.794 25.822 23.643 38.762 4.913 8.098 9.659 16.296 14.541 24.412 1.829 3.041 3.019 3.556 6.107 2.009 4.415-2.212 8.648-4.788 12.945-7.231 1.445-.821 1.893-1.977 1.137-3.569-.835-1.757-1.43-3.639-2.371-5.333-7.52-13.539-15.098-27.044-22.659-40.559zm70.707-40.779-.362.198c-2.785-4.312-5.6-8.605-8.349-12.941-5.829-9.193-11.661-18.383-17.392-27.636-.96-1.55-1.794-1.987-3.359-.986-4.249 2.717-8.534 5.377-12.794 8.077-2.932 1.858-3.22 2.259-1.645 5.318 3.442 6.685 6.896 13.38 10.741 19.835 7.518 12.623 15.349 25.06 22.971 37.621 5.143 8.476 10.128 17.047 15.23 25.548 1.892 3.153 3 3.629 6.315 1.993 4.191-2.068 8.249-4.427 12.259-6.839 2.424-1.461 2.415-1.819 1.354-4.56a31 31 0 0 0-1.711-3.883 18099 18099 0 0 0-23.258-41.745m-43.359 65.353-37.682-60.362-3.376 2.757 37.131 59.565zM159.457 69.9l38.066 58.268q.526-.094 1.023-.291a13 13 0 0 0 1.472-1.03l-35.947-60.53c-1.561 1.124-2.78 1.99-3.986 2.879q-.347.322-.628.70342:T638,M150.457 92.578c-2.692-4.194-6.191-9.622-9.662-15.068-.615-.966-1.428-1.964-1.573-3.025-.129-.957.251-2.363.937-2.927.541-.445 2.187-.26 2.811.271a17.7 17.7 0 0 1 3.391 4.039A4854 4854 0 0 1 166.76 108.5a21.3 21.3 0 0 1 1.634 3.328c.542 1.355.397 2.707-.909 3.536s-2.666.381-3.591-.696a32 32 0 0 1-3.252-4.501c-3.199-5.407-6.305-10.869-10.185-17.59M184.28 149.285a3.4 3.4 0 0 1 1.298-4.584c1.799-.952 3.797-.508 4.648 1.033a3.717 3.717 0 0 1-1.369 4.842 3.37 3.37 0 0 1-4.577-1.291M178.294 139.634a3.417 3.417 0 0 1 1.196-4.62c1.614-.953 3.145-.457 4.175 1.355 1.065 1.871.673 3.657-.999 4.543a3.31 3.31 0 0 1-4.372-1.278M176.653 125.122a3.27 3.27 0 0 1-1.002 4.201 2.85 2.85 0 0 1-2.288.273 2.86 2.86 0 0 1-1.744-1.506 2.95 2.95 0 0 1 1.145-3.896 2.76 2.76 0 0 1 3.15.098c.3.224.552.506.739.83M186.448 73.394l-9.544-16.53c-.532-.921-1.097-1.819-1.591-2.756-.721-1.368-.93-2.801.481-3.807 1.576-1.122 2.881-.221 3.899 1.011a32 32 0 0 1 2.794 3.866 6645 6645 0 0 1 18.701 31.457c.85 1.44 1.751 2.946 2.124 4.54a4.12 4.12 0 0 1-.831 3.278c-1.052 1.098-2.52.558-3.536-.47a14 14 0 0 1-2.086-2.716c-3.482-5.951-6.912-11.932-10.36-17.903zM218.587 129.118a3.33 3.33 0 0 1 1.119-4.637c1.726-1.001 3.831-.586 4.695.927a3.713 3.713 0 0 1-1.16 4.722 3.33 3.33 0 0 1-3.748-.054 3.3 3.3 0 0 1-.906-.958M212.767 119.233a3.34 3.34 0 0 1-.409-2.529 3.34 3.34 0 0 1 1.498-2.077c1.601-.985 3.196-.519 4.197 1.232 1.063 1.859.697 3.653-.93 4.565a3.235 3.235 0 0 1-4.356-1.191M206.259 107.429a2.89 2.89 0 0 1 1.472-3.95c1.466-.588 2.78-.347 3.604 1.132a3.167 3.167 0 0 1-1.339 4.267 2.836 2.836 0 0 1-3.737-1.4493b:[\"$\",\"g\",null,{\"fill\":\"#E4D9C8\",\"clipPath\":\"url(#server-racks_svg__b)\",\"children\":[[\"$\",\"path\",null,{\"d\":\"$41\"}],[\"$\",\"path\",null,{\"d\":\"$42\"}]]}]\n43:T42c,M153.387 37.918c-.642.435-1.268.644-2.099.39-2.077-.636-4.188-1.16-6.288-1.721-.362-.097-.733-.152-1.24-.254.017.398.018.702.046 1.003q.151.977.186 1.965c-.034 1.143-.887 1.686-1.964 1.291a6.6 6.6 0 0 1-1.13-.59 66 66 0 0 1-9.969-7.292c-.604-.544-1.137-1.133-1.196-1.976.684-1.128 1.694-1.179 2.767-.912 1.184.295 2.352.657 3.669 1.028-.194-1.042-.355-2.004-.558-2.957-.122-.57-.289-1.165.244-1.576.593-.458 1.26-.33 1.87.029 1.309.766 2.609 1.547 3.91 2.326 2.6 1.56 5.208 3.105 7.792 4.69q1.576.952 3.028 2.082c.41.324.634.883.941 1.335zM168.257 2.815c.757.17 1.338.483 1.721 1.263.958 1.95 2.021 3.847 3.05 5.762.177.33.392.638.664 1.078.278-.285.499-.494.699-.72q.608-.78 1.305-1.48c.858-.759 1.837-.509 2.286.547q.232.598.343 1.228a66 66 0 0 1 1.509 12.259c.016.812-.048 1.604-.622 2.224-1.29.273-2.019-.428-2.559-1.393-.596-1.065-1.132-2.164-1.763-3.378-.626.854-1.217 1.63-1.773 2.43-.332.479-.651 1.009-1.315.902-.741-.12-1.103-.694-1.26-1.383-.337-1.48-.658-2.961-.98-4.444-.643-2.962-1.302-5.921-1.915-8.89a31 31 0 0 1-.555-3.632c-.045-.521.21-1.067.329-1.6z3c:[\"$\",\"path\",null,{\"fill\":\"#E4D9C8\",\"d\":\"$43\"}]\n3d:[\"$\",\"defs\",null,{\"children\":[[\"$\",\"clipPath\",null,{\"id\":\"server-racks_svg__a\",\"children\":[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"M48.644 83.479c0-9.395 7.616-17.012 17.01-17.012h102.878v79.386H48.644z\"}]}],[\"$\",\"clipPath\",null,{\"id\":\"server-racks_svg__b\",\"children\":[\"$\",\"path\",null,{\"fill\":\"#fff\",\"d\":\"m185.411 28.395 59.945 103.826-32.271 18.632L153.14 47.026z\"}]}]]}]\n3e:[\"$\",\"h2\",null,{\"className\":\"text-h3 md:text-h2 text-balance max-w-[588px]\",\"children\":\"So good, you’ll break things on purpose\"}]\n3f:[\"$\",\"p\",null,{\"className\":\"text-body md:text-body-lg max-w-[588px]\",\"children\":\"Ready for modern incident management? Book a call with one of our experts today.\"}]\n40:[\"$\",\"div\",null,{\"className\":\"flex gap-4 flex-col lg:flex-row lg:gap-4\",\"children\":[[\"$\",\"div\",null,{\"className\":\"basis-1/2\",\"children\":[\"$\",\"$L44\",null,{\"src\":{\"src\":\"/_next/static/immutable/media/signup-image.1f5nktjcty_re.png\",\"width\":1176,\"height\":641,\"blurWidth\":8,\"blurHeight\":4,\"blurDataURL\":\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAECAYAAACzzX7wAAAAhUlEQVR42g3KSwrCMBQF0OzWiWIRTHFgoUXBQRWDKJRC16JWTfET07wkuJnHtcMDR5SF5Mko4UKmvF/O+Lie82mz4GqXca1yFnmaQJU1uvYOo2/4dC3e+oqXvuD5OEOs5BjNoYK3FtF72ECgGBD7Hr/BIpNTbFUDY74gR7BDJCI45xBCwB+DRGEIc5vtKwAAAABJRU5ErkJggg==\"},\"alt\":\"Signup image\",\"className\":\"w-full h-full object-cover rounded-lg\"}]}],[\"$\",\"div\",null,{\"className\":\"basis-1/2 flex flex-col gap-6 bg-surface-secondary rounded-lg p-6\",\"children\":[[\"$\",\"div\",null,{\"className\":\"grow\",\"children\":[[\"$\",\"h3\",null,{\"className\":\"text-body-emphasis mb-2\",\"children\":\"We’d love to talk to you about\"}],[\"$\",\"ul\",null,{\"className\":\"flex flex-col gap-2 text-body-serif grow\",\"children\":[[\"$\",\"li\",\"0\",{\"className\":\"flex gap-1\",\"children\":[[\"$\",\"svg\",null,{\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":\"20\",\"height\":\"21\",\"viewBox\":\"0 0 20 21\",\"fill\":\"none\",\"className\":\"mt-0.5\",\"children\":[\"$\",\"path\",null,{\"d\":\"M9.14598 14.4365C10.6142 11.6377 12.4682 9.39312 14.6552 7.7673C15.0469 7.47672 15.1281 6.92263 14.8375 6.53205C14.547 6.1403 13.994 6.0603 13.6035 6.34971C11.5753 7.85671 9.81654 9.83428 8.35895 12.2401C7.8025 11.3648 7.19429 10.5978 6.5202 9.92016C6.1755 9.57311 5.61552 9.57312 5.27201 9.91663C4.92614 10.2601 4.92496 10.8201 5.26848 11.1648C6.14727 12.0483 6.90371 13.1165 7.58016 14.4306C7.73191 14.7247 8.03896 14.9129 8.36483 14.9094C8.69306 14.9094 8.99422 14.727 9.14598 14.4365Z\",\"fill\":\"#F25533\"}]}],\"All-in-one incident management\"]}],[\"$\",\"li\",\"1\",{\"className\":\"flex gap-1\",\"children\":[[\"$\",\"svg\",null,{\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":\"20\",\"height\":\"21\",\"viewBox\":\"0 0 20 21\",\"fill\":\"none\",\"className\":\"mt-0.5\",\"children\":[\"$\",\"path\",null,{\"d\":\"M9.14598 14.4365C10.6142 11.6377 12.4682 9.39312 14.6552 7.7673C15.0469 7.47672 15.1281 6.92263 14.8375 6.53205C14.547 6.1403 13.994 6.0603 13.6035 6.34971C11.5753 7.85671 9.81654 9.83428 8.35895 12.2401C7.8025 11.3648 7.19429 10.5978 6.5202 9.92016C6.1755 9.57311 5.61552 9.57312 5.27201 9.91663C4.92614 10.2601 4.92496 10.8201 5.26848 11.1648C6.14727 12.0483 6.90371 13.1165 7.58016 14.4306C7.73191 14.7247 8.03896 14.9129 8.36483 14.9094C8.69306 14.9094 8.99422 14.727 9.14598 14.4365Z\",\"fill\":\"#F25533\"}]}],\"Our unmatched speed of deployment\"]}],[\"$\",\"li\",\"2\",{\"className\":\"flex gap-1\",\"children\":[[\"$\",\"svg\",null,{\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":\"20\",\"height\":\"21\",\"viewBox\":\"0 0 20 21\",\"fill\":\"none\",\"className\":\"mt-0.5\",\"children\":[\"$\",\"path\",null,{\"d\":\"M9.14598 14.4365C10.6142 11.6377 12.4682 9.39312 14.6552 7.7673C15.0469 7.47672 15.1281 6.92263 14.8375 6.53205C14.547 6.1403 13.994 6.0603 13.6035 6.34971C11.5753 7.85671 9.81654 9.83428 8.35895 12.2401C7.8025 11.3648 7.19429 10.5978 6.5202 9.92016C6.1755 9.57311 5.61552 9.57312 5.27201 9.91663C4.92614 10.2601 4.92496 10.8201 5.26848 11.1648C6.14727 12.0483 6.90371 13.1165 7.58016 14.4306C7.73191 14.7247 8.03896 14.9129 8.36483 14.9094C8.69306 14.9094 8.99422 14.727 9.14598 14.4365Z\",\"fill\":\"#F25533\"}]}],\"Why we’re loved by users and easily adopted\"]}],[\"$\",\"li\",\"3\",{\"className\":\"flex gap-1\",\"children\":[[\"$\",\"svg\",null,{\"xmlns\":\"http://www.w3.org/2000/svg\",\"width\":\"20\",\"height\":\"21\",\"viewBox\":\"0 0 20 21\",\"fill\":\"none\",\"className\":\"mt-0.5\",\"children\":[\"$\",\"path\",null,{\"d\":\"M9.14598 14.4365C10.6142 11.6377 12.4682 9.39312 14.6552 7.7673C15.0469 7.47672 15.1281 6.92263 14.8375 6.53205C14.547 6.1403 13.994 6.0603 13.6035 6.34971C11.5753 7.85671 9.81654 9.83428 8.35895 12.2401C7.8025 11.3648 7.19429 10.5978 6.5202 9.92016C6.1755 9.57311 5.61552 9.57312 5.27201 9.91663C4.92614 10.2601 4.92496 10.8201 5.26848 11.1648C6.14727 12.0483 6.90371 13.1165 7.58016 14.4306C7.73191 14.7247 8.03896 14.9129 8.36483 14.9094C8.69306 14.9094 8.99422 14.727 9.14598 14.4365Z\",\"fill\":\"#F25533\"}]}],\"How we work for the whole organization\"]}]]}]]}],\"$L45\"]}]]}]\n"])</script><script>self.__next_f.push([1,"46:I[779397,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\",\"/_next/static/immutable/chunks/39pfb5kk8her1.js\",\"/_next/static/immutable/chunks/3qz9rwp_7jbnb.js\",\"/_next/static/immutable/chunks/1my_9vnng6gxj.js\",\"/_next/static/immutable/chunks/040yti440pn1b.js\",\"/_next/static/immutable/chunks/0v4j5g1ppvtg_.js\",\"/_next/static/immutable/chunks/3buml8s4fdaii.js\",\"/_next/static/immutable/chunks/2_frah5pspzdw.js\",\"/_next/static/immutable/chunks/1mcb8cvy5_n5b.js\",\"/_next/static/immutable/chunks/132szer1wwfxa.js\",\"/_next/static/immutable/chunks/2jjhnej_crwty.js\",\"/_next/static/immutable/chunks/19t-07qzgwjic.js\",\"/_next/static/immutable/chunks/1wpznbhwql3dd.js\",\"/_next/static/immutable/chunks/20izjjpsu0ag8.js\"],\"RequestADemoButton\"]\n47:I[379700,[\"/_next/static/immutable/chunks/0-l0sxraekmer.js\",\"/_next/static/immutable/chunks/1vj2-813oq6tg.js\",\"/_next/static/immutable/chunks/1vvlruwewhgbt.js\",\"/_next/static/immutable/chunks/27fxx2z424byj.js\",\"/_next/static/immutable/chunks/3c4tjp_3gm_rj.js\",\"/_next/static/immutable/chunks/3toyqodzjr-9d.js\",\"/_next/static/immutable/chunks/2hdxyct5upc3z.js\",\"/_next/static/immutable/chunks/1ah2s79s4e7mm.js\",\"/_next/static/immutable/chunks/24dbkag80ciqd.js\",\"/_next/static/immutable/chunks/34-yxyyt1i1n8.js\",\"/_next/static/immutable/chunks/1zi8qxi1p_fsn.js\",\"/_next/static/immutable/chunks/2eophjajue3_e.js\",\"/_next/static/immutable/chunks/3d48p4uv87yqq.js\",\"/_next/static/immutable/chunks/2y03n43y-fjfs.js\",\"/_next/static/immutable/chunks/39pfb5kk8her1.js\",\"/_next/static/immutable/chunks/3qz9rwp_7jbnb.js\",\"/_next/static/immutable/chunks/1my_9vnng6gxj.js\",\"/_next/static/immutable/chunks/040yti440pn1b.js\",\"/_next/static/immutable/chunks/0v4j5g1ppvtg_.js\",\"/_next/static/immutable/chunks/3buml8s4fdaii.js\",\"/_next/static/immutable/chunks/2_frah5pspzdw.js\",\"/_next/static/immutable/chunks/1mcb8cvy5_n5b.js\",\"/_next/static/immutable/chunks/132szer1wwfxa.js\",\"/_next/static/immutable/chunks/2jjhnej_crwty.js\",\"/_next/static/immutable/chunks/19t-07qzgwjic.js\",\"/_next/static/immutable/chunks/1wpznbhwql3dd.js\",\"/_next/static/immutable/chunks/20izjjpsu0ag8.js\"],\"StartATrialButton\"]\n45:[\"$\",\"div\",null,{\"className\":\"flex justify-between items-center\",\"children\":[[\"$\",\"div\",null,{\"className\":\"w-full flex gap-4 flex-col sm:flex-row whitespace-nowrap\",\"children\":[\"$undefined\",[\"$\",\"$L46\",null,{\"analyticsSource\":\"homepage-signup-card\",\"size\":\"lg\",\"children\":\"Get a demo\"}],[\"$\",\"$L47\",null,{\"href\":\"$undefined\",\"size\":\"lg\",\"theme\":\"tertiary\",\"analyticsSource\":\"homepage-signup-card\",\"children\":\"$undefined\"}]]}],\"$L48\"]}]\n"])</script><script>self.__next_f.push([1,"48:[\"$\",\"div\",null,{\"className\":\"-space-x-1 hidden md:flex\",\"children\":[[[\"$\",\"$L36\",\"Rory Malcolm\",{\"src\":{\"_type\":\"image\",\"asset\":{\"_ref\":\"image-1e715111dcf04d620a13db5d6b70d9a19b74033f-3024x4032-jpg\",\"_type\":\"reference\"}},\"alt\":\"Rory Malcolm\",\"className\":\"size-10 rounded-full border-2 border-surface-secondary\"}],[\"$\",\"$L36\",\"Lucy Jennings\",{\"src\":{\"_type\":\"image\",\"asset\":{\"_ref\":\"image-4a76d0ae3b8eef5d16f525ac375b3570563346a3-800x800-jpg\",\"_type\":\"reference\"},\"crop\":{\"_type\":\"sanity.imageCrop\",\"bottom\":0.35513671874999986,\"left\":0.32833007812500015,\"right\":0.23209960937499996,\"top\":0.21847656250000017},\"hotspot\":{\"_type\":\"sanity.imageHotspot\",\"height\":0.42638671874999995,\"width\":0.4395703124999999,\"x\":0.5481152343750001,\"y\":0.43166992187500014}},\"alt\":\"Lucy Jennings\",\"className\":\"size-10 rounded-full border-2 border-surface-secondary\"}],[\"$\",\"$L36\",\"Eryn Carman\",{\"src\":{\"_type\":\"image\",\"asset\":{\"_ref\":\"image-13c3df986e33da4b270a0ede8c6859982f9f87f9-943x943-jpg\",\"_type\":\"reference\"}},\"alt\":\"Eryn Carman\",\"className\":\"size-10 rounded-full border-2 border-surface-secondary\"}]],false]}]\n"])</script></body></html>