Files
nexus/sreweekly/articles/474/08-incident-report-april-23rd-2025.html
2026-09-12 17:23:01 +08:00

261 lines
44 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8" data-next-head=""/><meta name="viewport" content="width=device-width" data-next-head=""/><meta name="twitter:card" content="summary_large_image" data-next-head=""/><meta name="twitter:site" content="@Railway" data-next-head=""/><meta name="twitter:creator" content="@Railway" data-next-head=""/><meta property="og:image:type" content="image/png" data-next-head=""/><meta property="og:image:width" content="1200" data-next-head=""/><meta property="og:image:height" content="630" data-next-head=""/><meta property="og:site_name" content="Railway Blog" data-next-head=""/><title data-next-head="">Incident Report: April 23rd, 2025</title><meta name="robots" content="index,follow" data-next-head=""/><meta name="description" content="We recently experienced an outage in our networking control plane that left public and private networking unavailable for a portion of users." data-next-head=""/><meta property="og:title" content="Incident Report: April 23rd, 2025" data-next-head=""/><meta property="og:description" content="We recently experienced an outage in our networking control plane that left public and private networking unavailable for a portion of users." data-next-head=""/><meta property="og:url" content="https://blog.railway.com/p/incident-report-april-23-2025" data-next-head=""/><meta property="og:type" content="article" data-next-head=""/><meta property="article:published_time" content="2025-04-23T00:00:00.000Z" data-next-head=""/><meta property="article:modified_time" content="2025-04-23T00:00:00.000Z" data-next-head=""/><meta property="article:author" content="Ray Chen" data-next-head=""/><meta property="article:section" content="Engineering" data-next-head=""/><meta property="og:image" content="https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png?w=1200&amp;q=90" data-next-head=""/><link rel="canonical" href="https://blog.railway.com/p/incident-report-april-23-2025" data-next-head=""/><meta name="twitter:title" content="Incident Report: April 23rd, 2025" data-next-head=""/><meta name="twitter:description" content="We recently experienced an outage in our networking control plane that left public and private networking unavailable for a portion of users." data-next-head=""/><meta name="twitter:image" content="https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png?w=1200&amp;q=90" data-next-head=""/><link rel="alternate" type="text/markdown" href="/p/incident-report-april-23-2025.md" data-next-head=""/><link rel="alternate" type="application/rss+xml" title="Railway Blog" href="https://blog.railway.com/rss.xml"/><link rel="preload" href="/_next/static/media/e4af272ccee01ff0-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/91c3caac73720166-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/3359c9acc30ef440-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/aff755c8b4bb3089-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/3988c35203e0830a-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/427e4a37d3642943-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/css/619a01ff31a99955.css" as="style"/><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"BlogPosting","headline":"Incident Report: April 23rd, 2025","description":"We recently experienced an outage in our networking control plane that left public and private networking unavailable for a portion of users.","image":["https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png?w=1200&q=90"],"datePublished":"2025-04-23T00:00:00.000Z","dateModified":"2025-04-23T00:00:00.000Z","author":{"@type":"Person","name":"Ray Chen"},"publisher":{"@type":"Organization","name":"Railway","logo":{"@type":"ImageObject","url":"https://railway.com/brand/logo-dark.png"}},"mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.railway.com/p/incident-report-april-23-2025"}}</script><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https://blog.railway.com"},{"@type":"ListItem","position":2,"name":"Engineering","item":"https://blog.railway.com/engineering"},{"@type":"ListItem","position":3,"name":"Incident Report: April 23rd, 2025","item":"https://blog.railway.com/p/incident-report-april-23-2025"}]}</script><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"FAQPage","mainEntity":[{"@type":"Question","name":"What Happened?","acceptedAnswer":{"@type":"Answer","text":"At 09:10 UTC , we started noticing severe networking degradation on our Public TCP/HTTP monitors."}}]}</script><style>:root {
--foreground: hsl(0, 0%, 100%);
--background: #181622;
--secondaryBg: hsl(250, 21%, 11%);
--blue-50: hsl(220, 55%, 10%);
--blue-100: hsl(220, 55%, 13%);
--blue-200: hsl(220, 62%, 25%);
--blue-300: hsl(220, 68%, 35%);
--blue-400: hsl(220, 72%, 45%);
--blue-500: hsl(220, 80%, 55%);
--blue-600: hsl(220, 80%, 65%);
--blue-700: hsl(220, 80%, 75%);
--blue-800: hsl(220, 80%, 85%);
--blue-900: hsl(220, 80%, 95%);
--blue-950: hsl(220, 55%, 97%);
--gray-50: hsl(248, 21%, 13%);
--gray-100: hsl(246, 18%, 15%);
--gray-200: hsl(246, 11%, 22%);
--gray-300: hsl(246, 8%, 35%);
--gray-400: hsl(246, 7%, 45%);
--gray-500: hsl(246, 6%, 55%);
--gray-600: hsl(246, 6%, 65%);
--gray-700: hsl(246, 6%, 78%);
--gray-800: hsl(246, 6%, 87%);
--gray-900: hsl(246, 6%, 95%);
--gray-950: hsl(246, 6%, 95%);
--red-50: hsl(1, 35%, 10%);
--red-100: hsl(1, 45%, 12%);
--red-200: hsl(1, 55%, 20%);
--red-300: hsl(1, 62%, 28%);
--red-400: hsl(1, 62%, 35%);
--red-500: hsl(1, 62%, 44%);
--red-600: hsl(1, 62%, 60%);
--red-700: hsl(1, 62%, 76%);
--red-800: hsl(1, 64%, 85%);
--red-900: hsl(1, 68%, 95%);
--red-950: hsl(1, 55%, 98%);
--pink-50: hsl(270, 40%, 16%);
--pink-100: hsl(270, 40%, 16%);
--pink-200: hsl(270, 45%, 24%);
--pink-300: hsl(270, 50%, 32%);
--pink-400: hsl(270, 55%, 43%);
--pink-500: hsl(270, 60%, 52%);
--pink-600: hsl(270, 70%, 65%);
--pink-700: hsl(270, 70%, 75%);
--pink-800: hsl(270, 70%, 85%);
--pink-900: hsl(270, 70%, 95%);
--pink-950: hsl(270, 70%, 95%);
--green-50: hsl(152, 15%, 10%);
--green-100: hsl(152, 26%, 11%);
--green-200: hsl(152, 32%, 16%);
--green-300: hsl(152, 38%, 24%);
--green-400: hsl(152, 38%, 34%);
--green-500: hsl(152, 38%, 42%);
--green-600: hsl(152, 38%, 60%);
--green-700: hsl(152, 38%, 70%);
--green-800: hsl(152, 38%, 80%);
--green-900: hsl(152, 38%, 91%);
--green-950: hsl(152, 40%, 97%);
--yellow-50: hsl(44, 95%, 12%);
--yellow-100: hsl(44, 95%, 12%);
--yellow-200: hsl(44, 96%, 24%);
--yellow-300: hsl(44, 95%, 36%);
--yellow-400: hsl(44, 95%, 48%);
--yellow-500: hsl(44, 95%, 60%);
--yellow-600: hsl(44, 95%, 69%);
--yellow-700: hsl(44, 95%, 78%);
--yellow-800: hsl(44, 95%, 86%);
--yellow-900: hsl(44, 95%, 95%);
--yellow-950: hsl(44, 95%, 95%);
}
.light {
--foreground: hsl(250, 24%, 9%);
--background: #F1F0EF;
--secondaryBg: hsl(0, 0%, 98%);
--blue-50: hsl(220, 55%, 97%);
--blue-100: hsl(220, 80%, 95%);
--blue-200: hsl(220, 80%, 85%);
--blue-300: hsl(220, 80%, 75%);
--blue-400: hsl(220, 80%, 65%);
--blue-500: hsl(220, 80%, 55%);
--blue-600: hsl(220, 72%, 45%);
--blue-700: hsl(220, 68%, 35%);
--blue-800: hsl(220, 62%, 25%);
--blue-900: hsl(220, 55%, 13%);
--blue-950: hsl(220, 55%, 10%);
--gray-50: hsl(246, 6%, 95%);
--gray-100: hsl(246, 6%, 95%);
--gray-200: hsl(246, 6%, 87%);
--gray-300: hsl(246, 6%, 78%);
--gray-400: hsl(246, 6%, 65%);
--gray-500: hsl(246, 6%, 55%);
--gray-600: hsl(246, 7%, 45%);
--gray-700: hsl(246, 8%, 35%);
--gray-800: hsl(246, 11%, 22%);
--gray-900: hsl(246, 18%, 15%);
--gray-950: hsl(248, 21%, 13%);
--red-50: hsl(1, 55%, 98%);
--red-100: hsl(1, 68%, 95%);
--red-200: hsl(1, 64%, 85%);
--red-300: hsl(1, 62%, 76%);
--red-400: hsl(1, 62%, 60%);
--red-500: hsl(1, 62%, 44%);
--red-600: hsl(1, 62%, 35%);
--red-700: hsl(1, 62%, 28%);
--red-800: hsl(1, 55%, 20%);
--red-900: hsl(1, 45%, 12%);
--red-950: hsl(1, 35%, 10%);
--pink-50: hsl(270, 70%, 95%);
--pink-100: hsl(270, 70%, 95%);
--pink-200: hsl(270, 70%, 85%);
--pink-300: hsl(270, 70%, 75%);
--pink-400: hsl(270, 70%, 65%);
--pink-500: hsl(270, 60%, 52%);
--pink-600: hsl(270, 55%, 43%);
--pink-700: hsl(270, 50%, 32%);
--pink-800: hsl(270, 45%, 24%);
--pink-900: hsl(270, 40%, 16%);
--pink-950: hsl(270, 40%, 16%);
--green-50: hsl(152, 40%, 97%);
--green-100: hsl(152, 38%, 91%);
--green-200: hsl(152, 38%, 80%);
--green-300: hsl(152, 38%, 70%);
--green-400: hsl(152, 38%, 60%);
--green-500: hsl(152, 38%, 42%);
--green-600: hsl(152, 38%, 34%);
--green-700: hsl(152, 38%, 24%);
--green-800: hsl(152, 32%, 16%);
--green-900: hsl(152, 26%, 11%);
--green-950: hsl(152, 15%, 10%);
--yellow-50: hsl(44, 95%, 95%);
--yellow-100: hsl(44, 95%, 95%);
--yellow-200: hsl(44, 95%, 86%);
--yellow-300: hsl(44, 95%, 78%);
--yellow-400: hsl(44, 95%, 69%);
--yellow-500: hsl(44, 95%, 60%);
--yellow-600: hsl(44, 95%, 48%);
--yellow-700: hsl(44, 95%, 36%);
--yellow-800: hsl(44, 96%, 24%);
--yellow-900: hsl(44, 95%, 12%);
--yellow-950: hsl(44, 95%, 12%);
}
.dark {
--foreground: hsl(0, 0%, 100%);
--background: #181622;
--secondaryBg: hsl(250, 21%, 11%);
--blue-50: hsl(220, 55%, 10%);
--blue-100: hsl(220, 55%, 13%);
--blue-200: hsl(220, 62%, 25%);
--blue-300: hsl(220, 68%, 35%);
--blue-400: hsl(220, 72%, 45%);
--blue-500: hsl(220, 80%, 55%);
--blue-600: hsl(220, 80%, 65%);
--blue-700: hsl(220, 80%, 75%);
--blue-800: hsl(220, 80%, 85%);
--blue-900: hsl(220, 80%, 95%);
--blue-950: hsl(220, 55%, 97%);
--gray-50: hsl(248, 21%, 13%);
--gray-100: hsl(246, 18%, 15%);
--gray-200: hsl(246, 11%, 22%);
--gray-300: hsl(246, 8%, 35%);
--gray-400: hsl(246, 7%, 45%);
--gray-500: hsl(246, 6%, 55%);
--gray-600: hsl(246, 6%, 65%);
--gray-700: hsl(246, 6%, 78%);
--gray-800: hsl(246, 6%, 87%);
--gray-900: hsl(246, 6%, 95%);
--gray-950: hsl(246, 6%, 95%);
--red-50: hsl(1, 35%, 10%);
--red-100: hsl(1, 45%, 12%);
--red-200: hsl(1, 55%, 20%);
--red-300: hsl(1, 62%, 28%);
--red-400: hsl(1, 62%, 35%);
--red-500: hsl(1, 62%, 44%);
--red-600: hsl(1, 62%, 60%);
--red-700: hsl(1, 62%, 76%);
--red-800: hsl(1, 64%, 85%);
--red-900: hsl(1, 68%, 95%);
--red-950: hsl(1, 55%, 98%);
--pink-50: hsl(270, 40%, 16%);
--pink-100: hsl(270, 40%, 16%);
--pink-200: hsl(270, 45%, 24%);
--pink-300: hsl(270, 50%, 32%);
--pink-400: hsl(270, 55%, 43%);
--pink-500: hsl(270, 60%, 52%);
--pink-600: hsl(270, 70%, 65%);
--pink-700: hsl(270, 70%, 75%);
--pink-800: hsl(270, 70%, 85%);
--pink-900: hsl(270, 70%, 95%);
--pink-950: hsl(270, 70%, 95%);
--green-50: hsl(152, 15%, 10%);
--green-100: hsl(152, 26%, 11%);
--green-200: hsl(152, 32%, 16%);
--green-300: hsl(152, 38%, 24%);
--green-400: hsl(152, 38%, 34%);
--green-500: hsl(152, 38%, 42%);
--green-600: hsl(152, 38%, 60%);
--green-700: hsl(152, 38%, 70%);
--green-800: hsl(152, 38%, 80%);
--green-900: hsl(152, 38%, 91%);
--green-950: hsl(152, 40%, 97%);
--yellow-50: hsl(44, 95%, 12%);
--yellow-100: hsl(44, 95%, 12%);
--yellow-200: hsl(44, 96%, 24%);
--yellow-300: hsl(44, 95%, 36%);
--yellow-400: hsl(44, 95%, 48%);
--yellow-500: hsl(44, 95%, 60%);
--yellow-600: hsl(44, 95%, 69%);
--yellow-700: hsl(44, 95%, 78%);
--yellow-800: hsl(44, 95%, 86%);
--yellow-900: hsl(44, 95%, 95%);
--yellow-950: hsl(44, 95%, 95%);
}</style><link rel="stylesheet" href="/_next/static/css/619a01ff31a99955.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" noModule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-0b474a604a067f92.js" defer=""></script><script src="/_next/static/chunks/framework-90fc9b0f7c4706c1.js" defer=""></script><script src="/_next/static/chunks/main-e89abd6570bbb9dc.js" defer=""></script><script src="/_next/static/chunks/pages/_app-3049b372678b389f.js" defer=""></script><script src="/_next/static/chunks/5399-d49efedd870f41de.js" defer=""></script><script src="/_next/static/chunks/3801-cdd2681d27083ad8.js" defer=""></script><script src="/_next/static/chunks/9594-1c44b12020a3af0a.js" defer=""></script><script src="/_next/static/chunks/4696-fd1712bc36a5bc50.js" defer=""></script><script src="/_next/static/chunks/pages/p/%5Bslug%5D-ddadb1ca54eaab0f.js" defer=""></script><script src="/_next/static/nFX7R3M0MdLMAf-Ufh9qa/_buildManifest.js" defer=""></script><script src="/_next/static/nFX7R3M0MdLMAf-Ufh9qa/_ssgManifest.js" defer=""></script></head><body><div id="__next"><script>!function(){try{var d=document.documentElement,c=d.classList;c.remove('light','dark');var e=localStorage.getItem('theme');if('system'===e||(!e&&true)){var t='(prefers-color-scheme: dark)',m=window.matchMedia(t);if(m.media!==t||m.matches){d.style.colorScheme = 'dark';c.add('dark')}else{d.style.colorScheme = 'light';c.add('light')}}else if(e){c.add(e|| '')}if(e==='light'||e==='dark')d.style.colorScheme=e}catch(e){}}()</script><div class="__variable_f367f3 __variable_0b668e __variable_ecea63 font-sans"><div class="px-5 md:px-8"><nav class="max-w-6xl mx-auto py-6 flex justify-between items-center border-b border-gray-100"><a class="flex items-center space-x-4" href="/"><svg data-v-423bf9ae="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 59.99945071644884 60" class="logo w-8"><g data-v-423bf9ae="" id="6f639fe5-02e1-4640-a1fb-f4b22b64e5ef" transform="matrix(0.5999641134746578,0,0,0.5999641134746578,-437.5538206497148,-266.68705118956507)" stroke="none"><path d="M729.872 487.327a50.86 50.86 0 0 0-.464 5.033h75.879c-.265-.518-.621-.985-.98-1.442-12.972-16.769-19.95-15.315-29.932-15.741-3.328-.137-5.585-.192-18.832-.192-7.09 0-14.798.018-22.304.038.737-1.746 1.6-3.419 2.525-5.061-2.885 5.106-4.865 10.771-5.805 16.789l.891-4.397c.007-.031.018-.063.024-.094h38.883v5.067h-39.885zM805.885 497.432h-76.438c.08 1.352.206 2.686.388 4.002h70.571c3.146 0 4.907-1.786 5.479-4.002zM733.851 515.257a52.226 52.226 0 0 1-1.98-4.997c6.608 19.89 25.328 34.251 47.433 34.251 20.205 0 37.566-12.007 45.452-29.254h-90.905zM729.38 492.915c-.018.531-.08 1.055-.08 1.589 0 .538.063 1.059.08 1.59v-3.179zM824.77 515.229z"></path><path d="M779.303 444.505c-18.682 0-34.939 10.265-43.524 25.439 6.709-.014 19.775-.022 19.775-.022h.003v-.005c15.444 0 16.018.069 19.035.195l1.868.069c6.507.217 14.505.916 20.798 5.68 3.416 2.584 8.348 8.287 11.288 12.35 2.718 3.758 3.5 8.078 1.652 12.217-1.701 3.804-5.361 6.073-9.793 6.073H730.85l-.884-4.201c.426 2.707 1.037 5.344 1.879 7.886h94.914a49.863 49.863 0 0 0 2.546-15.682c.001-27.611-22.386-49.999-50.002-49.999z"></path></g></svg><span class="text-xl font-semibold">Blog</span></a><div class="text-gray-600 flex items-center space-x-6"><a class="text-sm hover:text-pink-600" href="https://railway.com" target="_blank" rel="noreferrer noopener">Go to Homepage</a><a class="hover:text-pink-600" href="/rss.xml" title="RSS Feed"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4 11a9 9 0 0 1 9 9"></path><path d="M4 4a16 16 0 0 1 16 16"></path><circle cx="5" cy="19" r="1"></circle></svg></a></div></nav></div><div class="min-h-screen overflow-x-hidden relative"><div class="mt-10 mb-5 px-5 md:px-8 mx-auto"><article class="max-w-6xl mx-auto mt-24 mb-12 border-b border-gray-100 pb-32"><div class="max-w-[704px] mx-auto flex items-center text-sm text-gray-600 space-x-3"><div class="flex items-center space-x-3"><div class="flex items-center"><img src="https://cms.railway.com/media/person-ray-chen-dd0fa1297e94.jpg" alt="Avatar of Ray Chen" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/></div><span>Ray Chen</span></div><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><time dateTime="2025-04-23T00:00:00.000Z">Apr 23, 2025</time></div><header class="mt-5 mb-16 max-w-[704px] mx-auto"><h1 class="text-post-title font-medium font-serif">Incident Report: April 23rd, 2025</h1></header><section class="max-w-[704px] mx-auto text-base sm:text-lg"><nav style="position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border-width:0" aria-label="Table of Contents"><h2>Table of Contents</h2><ol><li style="margin-left:1.5rem"><a href="#impact">Impact</a></li><li style="margin-left:1.5rem"><a href="#incident-timeline">Incident Timeline</a></li><li style="margin-left:1.5rem"><a href="#what-happened">What Happened?</a></li><li style="margin-left:1.5rem"><a href="#preventative-measures">Preventative Measures</a></li></ol></nav><div><p class="mb-4 text-gray-800">We recently experienced an outage that affected our networking control plane. During this outage, public and private networking on Railway was unavailable for a portion of users.</p>
<p class="mb-4 text-gray-800">When a Major Outage occurs, it is Railway’s policy to share the public details of what happened.</p>
<a class="relative no-underline" href="/p/incident-report-april-23-2025#impact"><span id="impact" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Impact</h2></a>
<p class="mb-4 text-gray-800">This outage impacted our Edge Network, Private Network, and TCP Proxy across all regions.</p>
<p class="mb-4 text-gray-800">If you were visiting a Railway-hosted domain or if your application was communicating over our private network, the connections may have been highly unstable.</p>
<p class="mb-4 text-gray-800">We were also unable to provision new domains and SSL certificates during this period. If you tried adding a new domain to your Railway service, you would have encountered an error.</p>
<p class="mb-4 text-gray-800">Our Asia region had the highest impact. All other regions had a significantly lower impact, and network connectivity was only intermittently unavailable for a small subset of users in those regions. We also observed that newer deployments had the highest incidence rate.</p>
<a class="relative no-underline" href="/p/incident-report-april-23-2025#incident-timeline"><span id="incident-timeline" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Incident Timeline</h2></a>
<p class="mb-4 text-gray-800"><a class="markdown-inline-link" href="https://status.railway.com/cm9tq9j0a007mlx4779v3kcyb" target="_blank" rel="noreferrer noopener">Incident on our Status Page</a></p>
<ul class="list-disc pl-6 mb-4 space-y-4 text-gray-800">
<li><strong>09:10 UTC</strong>: Our on-call engineers were paged for network monitoring failures, and started investigating immediately</li>
<li><strong>09:15 UTC</strong>: We started seeing partial recovery from users, and continued our investigations into the root cause</li>
<li><strong>09:28 UTC</strong>: We started seeing full recovery across the platform</li>
<li><strong>10:36 UTC</strong>: Incident resolved</li>
</ul>
<a class="relative no-underline" href="/p/incident-report-april-23-2025#what-happened"><span id="what-happened" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">What Happened?</h2></a>
<p class="mb-4 text-gray-800">At <strong>09:10 UTC</strong>, we started noticing severe networking degradation on our Public TCP/HTTP monitors.</p>
<p class="mb-4 text-gray-800">At <strong>09:11 UTC</strong>, we saw the same failures for our Private Network monitors.</p>
<p class="mb-4 text-gray-800">Our monitors actively probe various networking endpoints to monitor the health of public and private networking on Railway. When a probe fails, the on-call engineer is paged immediately.</p>
<p class="mb-4 text-gray-800">The root cause of this outage was traced to our networking control plane. The control plane is responsible for provisioning and maintaining all user networking resources on Railway. For example:</p>
<ul class="list-disc pl-6 mb-4 space-y-4 text-gray-800">
<li>When you create a domain on Railway, the control plane provisions SSL certificates and informs our platform about its existence;</li>
<li>When you issue a new deploy on Railway, the control plane is responsible for updating its public and private routing information across our fleet;</li>
<li>When you visit a Railway domain, the control plane is consulted for its routing information so we know where to send traffic to;</li>
<li>etc.</li>
</ul>
<p class="mb-4 text-gray-800">This control plane uses Postgres as its persistent data store. We use this to persist traffic routing information (among other types of critical data) that lets us know where to send traffic to within our fleet. This routing information is also cached in-memory.</p>
<p class="mb-4 text-gray-800">The Postgres service lives on Google Cloud SQL and is configured to be highly-available with a read replica.</p>
<p class="mb-4 text-gray-800">At <strong>09:07 UTC</strong>, Google Cloud SQL started a maintenance on this Postgres service that included the read replica, causing all connections to be terminated.</p>
<p class="mb-4 text-gray-800">At <strong>09:08 UTC</strong>, we started seeing database-related errors across all instances of our network control plane that mentioned read replica had to sync with the primary instance.</p>
<p class="mb-4 text-gray-800">At <strong>09:17 UTC</strong>, the maintenance initiated by Google Cloud SQL was completed. This lasted 10 minutes, and effectively meant our network control plane’s read-only database replica was unreachable.</p>
<p class="mb-4 text-gray-800">Some instances of our control plane started crashing when the database went offline due to improper error handling in its code. While they were configured to restart automatically, all routing information would be evicted from memory on each restart, and had to be re-fetched from Postgres which was offline.</p>
<p class="mb-4 text-gray-800">At around <strong>09:15 UTC</strong>, we started seeing partial recovery from user reports, along with blips in our monitors which suggested the issue was intermittent. The healthy instances of our control plane continued serving traffic normally, so our networking was in a degraded state where a subset of user traffic may be routable, while others were not.</p>
<p class="mb-4 text-gray-800">After Postgres was back online, our control plane started to reboot itself successfully and resume normal operations. The boot process took longer than expected due to post-boot operations delaying it from being able to serve traffic (warming its local cache, etc.)</p>
<p class="mb-4 text-gray-800">At <strong>09:28 UTC</strong>, we started seeing full recovery after the control plane and database became fully healthy and operational.</p>
<a class="relative no-underline" href="/p/incident-report-april-23-2025#preventative-measures"><span id="preventative-measures" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Preventative Measures</h2></a>
<p class="mb-4 text-gray-800">There are multiple parts of our control plane that should have gracefully degraded instead of failing over.</p>
<p class="mb-4 text-gray-800">We have prioritized fixing this as an important project for 2025Q2. As part of this work, we’re going to create regional control planes instead of a global control plane, which will help us contain the blast radius of control plane failures, and allow us to mitigate any single-region failure by failing over route information lookup to other regions.</p>
<p class="mb-4 text-gray-800">On top of improving the architecture of our control plane, we’re going to:</p>
<ul class="list-disc pl-6 mb-4 space-y-4 text-gray-800">
<li>Fix the bug that causes the control plane to crash when it encounters a database connection error</li>
<li>Introduce a layer of persistent cache on routing information as a secondary fallback</li>
<li>Lower the start-up time of the control plane so we can recover from similar failures faster</li>
<li>Re-visit our Google Cloud SQL Postgres configuration to ensure its primary and replica aren’t under maintenance at the same time, and to notify us on upcoming maintenance(s) so we can monitor its progress and impact closely</li>
</ul>
<hr class="my-8"/>
<p class="mb-4 text-gray-800">Railway is committed to providing the best-in-class cloud experience. Any downtime is unacceptable for us. We apologize for any inconvenience caused by this, and we are going to work towards eliminating the entire class of issues contributing to this incident.</p></div></section></article><div class="max-w-6xl mx-auto"><div><header class="flex items-center justify-between mb-8"><h3 class="text-gray-500 text-h3">Continue Reading...</h3><a class="text-pink-500 hover:underline" href="/engineering">View All <!-- -->Engineering<!-- --> -&gt;</a></header><div class="grid grid-cols-1 md:grid-cols-2 gap-8"><a class="flex flex-col bg-secondaryBg p-6 rounded-lg hover:bg-gray-100 group" href="/p/dev-new"><div class="flex gap-2"><div class="post-category-pill font-medium max-w-max text-xs px-2 py-[3px] rounded-[4px] tracking-[0.06em] uppercase" data-category-tone="plum">Engineering</div></div><div class="flex-grow"><header class="font-medium font-serif text-lg mt-2 mb-1">Your Cloud should come with an app builder </header><p class="text-base text-gray-800 line-clamp-2">Announcing: dev.new, an agent that can build, preview, test, and deploy an app from the same place. We tested the whole loop by asking it to ship a real-time multiplayer FPS game that runs in the browser</p></div><div class="flex items-center gap-3 mt-6"><div class="flex items-center"><img src="https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg" alt="Avatar of Mahmoud Abdelwahab" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/></div><span class="text-sm text-gray-600">Mahmoud Abdelwahab</span><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><span class="text-sm text-gray-600">Aug 13, 2026</span></div></a><a class="flex flex-col bg-secondaryBg p-6 rounded-lg hover:bg-gray-100 group" href="/p/feature-flags"><div class="flex gap-2"><div class="post-category-pill font-medium max-w-max text-xs px-2 py-[3px] rounded-[4px] tracking-[0.06em] uppercase" data-category-tone="plum">Engineering</div></div><div class="flex-grow"><header class="font-medium font-serif text-lg mt-2 mb-1">Roll it out, roll it back, never redeploy</header><p class="text-base text-gray-800 line-clamp-2">Merge now, release when you&#x27;re ready: turn a feature on for your team, then 10% of users, then everyone. Feature flags are now built into every Railway project, and your agents can run the rollout from the CLI, SDK, and MCP. No third-party service required.</p></div><div class="flex items-center gap-3 mt-6"><div class="flex items-center"><img src="https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg" alt="Avatar of Victor Ramirez" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/><img src="https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg" alt="Avatar of Mahmoud Abdelwahab" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:-8px" loading="lazy" decoding="async" width="24" height="24"/></div><span class="text-sm text-gray-600">Victor Ramirez &amp; Mahmoud Abdelwahab</span><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><span class="text-sm text-gray-600">Jul 17, 2026</span></div></a></div></div><section class="post-bottom-cta my-16 border rounded-[16px] px-6 py-16 sm:px-12 sm:py-24 flex flex-col items-center justify-center text-center" aria-labelledby="post-bottom-cta-title"><h3 id="post-bottom-cta-title" class="text-h2 font-serif font-medium">Ready to get started?</h3><p class="mt-4 text-base sm:text-lg text-gray-600">Join millions of developers deploying applications on Railway</p><div class="mt-8 flex w-full flex-col items-stretch justify-center gap-3 sm:w-auto sm:flex-row"><a class="post-bottom-cta-primary inline-flex min-h-[48px] items-center justify-center rounded-[8px] px-6 py-3 text-base font-medium no-underline transition-colors duration-100" href="https://railway.com" target="_blank" rel="noreferrer noopener">Deploy a new project</a><a class="post-bottom-cta-secondary inline-flex min-h-[48px] items-center justify-center rounded-[8px] border px-6 py-3 text-base font-medium no-underline transition-colors duration-100" href="https://railway.com/enterprise" target="_blank" rel="noreferrer noopener">Book a demo</a></div></section></div></div></div><footer class="px-5 md:px-8 py-12 border-t border-gray-100"><div class="grid grid-cols-2 sm:grid-cols-12 gap-8 sm:gap-4 lg:gap-16 max-w-6xl mx-auto"><div class="hidden sm:flex col-span-3 lg:col-span-4 flex-col justify-between"><a class="underline" href="https://railway.com" target="_blank" rel="noreferrer noopener"><svg data-v-423bf9ae="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 59.99945071644884 60" class="logo w-8"><g data-v-423bf9ae="" id="6f639fe5-02e1-4640-a1fb-f4b22b64e5ef" transform="matrix(0.5999641134746578,0,0,0.5999641134746578,-437.5538206497148,-266.68705118956507)" stroke="none"><path d="M729.872 487.327a50.86 50.86 0 0 0-.464 5.033h75.879c-.265-.518-.621-.985-.98-1.442-12.972-16.769-19.95-15.315-29.932-15.741-3.328-.137-5.585-.192-18.832-.192-7.09 0-14.798.018-22.304.038.737-1.746 1.6-3.419 2.525-5.061-2.885 5.106-4.865 10.771-5.805 16.789l.891-4.397c.007-.031.018-.063.024-.094h38.883v5.067h-39.885zM805.885 497.432h-76.438c.08 1.352.206 2.686.388 4.002h70.571c3.146 0 4.907-1.786 5.479-4.002zM733.851 515.257a52.226 52.226 0 0 1-1.98-4.997c6.608 19.89 25.328 34.251 47.433 34.251 20.205 0 37.566-12.007 45.452-29.254h-90.905zM729.38 492.915c-.018.531-.08 1.055-.08 1.589 0 .538.063 1.059.08 1.59v-3.179zM824.77 515.229z"></path><path d="M779.303 444.505c-18.682 0-34.939 10.265-43.524 25.439 6.709-.014 19.775-.022 19.775-.022h.003v-.005c15.444 0 16.018.069 19.035.195l1.868.069c6.507.217 14.505.916 20.798 5.68 3.416 2.584 8.348 8.287 11.288 12.35 2.718 3.758 3.5 8.078 1.652 12.217-1.701 3.804-5.361 6.073-9.793 6.073H730.85l-.884-4.201c.426 2.707 1.037 5.344 1.879 7.886h94.914a49.863 49.863 0 0 0 2.546-15.682c.001-27.611-22.386-49.999-50.002-49.999z"></path></g></svg></a><div class="text-xs font-medium text-gray-600 w-full">Copyright © <!-- -->2026<!-- --> Railway Corp. <br/>All rights reserved.</div></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Product</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/changelog" target="_blank" rel="noreferrer noopener">Changelog</a></li><li><a class="hover:text-foreground" href="https://railway.com/pricing" target="_blank" rel="noreferrer noopener">Pricing</a></li><li><a class="hover:text-foreground" href="https://railway.com/deploy" target="_blank" rel="noreferrer noopener">Templates</a></li><li><a class="hover:text-foreground" href="https://station.railway.com/feedback" target="_blank" rel="noreferrer noopener">Feedback</a></li><li><a class="hover:text-foreground" href="https://railway.com/open-source-kickback" target="_blank" rel="noreferrer noopener">OSS Kickback</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Company</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/about" target="_blank" rel="noreferrer noopener">About</a></li><li><a class="hover:text-foreground" href="https://railway.com/careers" target="_blank" rel="noreferrer noopener">Careers</a></li><li><a class="hover:text-foreground" href="/">Blog</a></li><li><a class="hover:text-foreground" href="https://shop.railway.com" target="_blank" rel="noreferrer noopener">Shop</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Contact</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://discord.gg/railway" target="_blank" rel="noreferrer noopener">Discord</a></li><li><a class="hover:text-foreground" href="https://twitter.com/railway" target="_blank" rel="noreferrer noopener">Twitter</a></li><li><a class="hover:text-foreground" href="https://github.com/railwayapp" target="_blank" rel="noreferrer noopener">GitHub</a></li><li><a class="hover:text-foreground" href="mailto:contact@railway.com">Email</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Legal</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/legal/acceptable-use" target="_blank" rel="noreferrer noopener">Acceptable Use</a></li><li><a class="hover:text-foreground" href="https://railway.com/legal/privacy" target="_blank" rel="noreferrer noopener">Privacy Policy</a></li><li><a class="hover:text-foreground" href="https://railway.com/legal/terms" target="_blank" rel="noreferrer noopener">Terms of Service</a></li></ul></div></div></footer></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"page":{"authors":[{"avatar":{"alt":"Ray Chen headshot","height":1096,"id":"1853","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-ray-chen-dd0fa1297e94.jpg","width":1152},"avatarUrl":"https://cms.railway.com/media/person-ray-chen-dd0fa1297e94.jpg","githubUrl":null,"id":"29","name":"Ray Chen","slug":"ray-chen","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":"We recently experienced an outage that affected our networking control plane. During this outage, public and private networking on Railway was unavailable for a portion of users.\n\nWhen a Major Outage occurs, it is Railway’s policy to share the public details of what happened.\n\n## Impact\n\nThis outage impacted our Edge Network, Private Network, and TCP Proxy across all regions.\n\nIf you were visiting a Railway-hosted domain or if your application was communicating over our private network, the connections may have been highly unstable.\n\nWe were also unable to provision new domains and SSL certificates during this period. If you tried adding a new domain to your Railway service, you would have encountered an error.\n\nOur Asia region had the highest impact. All other regions had a significantly lower impact, and network connectivity was only intermittently unavailable for a small subset of users in those regions. We also observed that newer deployments had the highest incidence rate.\n\n## Incident Timeline\n\n[Incident on our Status Page](https://status.railway.com/cm9tq9j0a007mlx4779v3kcyb)\n\n- **09:10 UTC**: Our on-call engineers were paged for network monitoring failures, and started investigating immediately\n- **09:15 UTC**: We started seeing partial recovery from users, and continued our investigations into the root cause\n- **09:28 UTC**: We started seeing full recovery across the platform\n- **10:36 UTC**: Incident resolved\n\n## What Happened?\n\nAt **09:10 UTC**, we started noticing severe networking degradation on our Public TCP/HTTP monitors.\n\nAt **09:11 UTC**, we saw the same failures for our Private Network monitors.\n\nOur monitors actively probe various networking endpoints to monitor the health of public and private networking on Railway. When a probe fails, the on-call engineer is paged immediately.\n\nThe root cause of this outage was traced to our networking control plane. The control plane is responsible for provisioning and maintaining all user networking resources on Railway. For example:\n\n- When you create a domain on Railway, the control plane provisions SSL certificates and informs our platform about its existence;\n- When you issue a new deploy on Railway, the control plane is responsible for updating its public and private routing information across our fleet;\n- When you visit a Railway domain, the control plane is consulted for its routing information so we know where to send traffic to;\n- etc.\n\nThis control plane uses Postgres as its persistent data store. We use this to persist traffic routing information (among other types of critical data) that lets us know where to send traffic to within our fleet. This routing information is also cached in-memory.\n\nThe Postgres service lives on Google Cloud SQL and is configured to be highly-available with a read replica.\n\nAt **09:07 UTC**, Google Cloud SQL started a maintenance on this Postgres service that included the read replica, causing all connections to be terminated.\n\nAt **09:08 UTC**, we started seeing database-related errors across all instances of our network control plane that mentioned read replica had to sync with the primary instance.\n\nAt **09:17 UTC**, the maintenance initiated by Google Cloud SQL was completed. This lasted 10 minutes, and effectively meant our network control plane’s read-only database replica was unreachable.\n\nSome instances of our control plane started crashing when the database went offline due to improper error handling in its code. While they were configured to restart automatically, all routing information would be evicted from memory on each restart, and had to be re-fetched from Postgres which was offline.\n\nAt around **09:15 UTC**, we started seeing partial recovery from user reports, along with blips in our monitors which suggested the issue was intermittent. The healthy instances of our control plane continued serving traffic normally, so our networking was in a degraded state where a subset of user traffic may be routable, while others were not.\n\nAfter Postgres was back online, our control plane started to reboot itself successfully and resume normal operations. The boot process took longer than expected due to post-boot operations delaying it from being able to serve traffic (warming its local cache, etc.)\n\nAt **09:28 UTC**, we started seeing full recovery after the control plane and database became fully healthy and operational.\n\n## Preventative Measures\n\nThere are multiple parts of our control plane that should have gracefully degraded instead of failing over. \n\nWe have prioritized fixing this as an important project for 2025Q2. As part of this work, we’re going to create regional control planes instead of a global control plane, which will help us contain the blast radius of control plane failures, and allow us to mitigate any single-region failure by failing over route information lookup to other regions.\n\nOn top of improving the architecture of our control plane, we’re going to:\n\n- Fix the bug that causes the control plane to crash when it encounters a database connection error\n- Introduce a layer of persistent cache on routing information as a secondary fallback\n- Lower the start-up time of the control plane so we can recover from similar failures faster\n- Re-visit our Google Cloud SQL Postgres configuration to ensure its primary and replica aren’t under maintenance at the same time, and to notify us on upcoming maintenance(s) so we can monitor its progress and impact closely\n\n---\n\nRailway is committed to providing the best-in-class cloud experience. Any downtime is unacceptable for us. We apologize for any inconvenience caused by this, and we are going to work towards eliminating the entire class of issues contributing to this incident.","createdAt":"2026-06-09T12:15:06.870Z","description":"We recently experienced an outage that affected our networking control plane. During this outage, public and private networking on Railway was unavailable for a portion of users.","externalAuthor":false,"featured":false,"featuredImage":{"alt":"Incident Report: April 23rd, 2025 social image","height":null,"id":"375","mimeType":"image/png","url":"https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png","width":null},"id":"69","publishedAt":"2025-04-23T00:00:00.000Z","seoDescription":"We recently experienced an outage in our networking control plane that left public and private networking unavailable for a portion of users.","seoTitle":"Incident Report: April 23rd, 2025","slug":"incident-report-april-23-2025","socialImage":{"alt":"Incident Report: April 23rd, 2025 social image","height":null,"id":"375","mimeType":"image/png","url":"https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png","width":null},"title":"Incident Report: April 23rd, 2025","updatedAt":"2025-04-23T00:00:00.000Z"},"relatedPosts":[{"authors":[{"avatar":{"alt":"Mahmoud Abdelwahab headshot","height":460,"id":"1846","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","width":460},"avatarUrl":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","githubUrl":null,"id":"4","name":"Mahmoud Abdelwahab","slug":"mahmoud-abdelwahab","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":null,"createdAt":"2026-08-06T08:07:06.228Z","description":"Announcing: dev.new, an agent that can build, preview, test, and deploy an app from the same place. We tested the whole loop by asking it to ship a real-time multiplayer FPS game that runs in the browser","externalAuthor":false,"featured":true,"featuredImage":{"alt":"dev.new featured image","height":1440,"id":"1983","mimeType":"image/png","url":"https://cms.railway.com/media/dev.new-featured-image-4.png","width":3264},"id":"289","publishedAt":"2026-08-13T08:07:00.000Z","seoDescription":null,"seoTitle":null,"slug":"dev-new","socialImage":{"alt":"dev.new featured image 1","height":720,"id":"1976","mimeType":"image/png","url":"https://cms.railway.com/media/dev.new-featured-image-3.png","width":1632},"title":"Your Cloud should come with an app builder ","updatedAt":"2026-08-14T20:19:27.322Z"},{"authors":[{"avatar":{"alt":"Avatar of Victor Ramirez","height":null,"id":"1855","mimeType":"image/jpeg","url":"https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg","width":null},"avatarUrl":"https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg","githubUrl":null,"id":"34","name":"Victor Ramirez","slug":"victor-ramirez","title":null},{"avatar":{"alt":"Mahmoud Abdelwahab headshot","height":460,"id":"1846","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","width":460},"avatarUrl":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","githubUrl":null,"id":"4","name":"Mahmoud Abdelwahab","slug":"mahmoud-abdelwahab","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":null,"createdAt":"2026-07-15T22:26:48.036Z","description":"Merge now, release when you're ready: turn a feature on for your team, then 10% of users, then everyone. Feature flags are now built into every Railway project, and your agents can run the rollout from the CLI, SDK, and MCP. No third-party service required.","externalAuthor":false,"featured":false,"featuredImage":{"alt":"feature flags featured image","height":1440,"id":"1968","mimeType":"image/png","url":"https://cms.railway.com/media/feature-flags-featured-image.png","width":3264},"id":"278","publishedAt":"2026-07-17T07:00:00.000Z","seoDescription":null,"seoTitle":null,"slug":"feature-flags","socialImage":null,"title":"Roll it out, roll it back, never redeploy","updatedAt":"2026-08-13T16:53:55.126Z"}]},"__N_SSG":true},"page":"/p/[slug]","query":{"slug":"incident-report-april-23-2025"},"buildId":"nFX7R3M0MdLMAf-Ufh9qa","isFallback":false,"isExperimentalCompile":false,"gsp":true,"scriptLoader":[]}</script></body></html>