Files
nexus/sreweekly/articles/476/06-incident-report-april-30th-2025.html
2026-09-12 17:23:01 +08:00

266 lines
45 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8" data-next-head=""/><meta name="viewport" content="width=device-width" data-next-head=""/><meta name="twitter:card" content="summary_large_image" data-next-head=""/><meta name="twitter:site" content="@Railway" data-next-head=""/><meta name="twitter:creator" content="@Railway" data-next-head=""/><meta property="og:image:type" content="image/png" data-next-head=""/><meta property="og:image:width" content="1200" data-next-head=""/><meta property="og:image:height" content="630" data-next-head=""/><meta property="og:site_name" content="Railway Blog" data-next-head=""/><title data-next-head="">Incident Report: April 30th, 2025</title><meta name="robots" content="index,follow" data-next-head=""/><meta name="description" content="We recently experienced an outage that affected our backend API. During this outage, the Railway dashboard, CLI, and Public GraphQL API was unavailable." data-next-head=""/><meta property="og:title" content="Incident Report: April 30th, 2025" data-next-head=""/><meta property="og:description" content="We recently experienced an outage that affected our backend API. During this outage, the Railway dashboard, CLI, and Public GraphQL API was unavailable." data-next-head=""/><meta property="og:url" content="https://blog.railway.com/p/incident-report-april-30-2025" data-next-head=""/><meta property="og:type" content="article" data-next-head=""/><meta property="article:published_time" content="2025-04-30T00:00:00.000Z" data-next-head=""/><meta property="article:modified_time" content="2025-04-30T00:00:00.000Z" data-next-head=""/><meta property="article:author" content="Ray Chen" data-next-head=""/><meta property="article:section" content="Engineering" data-next-head=""/><meta property="og:image" content="https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png?w=1200&amp;q=90" data-next-head=""/><link rel="canonical" href="https://blog.railway.com/p/incident-report-april-30-2025" data-next-head=""/><meta name="twitter:title" content="Incident Report: April 30th, 2025" data-next-head=""/><meta name="twitter:description" content="We recently experienced an outage that affected our backend API. During this outage, the Railway dashboard, CLI, and Public GraphQL API was unavailable." data-next-head=""/><meta name="twitter:image" content="https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png?w=1200&amp;q=90" data-next-head=""/><link rel="alternate" type="text/markdown" href="/p/incident-report-april-30-2025.md" data-next-head=""/><link rel="alternate" type="application/rss+xml" title="Railway Blog" href="https://blog.railway.com/rss.xml"/><link rel="preload" href="/_next/static/media/e4af272ccee01ff0-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/91c3caac73720166-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/3359c9acc30ef440-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/aff755c8b4bb3089-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/3988c35203e0830a-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/427e4a37d3642943-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/css/619a01ff31a99955.css" as="style"/><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"BlogPosting","headline":"Incident Report: April 30th, 2025","description":"We recently experienced an outage that affected our backend API. During this outage, the Railway dashboard, CLI, and Public GraphQL API was unavailable.","image":["https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png?w=1200&q=90"],"datePublished":"2025-04-30T00:00:00.000Z","dateModified":"2025-04-30T00:00:00.000Z","author":{"@type":"Person","name":"Ray Chen"},"publisher":{"@type":"Organization","name":"Railway","logo":{"@type":"ImageObject","url":"https://railway.com/brand/logo-dark.png"}},"mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.railway.com/p/incident-report-april-30-2025"}}</script><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https://blog.railway.com"},{"@type":"ListItem","position":2,"name":"Engineering","item":"https://blog.railway.com/engineering"},{"@type":"ListItem","position":3,"name":"Incident Report: April 30th, 2025","item":"https://blog.railway.com/p/incident-report-april-30-2025"}]}</script><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"FAQPage","mainEntity":[{"@type":"Question","name":"What Happened?","acceptedAnswer":{"@type":"Answer","text":"Railway’s dashboard, CLI, and Public GraphQL API relies on our API backend. For example:"}}]}</script><style>:root {
--foreground: hsl(0, 0%, 100%);
--background: #181622;
--secondaryBg: hsl(250, 21%, 11%);
--blue-50: hsl(220, 55%, 10%);
--blue-100: hsl(220, 55%, 13%);
--blue-200: hsl(220, 62%, 25%);
--blue-300: hsl(220, 68%, 35%);
--blue-400: hsl(220, 72%, 45%);
--blue-500: hsl(220, 80%, 55%);
--blue-600: hsl(220, 80%, 65%);
--blue-700: hsl(220, 80%, 75%);
--blue-800: hsl(220, 80%, 85%);
--blue-900: hsl(220, 80%, 95%);
--blue-950: hsl(220, 55%, 97%);
--gray-50: hsl(248, 21%, 13%);
--gray-100: hsl(246, 18%, 15%);
--gray-200: hsl(246, 11%, 22%);
--gray-300: hsl(246, 8%, 35%);
--gray-400: hsl(246, 7%, 45%);
--gray-500: hsl(246, 6%, 55%);
--gray-600: hsl(246, 6%, 65%);
--gray-700: hsl(246, 6%, 78%);
--gray-800: hsl(246, 6%, 87%);
--gray-900: hsl(246, 6%, 95%);
--gray-950: hsl(246, 6%, 95%);
--red-50: hsl(1, 35%, 10%);
--red-100: hsl(1, 45%, 12%);
--red-200: hsl(1, 55%, 20%);
--red-300: hsl(1, 62%, 28%);
--red-400: hsl(1, 62%, 35%);
--red-500: hsl(1, 62%, 44%);
--red-600: hsl(1, 62%, 60%);
--red-700: hsl(1, 62%, 76%);
--red-800: hsl(1, 64%, 85%);
--red-900: hsl(1, 68%, 95%);
--red-950: hsl(1, 55%, 98%);
--pink-50: hsl(270, 40%, 16%);
--pink-100: hsl(270, 40%, 16%);
--pink-200: hsl(270, 45%, 24%);
--pink-300: hsl(270, 50%, 32%);
--pink-400: hsl(270, 55%, 43%);
--pink-500: hsl(270, 60%, 52%);
--pink-600: hsl(270, 70%, 65%);
--pink-700: hsl(270, 70%, 75%);
--pink-800: hsl(270, 70%, 85%);
--pink-900: hsl(270, 70%, 95%);
--pink-950: hsl(270, 70%, 95%);
--green-50: hsl(152, 15%, 10%);
--green-100: hsl(152, 26%, 11%);
--green-200: hsl(152, 32%, 16%);
--green-300: hsl(152, 38%, 24%);
--green-400: hsl(152, 38%, 34%);
--green-500: hsl(152, 38%, 42%);
--green-600: hsl(152, 38%, 60%);
--green-700: hsl(152, 38%, 70%);
--green-800: hsl(152, 38%, 80%);
--green-900: hsl(152, 38%, 91%);
--green-950: hsl(152, 40%, 97%);
--yellow-50: hsl(44, 95%, 12%);
--yellow-100: hsl(44, 95%, 12%);
--yellow-200: hsl(44, 96%, 24%);
--yellow-300: hsl(44, 95%, 36%);
--yellow-400: hsl(44, 95%, 48%);
--yellow-500: hsl(44, 95%, 60%);
--yellow-600: hsl(44, 95%, 69%);
--yellow-700: hsl(44, 95%, 78%);
--yellow-800: hsl(44, 95%, 86%);
--yellow-900: hsl(44, 95%, 95%);
--yellow-950: hsl(44, 95%, 95%);
}
.light {
--foreground: hsl(250, 24%, 9%);
--background: #F1F0EF;
--secondaryBg: hsl(0, 0%, 98%);
--blue-50: hsl(220, 55%, 97%);
--blue-100: hsl(220, 80%, 95%);
--blue-200: hsl(220, 80%, 85%);
--blue-300: hsl(220, 80%, 75%);
--blue-400: hsl(220, 80%, 65%);
--blue-500: hsl(220, 80%, 55%);
--blue-600: hsl(220, 72%, 45%);
--blue-700: hsl(220, 68%, 35%);
--blue-800: hsl(220, 62%, 25%);
--blue-900: hsl(220, 55%, 13%);
--blue-950: hsl(220, 55%, 10%);
--gray-50: hsl(246, 6%, 95%);
--gray-100: hsl(246, 6%, 95%);
--gray-200: hsl(246, 6%, 87%);
--gray-300: hsl(246, 6%, 78%);
--gray-400: hsl(246, 6%, 65%);
--gray-500: hsl(246, 6%, 55%);
--gray-600: hsl(246, 7%, 45%);
--gray-700: hsl(246, 8%, 35%);
--gray-800: hsl(246, 11%, 22%);
--gray-900: hsl(246, 18%, 15%);
--gray-950: hsl(248, 21%, 13%);
--red-50: hsl(1, 55%, 98%);
--red-100: hsl(1, 68%, 95%);
--red-200: hsl(1, 64%, 85%);
--red-300: hsl(1, 62%, 76%);
--red-400: hsl(1, 62%, 60%);
--red-500: hsl(1, 62%, 44%);
--red-600: hsl(1, 62%, 35%);
--red-700: hsl(1, 62%, 28%);
--red-800: hsl(1, 55%, 20%);
--red-900: hsl(1, 45%, 12%);
--red-950: hsl(1, 35%, 10%);
--pink-50: hsl(270, 70%, 95%);
--pink-100: hsl(270, 70%, 95%);
--pink-200: hsl(270, 70%, 85%);
--pink-300: hsl(270, 70%, 75%);
--pink-400: hsl(270, 70%, 65%);
--pink-500: hsl(270, 60%, 52%);
--pink-600: hsl(270, 55%, 43%);
--pink-700: hsl(270, 50%, 32%);
--pink-800: hsl(270, 45%, 24%);
--pink-900: hsl(270, 40%, 16%);
--pink-950: hsl(270, 40%, 16%);
--green-50: hsl(152, 40%, 97%);
--green-100: hsl(152, 38%, 91%);
--green-200: hsl(152, 38%, 80%);
--green-300: hsl(152, 38%, 70%);
--green-400: hsl(152, 38%, 60%);
--green-500: hsl(152, 38%, 42%);
--green-600: hsl(152, 38%, 34%);
--green-700: hsl(152, 38%, 24%);
--green-800: hsl(152, 32%, 16%);
--green-900: hsl(152, 26%, 11%);
--green-950: hsl(152, 15%, 10%);
--yellow-50: hsl(44, 95%, 95%);
--yellow-100: hsl(44, 95%, 95%);
--yellow-200: hsl(44, 95%, 86%);
--yellow-300: hsl(44, 95%, 78%);
--yellow-400: hsl(44, 95%, 69%);
--yellow-500: hsl(44, 95%, 60%);
--yellow-600: hsl(44, 95%, 48%);
--yellow-700: hsl(44, 95%, 36%);
--yellow-800: hsl(44, 96%, 24%);
--yellow-900: hsl(44, 95%, 12%);
--yellow-950: hsl(44, 95%, 12%);
}
.dark {
--foreground: hsl(0, 0%, 100%);
--background: #181622;
--secondaryBg: hsl(250, 21%, 11%);
--blue-50: hsl(220, 55%, 10%);
--blue-100: hsl(220, 55%, 13%);
--blue-200: hsl(220, 62%, 25%);
--blue-300: hsl(220, 68%, 35%);
--blue-400: hsl(220, 72%, 45%);
--blue-500: hsl(220, 80%, 55%);
--blue-600: hsl(220, 80%, 65%);
--blue-700: hsl(220, 80%, 75%);
--blue-800: hsl(220, 80%, 85%);
--blue-900: hsl(220, 80%, 95%);
--blue-950: hsl(220, 55%, 97%);
--gray-50: hsl(248, 21%, 13%);
--gray-100: hsl(246, 18%, 15%);
--gray-200: hsl(246, 11%, 22%);
--gray-300: hsl(246, 8%, 35%);
--gray-400: hsl(246, 7%, 45%);
--gray-500: hsl(246, 6%, 55%);
--gray-600: hsl(246, 6%, 65%);
--gray-700: hsl(246, 6%, 78%);
--gray-800: hsl(246, 6%, 87%);
--gray-900: hsl(246, 6%, 95%);
--gray-950: hsl(246, 6%, 95%);
--red-50: hsl(1, 35%, 10%);
--red-100: hsl(1, 45%, 12%);
--red-200: hsl(1, 55%, 20%);
--red-300: hsl(1, 62%, 28%);
--red-400: hsl(1, 62%, 35%);
--red-500: hsl(1, 62%, 44%);
--red-600: hsl(1, 62%, 60%);
--red-700: hsl(1, 62%, 76%);
--red-800: hsl(1, 64%, 85%);
--red-900: hsl(1, 68%, 95%);
--red-950: hsl(1, 55%, 98%);
--pink-50: hsl(270, 40%, 16%);
--pink-100: hsl(270, 40%, 16%);
--pink-200: hsl(270, 45%, 24%);
--pink-300: hsl(270, 50%, 32%);
--pink-400: hsl(270, 55%, 43%);
--pink-500: hsl(270, 60%, 52%);
--pink-600: hsl(270, 70%, 65%);
--pink-700: hsl(270, 70%, 75%);
--pink-800: hsl(270, 70%, 85%);
--pink-900: hsl(270, 70%, 95%);
--pink-950: hsl(270, 70%, 95%);
--green-50: hsl(152, 15%, 10%);
--green-100: hsl(152, 26%, 11%);
--green-200: hsl(152, 32%, 16%);
--green-300: hsl(152, 38%, 24%);
--green-400: hsl(152, 38%, 34%);
--green-500: hsl(152, 38%, 42%);
--green-600: hsl(152, 38%, 60%);
--green-700: hsl(152, 38%, 70%);
--green-800: hsl(152, 38%, 80%);
--green-900: hsl(152, 38%, 91%);
--green-950: hsl(152, 40%, 97%);
--yellow-50: hsl(44, 95%, 12%);
--yellow-100: hsl(44, 95%, 12%);
--yellow-200: hsl(44, 96%, 24%);
--yellow-300: hsl(44, 95%, 36%);
--yellow-400: hsl(44, 95%, 48%);
--yellow-500: hsl(44, 95%, 60%);
--yellow-600: hsl(44, 95%, 69%);
--yellow-700: hsl(44, 95%, 78%);
--yellow-800: hsl(44, 95%, 86%);
--yellow-900: hsl(44, 95%, 95%);
--yellow-950: hsl(44, 95%, 95%);
}</style><link rel="stylesheet" href="/_next/static/css/619a01ff31a99955.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" noModule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-0b474a604a067f92.js" defer=""></script><script src="/_next/static/chunks/framework-90fc9b0f7c4706c1.js" defer=""></script><script src="/_next/static/chunks/main-e89abd6570bbb9dc.js" defer=""></script><script src="/_next/static/chunks/pages/_app-3049b372678b389f.js" defer=""></script><script src="/_next/static/chunks/5399-d49efedd870f41de.js" defer=""></script><script src="/_next/static/chunks/3801-cdd2681d27083ad8.js" defer=""></script><script src="/_next/static/chunks/9594-1c44b12020a3af0a.js" defer=""></script><script src="/_next/static/chunks/4696-fd1712bc36a5bc50.js" defer=""></script><script src="/_next/static/chunks/pages/p/%5Bslug%5D-ddadb1ca54eaab0f.js" defer=""></script><script src="/_next/static/nFX7R3M0MdLMAf-Ufh9qa/_buildManifest.js" defer=""></script><script src="/_next/static/nFX7R3M0MdLMAf-Ufh9qa/_ssgManifest.js" defer=""></script></head><body><div id="__next"><script>!function(){try{var d=document.documentElement,c=d.classList;c.remove('light','dark');var e=localStorage.getItem('theme');if('system'===e||(!e&&true)){var t='(prefers-color-scheme: dark)',m=window.matchMedia(t);if(m.media!==t||m.matches){d.style.colorScheme = 'dark';c.add('dark')}else{d.style.colorScheme = 'light';c.add('light')}}else if(e){c.add(e|| '')}if(e==='light'||e==='dark')d.style.colorScheme=e}catch(e){}}()</script><div class="__variable_f367f3 __variable_0b668e __variable_ecea63 font-sans"><div class="px-5 md:px-8"><nav class="max-w-6xl mx-auto py-6 flex justify-between items-center border-b border-gray-100"><a class="flex items-center space-x-4" href="/"><svg data-v-423bf9ae="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 59.99945071644884 60" class="logo w-8"><g data-v-423bf9ae="" id="6f639fe5-02e1-4640-a1fb-f4b22b64e5ef" transform="matrix(0.5999641134746578,0,0,0.5999641134746578,-437.5538206497148,-266.68705118956507)" stroke="none"><path d="M729.872 487.327a50.86 50.86 0 0 0-.464 5.033h75.879c-.265-.518-.621-.985-.98-1.442-12.972-16.769-19.95-15.315-29.932-15.741-3.328-.137-5.585-.192-18.832-.192-7.09 0-14.798.018-22.304.038.737-1.746 1.6-3.419 2.525-5.061-2.885 5.106-4.865 10.771-5.805 16.789l.891-4.397c.007-.031.018-.063.024-.094h38.883v5.067h-39.885zM805.885 497.432h-76.438c.08 1.352.206 2.686.388 4.002h70.571c3.146 0 4.907-1.786 5.479-4.002zM733.851 515.257a52.226 52.226 0 0 1-1.98-4.997c6.608 19.89 25.328 34.251 47.433 34.251 20.205 0 37.566-12.007 45.452-29.254h-90.905zM729.38 492.915c-.018.531-.08 1.055-.08 1.589 0 .538.063 1.059.08 1.59v-3.179zM824.77 515.229z"></path><path d="M779.303 444.505c-18.682 0-34.939 10.265-43.524 25.439 6.709-.014 19.775-.022 19.775-.022h.003v-.005c15.444 0 16.018.069 19.035.195l1.868.069c6.507.217 14.505.916 20.798 5.68 3.416 2.584 8.348 8.287 11.288 12.35 2.718 3.758 3.5 8.078 1.652 12.217-1.701 3.804-5.361 6.073-9.793 6.073H730.85l-.884-4.201c.426 2.707 1.037 5.344 1.879 7.886h94.914a49.863 49.863 0 0 0 2.546-15.682c.001-27.611-22.386-49.999-50.002-49.999z"></path></g></svg><span class="text-xl font-semibold">Blog</span></a><div class="text-gray-600 flex items-center space-x-6"><a class="text-sm hover:text-pink-600" href="https://railway.com" target="_blank" rel="noreferrer noopener">Go to Homepage</a><a class="hover:text-pink-600" href="/rss.xml" title="RSS Feed"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4 11a9 9 0 0 1 9 9"></path><path d="M4 4a16 16 0 0 1 16 16"></path><circle cx="5" cy="19" r="1"></circle></svg></a></div></nav></div><div class="min-h-screen overflow-x-hidden relative"><div class="mt-10 mb-5 px-5 md:px-8 mx-auto"><article class="max-w-6xl mx-auto mt-24 mb-12 border-b border-gray-100 pb-32"><div class="max-w-[704px] mx-auto flex items-center text-sm text-gray-600 space-x-3"><div class="flex items-center space-x-3"><div class="flex items-center"><img src="https://cms.railway.com/media/person-ray-chen-dd0fa1297e94.jpg" alt="Avatar of Ray Chen" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/></div><span>Ray Chen</span></div><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><time dateTime="2025-04-30T00:00:00.000Z">Apr 30, 2025</time></div><header class="mt-5 mb-16 max-w-[704px] mx-auto"><h1 class="text-post-title font-medium font-serif">Incident Report: April 30th, 2025</h1></header><section class="max-w-[704px] mx-auto text-base sm:text-lg"><nav style="position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border-width:0" aria-label="Table of Contents"><h2>Table of Contents</h2><ol><li style="margin-left:1.5rem"><a href="#impact">Impact</a></li><li style="margin-left:1.5rem"><a href="#incident-timeline">Incident Timeline</a></li><li style="margin-left:1.5rem"><a href="#what-happened">What Happened?</a></li><li style="margin-left:1.5rem"><a href="#preventative-measures">Preventative Measures</a></li></ol></nav><div><p class="mb-4 text-gray-800">We recently experienced an outage that affected our API backend.</p>
<p class="mb-4 text-gray-800">When a Major Outage occurs, it is Railway’s policy to share the public details of what happened.</p>
<a class="relative no-underline" href="/p/incident-report-april-30-2025#impact"><span id="impact" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Impact</h2></a>
<p class="mb-4 text-gray-800">This outage impacted our API backend. You may have experienced connection errors if you were using the Railway dashboard (<a class="markdown-inline-link" href="https://railway.com" target="_blank" rel="noreferrer noopener">https://railway.com</a>), the CLI, or Railway’s Public GraphQL API during this period. GitHub-based deployments were also temporarily delayed.</p>
<p class="mb-4 text-gray-800">All running deployments and platform-level networking features remained online throughout this period.</p>
<a class="relative no-underline" href="/p/incident-report-april-30-2025#incident-timeline"><span id="incident-timeline" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Incident Timeline</h2></a>
<p class="mb-4 text-gray-800"><a class="markdown-inline-link" href="https://status.railway.com/cma4dbm5h004zt60jp9f8sr0s" target="_blank" rel="noreferrer noopener">Incident on our Status Page</a></p>
<ul class="list-disc pl-6 mb-4 space-y-4 text-gray-800">
<li><strong>Around 19:50 UTC</strong>: We noticed user reports that the Railway dashboard was not loading. Our health check monitors were also failing</li>
<li><strong>20:00 UTC</strong>: We called an incident declaring the dashboard to be Partially Degraded. Investigations into the underlying cause started immediately</li>
<li><strong>20:36 UTC</strong>: We identified some potential issues with recent configuration changes and rolled them back</li>
<li><strong>20:36 UTC to 22:11 UTC</strong>: We continued our efforts, focusing on full service restoration</li>
<li><strong>21:48 UTC</strong>: We noticed wider impact than initially anticipated, and upgraded the incident to a Major Outage</li>
<li><strong>22:20 UTC</strong>: We started seeing full recovery and resolved the incident</li>
<li><strong>22:24 UTC</strong>: Our health check monitors started failing again</li>
<li><strong>22:49 UTC to 23:23 UTC</strong>: We started rolling back additional configuration changes</li>
<li><strong>23:23 UTC:</strong> We started seeing signs of full recovery and continued monitoring user reports and metrics</li>
<li><strong>23:40 UTC:</strong> Incident resolved and full recovery confirmed</li>
</ul>
<a class="relative no-underline" href="/p/incident-report-april-30-2025#what-happened"><span id="what-happened" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">What Happened?</h2></a>
<p class="mb-4 text-gray-800">Railway’s dashboard, CLI, and Public GraphQL API relies on our API backend. For example:</p>
<ul class="list-disc pl-6 mb-4 space-y-4 text-gray-800">
<li>When you visit <a class="markdown-inline-link" href="https://railway.com/dashboard" target="_blank" rel="noreferrer noopener">https://railway.com/dashboard</a>, we load data from our API backend;</li>
<li>When you run a command (e.g. <code class="inline-code">railway up</code>) from the CLI, we issue requests to our API backend;</li>
<li>When you merge a PR on GitHub, our API backend processes webhooks from GitHub to know when to re-deploy your service with its latest changes.</li>
</ul>
<p class="mb-4 text-gray-800">In the past 3 months, we have been receiving increased amount of reports regarding dashboard latency issues, along with a few other latency-related errors.</p>
<p class="mb-4 text-gray-800">During this period, Railway’s rate of signups grew by over 5X. The load on our API backend increased significantly as a result of this growth. To address this, we started an initiative to reduce dashboard latency across the board — this is among our top priorities for Q2’2025.</p>
<p class="mb-4 text-gray-800">As part of this work, we made various configuration changes to our API backend’s infrastructure that runs on Kubernetes. We discovered a few issues after investigating the high latency reports:</p>
<ul class="list-disc pl-6 mb-4 space-y-4 text-gray-800">
<li>We were relying on a service mesh for internal proxy-ing to various other services, even adjacent ones such as our database connection pooler, which has been historically flaky in our experience</li>
<li>We have many unoptimized SQL queries that were causing unnecessary load on our database</li>
</ul>
<p class="mb-4 text-gray-800">To address the above, we shipped various configuration changes in the past week. The changes we rolled out unfortunately caused this outage.</p>
<p class="mb-4 text-gray-800">Our Kubernetes pods were repeatedly crashing with database connection errors some time after they successfully start up. This resulted in a loop where pods come up healthy and start serving traffic before eventually crashing after a few minutes — this is what happened between <strong>22:20 UTC</strong> and <strong>22:24 UTC</strong> where we thought the incident was resolved, but it actually was not.</p>
<p class="mb-4 text-gray-800">Because we modified our healthcheck, it passed without being healthy. The new healthchecks did not account for Postgres liveness, which meant we marked unhealthy instances as healthy, and overloaded our Postgres instance.</p>
<p class="mb-4 text-gray-800">After a few unsuccessful attempts at recovery, we decided to fully revert all changes that were made. Despite a full code revert, no recovery was observed.</p>
<p class="mb-4 text-gray-800">The database connection issues were traced down to pgbouncer pods (our Postgres connection pooler) that were reporting themselves to be healthy, but was unable to serve any traffic through the service mesh, because the service mesh failed to inject its routing information causing DNS resolution to fail.</p>
<p class="mb-4 text-gray-800">Recovery took longer than it should have because we did not know about the service mesh’s behaviour on DNS propagation to pods, and we were rolling back configuration changes selectively.</p>
<p class="mb-4 text-gray-800">Even after a full revert, we had to cycle all pods because the service mesh pushed DNS updates that broke other pods’ routing.</p>
<p class="mb-4 text-gray-800">At <strong>23:23 UTC</strong>, we confirmed the issue was fixed and subsequently resolved the incident at <strong>23:40 UTC</strong> after 17 minutes of cautiously monitoring user reports and metrics.</p>
<a class="relative no-underline" href="/p/incident-report-april-30-2025#preventative-measures"><span id="preventative-measures" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Preventative Measures</h2></a>
<p class="mb-4 text-gray-800">We’re going to continue working on the dashboard latency issues while exercising caution.</p>
<p class="mb-4 text-gray-800">To prevent this from happening again, we have made a few changes in how we rollout configuration changes:</p>
<ul class="list-disc pl-6 mb-4 space-y-4 text-gray-800">
<li>All configuration changes will be appropriately staggered (according to their risk) instead of being rolled out all at once in a short interval</li>
<li>Each configuration change we rollout will now have a rollback plan, and will be optimized for faster rollbacks in case something goes awry</li>
<li>If something appears to be off, we will revert full configuration changes immediately instead of selectively</li>
</ul>
<p class="mb-4 text-gray-800">We have also removed the service mesh entirely from our API backend, and we’re going to fully remove it from all critical paths. A service mesh is ultimately a highly dangerous single point of failure that can and should be replaced with simple load balancers.</p>
<hr class="my-8"/>
<p class="mb-4 text-gray-800">Railway is committed to providing the best-in-class cloud experience. Any downtime is unacceptable for us. We apologize for any inconvenience caused by this, and we are going to work towards eliminating the entire class of issues contributing to this incident.</p></div></section></article><div class="max-w-6xl mx-auto"><div><header class="flex items-center justify-between mb-8"><h3 class="text-gray-500 text-h3">Continue Reading...</h3><a class="text-pink-500 hover:underline" href="/engineering">View All <!-- -->Engineering<!-- --> -&gt;</a></header><div class="grid grid-cols-1 md:grid-cols-2 gap-8"><a class="flex flex-col bg-secondaryBg p-6 rounded-lg hover:bg-gray-100 group" href="/p/dev-new"><div class="flex gap-2"><div class="post-category-pill font-medium max-w-max text-xs px-2 py-[3px] rounded-[4px] tracking-[0.06em] uppercase" data-category-tone="plum">Engineering</div></div><div class="flex-grow"><header class="font-medium font-serif text-lg mt-2 mb-1">Your Cloud should come with an app builder </header><p class="text-base text-gray-800 line-clamp-2">Announcing: dev.new, an agent that can build, preview, test, and deploy an app from the same place. We tested the whole loop by asking it to ship a real-time multiplayer FPS game that runs in the browser</p></div><div class="flex items-center gap-3 mt-6"><div class="flex items-center"><img src="https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg" alt="Avatar of Mahmoud Abdelwahab" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/></div><span class="text-sm text-gray-600">Mahmoud Abdelwahab</span><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><span class="text-sm text-gray-600">Aug 13, 2026</span></div></a><a class="flex flex-col bg-secondaryBg p-6 rounded-lg hover:bg-gray-100 group" href="/p/feature-flags"><div class="flex gap-2"><div class="post-category-pill font-medium max-w-max text-xs px-2 py-[3px] rounded-[4px] tracking-[0.06em] uppercase" data-category-tone="plum">Engineering</div></div><div class="flex-grow"><header class="font-medium font-serif text-lg mt-2 mb-1">Roll it out, roll it back, never redeploy</header><p class="text-base text-gray-800 line-clamp-2">Merge now, release when you&#x27;re ready: turn a feature on for your team, then 10% of users, then everyone. Feature flags are now built into every Railway project, and your agents can run the rollout from the CLI, SDK, and MCP. No third-party service required.</p></div><div class="flex items-center gap-3 mt-6"><div class="flex items-center"><img src="https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg" alt="Avatar of Victor Ramirez" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/><img src="https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg" alt="Avatar of Mahmoud Abdelwahab" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:-8px" loading="lazy" decoding="async" width="24" height="24"/></div><span class="text-sm text-gray-600">Victor Ramirez &amp; Mahmoud Abdelwahab</span><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><span class="text-sm text-gray-600">Jul 17, 2026</span></div></a></div></div><section class="post-bottom-cta my-16 border rounded-[16px] px-6 py-16 sm:px-12 sm:py-24 flex flex-col items-center justify-center text-center" aria-labelledby="post-bottom-cta-title"><h3 id="post-bottom-cta-title" class="text-h2 font-serif font-medium">Ready to get started?</h3><p class="mt-4 text-base sm:text-lg text-gray-600">Join millions of developers deploying applications on Railway</p><div class="mt-8 flex w-full flex-col items-stretch justify-center gap-3 sm:w-auto sm:flex-row"><a class="post-bottom-cta-primary inline-flex min-h-[48px] items-center justify-center rounded-[8px] px-6 py-3 text-base font-medium no-underline transition-colors duration-100" href="https://railway.com" target="_blank" rel="noreferrer noopener">Deploy a new project</a><a class="post-bottom-cta-secondary inline-flex min-h-[48px] items-center justify-center rounded-[8px] border px-6 py-3 text-base font-medium no-underline transition-colors duration-100" href="https://railway.com/enterprise" target="_blank" rel="noreferrer noopener">Book a demo</a></div></section></div></div></div><footer class="px-5 md:px-8 py-12 border-t border-gray-100"><div class="grid grid-cols-2 sm:grid-cols-12 gap-8 sm:gap-4 lg:gap-16 max-w-6xl mx-auto"><div class="hidden sm:flex col-span-3 lg:col-span-4 flex-col justify-between"><a class="underline" href="https://railway.com" target="_blank" rel="noreferrer noopener"><svg data-v-423bf9ae="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 59.99945071644884 60" class="logo w-8"><g data-v-423bf9ae="" id="6f639fe5-02e1-4640-a1fb-f4b22b64e5ef" transform="matrix(0.5999641134746578,0,0,0.5999641134746578,-437.5538206497148,-266.68705118956507)" stroke="none"><path d="M729.872 487.327a50.86 50.86 0 0 0-.464 5.033h75.879c-.265-.518-.621-.985-.98-1.442-12.972-16.769-19.95-15.315-29.932-15.741-3.328-.137-5.585-.192-18.832-.192-7.09 0-14.798.018-22.304.038.737-1.746 1.6-3.419 2.525-5.061-2.885 5.106-4.865 10.771-5.805 16.789l.891-4.397c.007-.031.018-.063.024-.094h38.883v5.067h-39.885zM805.885 497.432h-76.438c.08 1.352.206 2.686.388 4.002h70.571c3.146 0 4.907-1.786 5.479-4.002zM733.851 515.257a52.226 52.226 0 0 1-1.98-4.997c6.608 19.89 25.328 34.251 47.433 34.251 20.205 0 37.566-12.007 45.452-29.254h-90.905zM729.38 492.915c-.018.531-.08 1.055-.08 1.589 0 .538.063 1.059.08 1.59v-3.179zM824.77 515.229z"></path><path d="M779.303 444.505c-18.682 0-34.939 10.265-43.524 25.439 6.709-.014 19.775-.022 19.775-.022h.003v-.005c15.444 0 16.018.069 19.035.195l1.868.069c6.507.217 14.505.916 20.798 5.68 3.416 2.584 8.348 8.287 11.288 12.35 2.718 3.758 3.5 8.078 1.652 12.217-1.701 3.804-5.361 6.073-9.793 6.073H730.85l-.884-4.201c.426 2.707 1.037 5.344 1.879 7.886h94.914a49.863 49.863 0 0 0 2.546-15.682c.001-27.611-22.386-49.999-50.002-49.999z"></path></g></svg></a><div class="text-xs font-medium text-gray-600 w-full">Copyright © <!-- -->2026<!-- --> Railway Corp. <br/>All rights reserved.</div></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Product</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/changelog" target="_blank" rel="noreferrer noopener">Changelog</a></li><li><a class="hover:text-foreground" href="https://railway.com/pricing" target="_blank" rel="noreferrer noopener">Pricing</a></li><li><a class="hover:text-foreground" href="https://railway.com/deploy" target="_blank" rel="noreferrer noopener">Templates</a></li><li><a class="hover:text-foreground" href="https://station.railway.com/feedback" target="_blank" rel="noreferrer noopener">Feedback</a></li><li><a class="hover:text-foreground" href="https://railway.com/open-source-kickback" target="_blank" rel="noreferrer noopener">OSS Kickback</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Company</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/about" target="_blank" rel="noreferrer noopener">About</a></li><li><a class="hover:text-foreground" href="https://railway.com/careers" target="_blank" rel="noreferrer noopener">Careers</a></li><li><a class="hover:text-foreground" href="/">Blog</a></li><li><a class="hover:text-foreground" href="https://shop.railway.com" target="_blank" rel="noreferrer noopener">Shop</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Contact</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://discord.gg/railway" target="_blank" rel="noreferrer noopener">Discord</a></li><li><a class="hover:text-foreground" href="https://twitter.com/railway" target="_blank" rel="noreferrer noopener">Twitter</a></li><li><a class="hover:text-foreground" href="https://github.com/railwayapp" target="_blank" rel="noreferrer noopener">GitHub</a></li><li><a class="hover:text-foreground" href="mailto:contact@railway.com">Email</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Legal</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/legal/acceptable-use" target="_blank" rel="noreferrer noopener">Acceptable Use</a></li><li><a class="hover:text-foreground" href="https://railway.com/legal/privacy" target="_blank" rel="noreferrer noopener">Privacy Policy</a></li><li><a class="hover:text-foreground" href="https://railway.com/legal/terms" target="_blank" rel="noreferrer noopener">Terms of Service</a></li></ul></div></div></footer></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"page":{"authors":[{"avatar":{"alt":"Ray Chen headshot","height":1096,"id":"1853","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-ray-chen-dd0fa1297e94.jpg","width":1152},"avatarUrl":"https://cms.railway.com/media/person-ray-chen-dd0fa1297e94.jpg","githubUrl":null,"id":"29","name":"Ray Chen","slug":"ray-chen","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":"We recently experienced an outage that affected our API backend.\n\nWhen a Major Outage occurs, it is Railway’s policy to share the public details of what happened.\n\n## Impact\n\nThis outage impacted our API backend. You may have experienced connection errors if you were using the Railway dashboard ([https://railway.com](https://railway.com)), the CLI, or Railway’s Public GraphQL API during this period. GitHub-based deployments were also temporarily delayed.\n\nAll running deployments and platform-level networking features remained online throughout this period.\n\n## Incident Timeline\n\n[Incident on our Status Page](https://status.railway.com/cma4dbm5h004zt60jp9f8sr0s)\n\n- **Around 19:50 UTC**: We noticed user reports that the Railway dashboard was not loading. Our health check monitors were also failing\n- **20:00 UTC**: We called an incident declaring the dashboard to be Partially Degraded. Investigations into the underlying cause started immediately\n- **20:36 UTC**: We identified some potential issues with recent configuration changes and rolled them back\n- **20:36 UTC to 22:11 UTC**: We continued our efforts, focusing on full service restoration\n- **21:48 UTC**: We noticed wider impact than initially anticipated, and upgraded the incident to a Major Outage\n- **22:20 UTC**: We started seeing full recovery and resolved the incident\n- **22:24 UTC**: Our health check monitors started failing again\n- **22:49 UTC to 23:23 UTC**: We started rolling back additional configuration changes\n- **23:23 UTC:** We started seeing signs of full recovery and continued monitoring user reports and metrics\n- **23:40 UTC:** Incident resolved and full recovery confirmed\n\n## What Happened?\n\nRailway’s dashboard, CLI, and Public GraphQL API relies on our API backend. For example:\n\n- When you visit [https://railway.com/dashboard](https://railway.com/dashboard), we load data from our API backend;\n- When you run a command (e.g. `railway up`) from the CLI, we issue requests to our API backend;\n- When you merge a PR on GitHub, our API backend processes webhooks from GitHub to know when to re-deploy your service with its latest changes.\n\nIn the past 3 months, we have been receiving increased amount of reports regarding dashboard latency issues, along with a few other latency-related errors.\n\nDuring this period, Railway’s rate of signups grew by over 5X. The load on our API backend increased significantly as a result of this growth. To address this, we started an initiative to reduce dashboard latency across the board — this is among our top priorities for Q2’2025.\n\nAs part of this work, we made various configuration changes to our API backend’s infrastructure that runs on Kubernetes. We discovered a few issues after investigating the high latency reports:\n\n- We were relying on a service mesh for internal proxy-ing to various other services, even adjacent ones such as our database connection pooler, which has been historically flaky in our experience\n- We have many unoptimized SQL queries that were causing unnecessary load on our database\n\nTo address the above, we shipped various configuration changes in the past week. The changes we rolled out unfortunately caused this outage.\n\nOur Kubernetes pods were repeatedly crashing with database connection errors some time after they successfully start up. This resulted in a loop where pods come up healthy and start serving traffic before eventually crashing after a few minutes — this is what happened between **22:20 UTC** and **22:24 UTC** where we thought the incident was resolved, but it actually was not.\n\nBecause we modified our healthcheck, it passed without being healthy. The new healthchecks did not account for Postgres liveness, which meant we marked unhealthy instances as healthy, and overloaded our Postgres instance.\n\nAfter a few unsuccessful attempts at recovery, we decided to fully revert all changes that were made. Despite a full code revert, no recovery was observed.\n\nThe database connection issues were traced down to pgbouncer pods (our Postgres connection pooler) that were reporting themselves to be healthy, but was unable to serve any traffic through the service mesh, because the service mesh failed to inject its routing information causing DNS resolution to fail.\n\nRecovery took longer than it should have because we did not know about the service mesh’s behaviour on DNS propagation to pods, and we were rolling back configuration changes selectively.\n\nEven after a full revert, we had to cycle all pods because the service mesh pushed DNS updates that broke other pods’ routing.\n\nAt **23:23 UTC**, we confirmed the issue was fixed and subsequently resolved the incident at **23:40 UTC** after 17 minutes of cautiously monitoring user reports and metrics.\n\n## Preventative Measures\n\nWe’re going to continue working on the dashboard latency issues while exercising caution.\n\nTo prevent this from happening again, we have made a few changes in how we rollout configuration changes:\n\n- All configuration changes will be appropriately staggered (according to their risk) instead of being rolled out all at once in a short interval\n- Each configuration change we rollout will now have a rollback plan, and will be optimized for faster rollbacks in case something goes awry\n- If something appears to be off, we will revert full configuration changes immediately instead of selectively\n\nWe have also removed the service mesh entirely from our API backend, and we’re going to fully remove it from all critical paths. A service mesh is ultimately a highly dangerous single point of failure that can and should be replaced with simple load balancers.\n\n---\n\nRailway is committed to providing the best-in-class cloud experience. Any downtime is unacceptable for us. We apologize for any inconvenience caused by this, and we are going to work towards eliminating the entire class of issues contributing to this incident.","createdAt":"2026-06-09T12:15:06.926Z","description":"We recently experienced an outage that affected our backend API. During this outage, the Railway dashboard, CLI, and Public GraphQL API was unavailable.","externalAuthor":false,"featured":false,"featuredImage":{"alt":"Incident Report: April 23rd, 2025 social image","height":null,"id":"375","mimeType":"image/png","url":"https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png","width":null},"id":"70","publishedAt":"2025-04-30T00:00:00.000Z","seoDescription":"We recently experienced an outage that affected our backend API. During this outage, the Railway dashboard, CLI, and Public GraphQL API was unavailable.","seoTitle":"Incident Report: April 30th, 2025","slug":"incident-report-april-30-2025","socialImage":{"alt":"Incident Report: April 23rd, 2025 social image","height":null,"id":"375","mimeType":"image/png","url":"https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png","width":null},"title":"Incident Report: April 30th, 2025","updatedAt":"2025-04-30T00:00:00.000Z"},"relatedPosts":[{"authors":[{"avatar":{"alt":"Mahmoud Abdelwahab headshot","height":460,"id":"1846","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","width":460},"avatarUrl":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","githubUrl":null,"id":"4","name":"Mahmoud Abdelwahab","slug":"mahmoud-abdelwahab","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":null,"createdAt":"2026-08-06T08:07:06.228Z","description":"Announcing: dev.new, an agent that can build, preview, test, and deploy an app from the same place. We tested the whole loop by asking it to ship a real-time multiplayer FPS game that runs in the browser","externalAuthor":false,"featured":true,"featuredImage":{"alt":"dev.new featured image","height":1440,"id":"1983","mimeType":"image/png","url":"https://cms.railway.com/media/dev.new-featured-image-4.png","width":3264},"id":"289","publishedAt":"2026-08-13T08:07:00.000Z","seoDescription":null,"seoTitle":null,"slug":"dev-new","socialImage":{"alt":"dev.new featured image 1","height":720,"id":"1976","mimeType":"image/png","url":"https://cms.railway.com/media/dev.new-featured-image-3.png","width":1632},"title":"Your Cloud should come with an app builder ","updatedAt":"2026-08-14T20:19:27.322Z"},{"authors":[{"avatar":{"alt":"Avatar of Victor Ramirez","height":null,"id":"1855","mimeType":"image/jpeg","url":"https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg","width":null},"avatarUrl":"https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg","githubUrl":null,"id":"34","name":"Victor Ramirez","slug":"victor-ramirez","title":null},{"avatar":{"alt":"Mahmoud Abdelwahab headshot","height":460,"id":"1846","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","width":460},"avatarUrl":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","githubUrl":null,"id":"4","name":"Mahmoud Abdelwahab","slug":"mahmoud-abdelwahab","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":null,"createdAt":"2026-07-15T22:26:48.036Z","description":"Merge now, release when you're ready: turn a feature on for your team, then 10% of users, then everyone. Feature flags are now built into every Railway project, and your agents can run the rollout from the CLI, SDK, and MCP. No third-party service required.","externalAuthor":false,"featured":false,"featuredImage":{"alt":"feature flags featured image","height":1440,"id":"1968","mimeType":"image/png","url":"https://cms.railway.com/media/feature-flags-featured-image.png","width":3264},"id":"278","publishedAt":"2026-07-17T07:00:00.000Z","seoDescription":null,"seoTitle":null,"slug":"feature-flags","socialImage":null,"title":"Roll it out, roll it back, never redeploy","updatedAt":"2026-08-13T16:53:55.126Z"}]},"__N_SSG":true},"page":"/p/[slug]","query":{"slug":"incident-report-april-30-2025"},"buildId":"nFX7R3M0MdLMAf-Ufh9qa","isFallback":false,"isExperimentalCompile":false,"gsp":true,"scriptLoader":[]}</script></body></html>