Files
nexus/sreweekly/articles/497/07-incident-report-september-22nd-2025.html
2026-09-12 17:23:01 +08:00

269 lines
51 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8" data-next-head=""/><meta name="viewport" content="width=device-width" data-next-head=""/><meta name="twitter:card" content="summary_large_image" data-next-head=""/><meta name="twitter:site" content="@Railway" data-next-head=""/><meta name="twitter:creator" content="@Railway" data-next-head=""/><meta property="og:image:type" content="image/png" data-next-head=""/><meta property="og:image:width" content="1200" data-next-head=""/><meta property="og:image:height" content="630" data-next-head=""/><meta property="og:site_name" content="Railway Blog" data-next-head=""/><title data-next-head="">Incident Report: September 22nd, 2025</title><meta name="robots" content="index,follow" data-next-head=""/><meta name="description" content="We experienced an outage affecting our dashboard and deployment pipeline, resulting in a temporary suspension of deployments across all user tiers." data-next-head=""/><meta property="og:title" content="Incident Report: September 22nd, 2025" data-next-head=""/><meta property="og:description" content="We experienced an outage affecting our dashboard and deployment pipeline, resulting in a temporary suspension of deployments across all user tiers." data-next-head=""/><meta property="og:url" content="https://blog.railway.com/p/incident-report-sept-22-2025" data-next-head=""/><meta property="og:type" content="article" data-next-head=""/><meta property="article:published_time" content="2025-09-22T00:00:00.000Z" data-next-head=""/><meta property="article:modified_time" content="2025-09-22T00:00:00.000Z" data-next-head=""/><meta property="article:author" content="Ray Chen" data-next-head=""/><meta property="article:section" content="Engineering" data-next-head=""/><meta property="og:image" content="https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png?w=1200&amp;q=90" data-next-head=""/><link rel="canonical" href="https://blog.railway.com/p/incident-report-sept-22-2025" data-next-head=""/><meta name="twitter:title" content="Incident Report: September 22nd, 2025" data-next-head=""/><meta name="twitter:description" content="We experienced an outage affecting our dashboard and deployment pipeline, resulting in a temporary suspension of deployments across all user tiers." data-next-head=""/><meta name="twitter:image" content="https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png?w=1200&amp;q=90" data-next-head=""/><link rel="alternate" type="text/markdown" href="/p/incident-report-sept-22-2025.md" data-next-head=""/><link rel="alternate" type="application/rss+xml" title="Railway Blog" href="https://blog.railway.com/rss.xml"/><link rel="preload" href="/_next/static/media/e4af272ccee01ff0-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/91c3caac73720166-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/3359c9acc30ef440-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/aff755c8b4bb3089-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/3988c35203e0830a-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/427e4a37d3642943-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/css/619a01ff31a99955.css" as="style"/><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"BlogPosting","headline":"Incident Report: September 22nd, 2025","description":"We experienced an outage affecting our dashboard and deployment pipeline, resulting in a temporary suspension of deployments across all user tiers.","image":["https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png?w=1200&q=90"],"datePublished":"2025-09-22T00:00:00.000Z","dateModified":"2025-09-22T00:00:00.000Z","author":{"@type":"Person","name":"Ray Chen"},"publisher":{"@type":"Organization","name":"Railway","logo":{"@type":"ImageObject","url":"https://railway.com/brand/logo-dark.png"}},"mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.railway.com/p/incident-report-sept-22-2025"}}</script><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https://blog.railway.com"},{"@type":"ListItem","position":2,"name":"Engineering","item":"https://blog.railway.com/engineering"},{"@type":"ListItem","position":3,"name":"Incident Report: September 22nd, 2025","item":"https://blog.railway.com/p/incident-report-sept-22-2025"}]}</script><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"FAQPage","mainEntity":[{"@type":"Question","name":"What Happened?","acceptedAnswer":{"@type":"Answer","text":"The Railway dashboard and project canvas relies on our API to fetch data. During a code cleanup, we made a database schema change to remove an unused column. However, a subsequent code change reverted part of this unintentionally, leading to failures for a core API our frontend relies on."}}]}</script><style>:root {
--foreground: hsl(0, 0%, 100%);
--background: #181622;
--secondaryBg: hsl(250, 21%, 11%);
--blue-50: hsl(220, 55%, 10%);
--blue-100: hsl(220, 55%, 13%);
--blue-200: hsl(220, 62%, 25%);
--blue-300: hsl(220, 68%, 35%);
--blue-400: hsl(220, 72%, 45%);
--blue-500: hsl(220, 80%, 55%);
--blue-600: hsl(220, 80%, 65%);
--blue-700: hsl(220, 80%, 75%);
--blue-800: hsl(220, 80%, 85%);
--blue-900: hsl(220, 80%, 95%);
--blue-950: hsl(220, 55%, 97%);
--gray-50: hsl(248, 21%, 13%);
--gray-100: hsl(246, 18%, 15%);
--gray-200: hsl(246, 11%, 22%);
--gray-300: hsl(246, 8%, 35%);
--gray-400: hsl(246, 7%, 45%);
--gray-500: hsl(246, 6%, 55%);
--gray-600: hsl(246, 6%, 65%);
--gray-700: hsl(246, 6%, 78%);
--gray-800: hsl(246, 6%, 87%);
--gray-900: hsl(246, 6%, 95%);
--gray-950: hsl(246, 6%, 95%);
--red-50: hsl(1, 35%, 10%);
--red-100: hsl(1, 45%, 12%);
--red-200: hsl(1, 55%, 20%);
--red-300: hsl(1, 62%, 28%);
--red-400: hsl(1, 62%, 35%);
--red-500: hsl(1, 62%, 44%);
--red-600: hsl(1, 62%, 60%);
--red-700: hsl(1, 62%, 76%);
--red-800: hsl(1, 64%, 85%);
--red-900: hsl(1, 68%, 95%);
--red-950: hsl(1, 55%, 98%);
--pink-50: hsl(270, 40%, 16%);
--pink-100: hsl(270, 40%, 16%);
--pink-200: hsl(270, 45%, 24%);
--pink-300: hsl(270, 50%, 32%);
--pink-400: hsl(270, 55%, 43%);
--pink-500: hsl(270, 60%, 52%);
--pink-600: hsl(270, 70%, 65%);
--pink-700: hsl(270, 70%, 75%);
--pink-800: hsl(270, 70%, 85%);
--pink-900: hsl(270, 70%, 95%);
--pink-950: hsl(270, 70%, 95%);
--green-50: hsl(152, 15%, 10%);
--green-100: hsl(152, 26%, 11%);
--green-200: hsl(152, 32%, 16%);
--green-300: hsl(152, 38%, 24%);
--green-400: hsl(152, 38%, 34%);
--green-500: hsl(152, 38%, 42%);
--green-600: hsl(152, 38%, 60%);
--green-700: hsl(152, 38%, 70%);
--green-800: hsl(152, 38%, 80%);
--green-900: hsl(152, 38%, 91%);
--green-950: hsl(152, 40%, 97%);
--yellow-50: hsl(44, 95%, 12%);
--yellow-100: hsl(44, 95%, 12%);
--yellow-200: hsl(44, 96%, 24%);
--yellow-300: hsl(44, 95%, 36%);
--yellow-400: hsl(44, 95%, 48%);
--yellow-500: hsl(44, 95%, 60%);
--yellow-600: hsl(44, 95%, 69%);
--yellow-700: hsl(44, 95%, 78%);
--yellow-800: hsl(44, 95%, 86%);
--yellow-900: hsl(44, 95%, 95%);
--yellow-950: hsl(44, 95%, 95%);
}
.light {
--foreground: hsl(250, 24%, 9%);
--background: #F1F0EF;
--secondaryBg: hsl(0, 0%, 98%);
--blue-50: hsl(220, 55%, 97%);
--blue-100: hsl(220, 80%, 95%);
--blue-200: hsl(220, 80%, 85%);
--blue-300: hsl(220, 80%, 75%);
--blue-400: hsl(220, 80%, 65%);
--blue-500: hsl(220, 80%, 55%);
--blue-600: hsl(220, 72%, 45%);
--blue-700: hsl(220, 68%, 35%);
--blue-800: hsl(220, 62%, 25%);
--blue-900: hsl(220, 55%, 13%);
--blue-950: hsl(220, 55%, 10%);
--gray-50: hsl(246, 6%, 95%);
--gray-100: hsl(246, 6%, 95%);
--gray-200: hsl(246, 6%, 87%);
--gray-300: hsl(246, 6%, 78%);
--gray-400: hsl(246, 6%, 65%);
--gray-500: hsl(246, 6%, 55%);
--gray-600: hsl(246, 7%, 45%);
--gray-700: hsl(246, 8%, 35%);
--gray-800: hsl(246, 11%, 22%);
--gray-900: hsl(246, 18%, 15%);
--gray-950: hsl(248, 21%, 13%);
--red-50: hsl(1, 55%, 98%);
--red-100: hsl(1, 68%, 95%);
--red-200: hsl(1, 64%, 85%);
--red-300: hsl(1, 62%, 76%);
--red-400: hsl(1, 62%, 60%);
--red-500: hsl(1, 62%, 44%);
--red-600: hsl(1, 62%, 35%);
--red-700: hsl(1, 62%, 28%);
--red-800: hsl(1, 55%, 20%);
--red-900: hsl(1, 45%, 12%);
--red-950: hsl(1, 35%, 10%);
--pink-50: hsl(270, 70%, 95%);
--pink-100: hsl(270, 70%, 95%);
--pink-200: hsl(270, 70%, 85%);
--pink-300: hsl(270, 70%, 75%);
--pink-400: hsl(270, 70%, 65%);
--pink-500: hsl(270, 60%, 52%);
--pink-600: hsl(270, 55%, 43%);
--pink-700: hsl(270, 50%, 32%);
--pink-800: hsl(270, 45%, 24%);
--pink-900: hsl(270, 40%, 16%);
--pink-950: hsl(270, 40%, 16%);
--green-50: hsl(152, 40%, 97%);
--green-100: hsl(152, 38%, 91%);
--green-200: hsl(152, 38%, 80%);
--green-300: hsl(152, 38%, 70%);
--green-400: hsl(152, 38%, 60%);
--green-500: hsl(152, 38%, 42%);
--green-600: hsl(152, 38%, 34%);
--green-700: hsl(152, 38%, 24%);
--green-800: hsl(152, 32%, 16%);
--green-900: hsl(152, 26%, 11%);
--green-950: hsl(152, 15%, 10%);
--yellow-50: hsl(44, 95%, 95%);
--yellow-100: hsl(44, 95%, 95%);
--yellow-200: hsl(44, 95%, 86%);
--yellow-300: hsl(44, 95%, 78%);
--yellow-400: hsl(44, 95%, 69%);
--yellow-500: hsl(44, 95%, 60%);
--yellow-600: hsl(44, 95%, 48%);
--yellow-700: hsl(44, 95%, 36%);
--yellow-800: hsl(44, 96%, 24%);
--yellow-900: hsl(44, 95%, 12%);
--yellow-950: hsl(44, 95%, 12%);
}
.dark {
--foreground: hsl(0, 0%, 100%);
--background: #181622;
--secondaryBg: hsl(250, 21%, 11%);
--blue-50: hsl(220, 55%, 10%);
--blue-100: hsl(220, 55%, 13%);
--blue-200: hsl(220, 62%, 25%);
--blue-300: hsl(220, 68%, 35%);
--blue-400: hsl(220, 72%, 45%);
--blue-500: hsl(220, 80%, 55%);
--blue-600: hsl(220, 80%, 65%);
--blue-700: hsl(220, 80%, 75%);
--blue-800: hsl(220, 80%, 85%);
--blue-900: hsl(220, 80%, 95%);
--blue-950: hsl(220, 55%, 97%);
--gray-50: hsl(248, 21%, 13%);
--gray-100: hsl(246, 18%, 15%);
--gray-200: hsl(246, 11%, 22%);
--gray-300: hsl(246, 8%, 35%);
--gray-400: hsl(246, 7%, 45%);
--gray-500: hsl(246, 6%, 55%);
--gray-600: hsl(246, 6%, 65%);
--gray-700: hsl(246, 6%, 78%);
--gray-800: hsl(246, 6%, 87%);
--gray-900: hsl(246, 6%, 95%);
--gray-950: hsl(246, 6%, 95%);
--red-50: hsl(1, 35%, 10%);
--red-100: hsl(1, 45%, 12%);
--red-200: hsl(1, 55%, 20%);
--red-300: hsl(1, 62%, 28%);
--red-400: hsl(1, 62%, 35%);
--red-500: hsl(1, 62%, 44%);
--red-600: hsl(1, 62%, 60%);
--red-700: hsl(1, 62%, 76%);
--red-800: hsl(1, 64%, 85%);
--red-900: hsl(1, 68%, 95%);
--red-950: hsl(1, 55%, 98%);
--pink-50: hsl(270, 40%, 16%);
--pink-100: hsl(270, 40%, 16%);
--pink-200: hsl(270, 45%, 24%);
--pink-300: hsl(270, 50%, 32%);
--pink-400: hsl(270, 55%, 43%);
--pink-500: hsl(270, 60%, 52%);
--pink-600: hsl(270, 70%, 65%);
--pink-700: hsl(270, 70%, 75%);
--pink-800: hsl(270, 70%, 85%);
--pink-900: hsl(270, 70%, 95%);
--pink-950: hsl(270, 70%, 95%);
--green-50: hsl(152, 15%, 10%);
--green-100: hsl(152, 26%, 11%);
--green-200: hsl(152, 32%, 16%);
--green-300: hsl(152, 38%, 24%);
--green-400: hsl(152, 38%, 34%);
--green-500: hsl(152, 38%, 42%);
--green-600: hsl(152, 38%, 60%);
--green-700: hsl(152, 38%, 70%);
--green-800: hsl(152, 38%, 80%);
--green-900: hsl(152, 38%, 91%);
--green-950: hsl(152, 40%, 97%);
--yellow-50: hsl(44, 95%, 12%);
--yellow-100: hsl(44, 95%, 12%);
--yellow-200: hsl(44, 96%, 24%);
--yellow-300: hsl(44, 95%, 36%);
--yellow-400: hsl(44, 95%, 48%);
--yellow-500: hsl(44, 95%, 60%);
--yellow-600: hsl(44, 95%, 69%);
--yellow-700: hsl(44, 95%, 78%);
--yellow-800: hsl(44, 95%, 86%);
--yellow-900: hsl(44, 95%, 95%);
--yellow-950: hsl(44, 95%, 95%);
}</style><link rel="stylesheet" href="/_next/static/css/619a01ff31a99955.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" noModule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-0b474a604a067f92.js" defer=""></script><script src="/_next/static/chunks/framework-90fc9b0f7c4706c1.js" defer=""></script><script src="/_next/static/chunks/main-e89abd6570bbb9dc.js" defer=""></script><script src="/_next/static/chunks/pages/_app-3049b372678b389f.js" defer=""></script><script src="/_next/static/chunks/5399-d49efedd870f41de.js" defer=""></script><script src="/_next/static/chunks/3801-cdd2681d27083ad8.js" defer=""></script><script src="/_next/static/chunks/9594-1c44b12020a3af0a.js" defer=""></script><script src="/_next/static/chunks/4696-fd1712bc36a5bc50.js" defer=""></script><script src="/_next/static/chunks/pages/p/%5Bslug%5D-ddadb1ca54eaab0f.js" defer=""></script><script src="/_next/static/nFX7R3M0MdLMAf-Ufh9qa/_buildManifest.js" defer=""></script><script src="/_next/static/nFX7R3M0MdLMAf-Ufh9qa/_ssgManifest.js" defer=""></script></head><body><div id="__next"><script>!function(){try{var d=document.documentElement,c=d.classList;c.remove('light','dark');var e=localStorage.getItem('theme');if('system'===e||(!e&&true)){var t='(prefers-color-scheme: dark)',m=window.matchMedia(t);if(m.media!==t||m.matches){d.style.colorScheme = 'dark';c.add('dark')}else{d.style.colorScheme = 'light';c.add('light')}}else if(e){c.add(e|| '')}if(e==='light'||e==='dark')d.style.colorScheme=e}catch(e){}}()</script><div class="__variable_f367f3 __variable_0b668e __variable_ecea63 font-sans"><div class="px-5 md:px-8"><nav class="max-w-6xl mx-auto py-6 flex justify-between items-center border-b border-gray-100"><a class="flex items-center space-x-4" href="/"><svg data-v-423bf9ae="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 59.99945071644884 60" class="logo w-8"><g data-v-423bf9ae="" id="6f639fe5-02e1-4640-a1fb-f4b22b64e5ef" transform="matrix(0.5999641134746578,0,0,0.5999641134746578,-437.5538206497148,-266.68705118956507)" stroke="none"><path d="M729.872 487.327a50.86 50.86 0 0 0-.464 5.033h75.879c-.265-.518-.621-.985-.98-1.442-12.972-16.769-19.95-15.315-29.932-15.741-3.328-.137-5.585-.192-18.832-.192-7.09 0-14.798.018-22.304.038.737-1.746 1.6-3.419 2.525-5.061-2.885 5.106-4.865 10.771-5.805 16.789l.891-4.397c.007-.031.018-.063.024-.094h38.883v5.067h-39.885zM805.885 497.432h-76.438c.08 1.352.206 2.686.388 4.002h70.571c3.146 0 4.907-1.786 5.479-4.002zM733.851 515.257a52.226 52.226 0 0 1-1.98-4.997c6.608 19.89 25.328 34.251 47.433 34.251 20.205 0 37.566-12.007 45.452-29.254h-90.905zM729.38 492.915c-.018.531-.08 1.055-.08 1.589 0 .538.063 1.059.08 1.59v-3.179zM824.77 515.229z"></path><path d="M779.303 444.505c-18.682 0-34.939 10.265-43.524 25.439 6.709-.014 19.775-.022 19.775-.022h.003v-.005c15.444 0 16.018.069 19.035.195l1.868.069c6.507.217 14.505.916 20.798 5.68 3.416 2.584 8.348 8.287 11.288 12.35 2.718 3.758 3.5 8.078 1.652 12.217-1.701 3.804-5.361 6.073-9.793 6.073H730.85l-.884-4.201c.426 2.707 1.037 5.344 1.879 7.886h94.914a49.863 49.863 0 0 0 2.546-15.682c.001-27.611-22.386-49.999-50.002-49.999z"></path></g></svg><span class="text-xl font-semibold">Blog</span></a><div class="text-gray-600 flex items-center space-x-6"><a class="text-sm hover:text-pink-600" href="https://railway.com" target="_blank" rel="noreferrer noopener">Go to Homepage</a><a class="hover:text-pink-600" href="/rss.xml" title="RSS Feed"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4 11a9 9 0 0 1 9 9"></path><path d="M4 4a16 16 0 0 1 16 16"></path><circle cx="5" cy="19" r="1"></circle></svg></a></div></nav></div><div class="min-h-screen overflow-x-hidden relative"><div class="mt-10 mb-5 px-5 md:px-8 mx-auto"><article class="max-w-6xl mx-auto mt-24 mb-12 border-b border-gray-100 pb-32"><div class="max-w-[704px] mx-auto flex items-center text-sm text-gray-600 space-x-3"><div class="flex items-center space-x-3"><div class="flex items-center"><img src="https://cms.railway.com/media/person-ray-chen-dd0fa1297e94.jpg" alt="Avatar of Ray Chen" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/></div><span>Ray Chen</span></div><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><time dateTime="2025-09-22T00:00:00.000Z">Sep 22, 2025</time></div><header class="mt-5 mb-16 max-w-[704px] mx-auto"><h1 class="text-post-title font-medium font-serif">Incident Report: September 22nd, 2025</h1></header><section class="max-w-[704px] mx-auto text-base sm:text-lg"><nav style="position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border-width:0" aria-label="Table of Contents"><h2>Table of Contents</h2><ol><li style="margin-left:1.5rem"><a href="#impact">Impact</a></li><li style="margin-left:1.5rem"><a href="#incident-timeline">Incident Timeline</a></li><li style="margin-left:1.5rem"><a href="#what-happened">What Happened?</a></li><li style="margin-left:1.5rem"><a href="#preventative-measures">Preventative Measures</a></li></ol></nav><div><p class="mb-4 text-gray-800">We recently experienced an outage that impacted our dashboard and deployment pipeline.</p>
<p class="mb-4 text-gray-800">When a Major Outage occurs, it is Railway’s policy to share the public details of what happened.</p>
<a class="relative no-underline" href="/p/incident-report-sept-22-2025#impact"><span id="impact" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Impact</h2></a>
<p class="mb-4 text-gray-800">This outage impacted our dashboard and deployment pipeline.</p>
<p class="mb-4 text-gray-800">You may have experienced an infinite loading state when trying to load your projects’ canvas, delayed deployments, or a “Limited Access” message when initiating new deployments.</p>
<p class="mb-4 text-gray-800">All running deployments and platform-level networking features remained online throughout this period. Users who did not access the dashboard or initiate deployments during the incident window experienced no service disruption.</p>
<p class="mb-4 text-gray-800">Some users may have experienced issues with “Invalid database URL” or missing Private Networking during this period. These issues have been fixed.</p>
<a class="relative no-underline" href="/p/incident-report-sept-22-2025#incident-timeline"><span id="incident-timeline" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Incident Timeline</h2></a>
<ul class="list-disc pl-6 mb-4 space-y-4 text-gray-800">
<li><strong>11:05 UTC</strong> - Initial reports received of Railway project canvas failing to load for users</li>
<li><strong>11:14 UTC</strong> - Incident declared as Partial Outage affecting dashboard functionality; investigation commenced</li>
<li><strong>11:35 UTC</strong> - Root cause identified and remediated; partial service restoration confirmed</li>
<li><strong>11:45 UTC</strong> - Full service restoration achieved; incident marked as resolved</li>
<li><strong>11:48 UTC</strong> - Control Plane exhibited abnormal resource consumption and elevated API latency; new incident initiated. Deployment capabilities suspended for non-Pro tier users (Free, Trial, Hobby), who encountered &quot;Limited Access&quot; notifications when attempting to deploy</li>
<li><strong>12:14 UTC</strong> - Correlation established between dashboard and Control Plane issues; original incident reopened</li>
<li><strong>12:25 UTC</strong> - Control Plane degradation intensified; all deployments suspended across all tiers (including Pro)</li>
<li><strong>12:44 UTC</strong> - Deployment functionality restored for Pro users</li>
<li><strong>12:50 UTC</strong> - Deployment functionality restored for non-Pro users</li>
<li><strong>13:05 UTC</strong> - Rolled back deployment functionality for non-Pro users due to persistent Control Plane instability; Pro tier deployments remained operational</li>
<li><strong>14:01 UTC</strong> - Full service recovery confirmed; deployment capabilities restored for all user tiers</li>
<li><strong>21:27 UTC</strong> - Retroactive fixing of a rare edge case where private networks and their endpoints no longer appear to be visible on the dashboard</li>
</ul>
<p class="mb-4 text-gray-800">For further reference, please refer to the incident’s live updates on our Status Page on <a class="markdown-inline-link" href="https://status.railway.com/cmfv13zib12ij6f0fr7dtts3o" target="_blank" rel="noreferrer noopener">dashboard may fail to load for some users</a> and <a class="markdown-inline-link" href="https://status.railway.com/cmfv2ciuy00krz1jvwfghr4gz" target="_blank" rel="noreferrer noopener">control plane instability</a>.</p>
<a class="relative no-underline" href="/p/incident-report-sept-22-2025#what-happened"><span id="what-happened" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">What Happened?</h2></a>
<p class="mb-4 text-gray-800">The Railway dashboard and project canvas relies on our API to fetch data. During a code cleanup, we made a database schema change to remove an unused column. However, a subsequent code change reverted part of this unintentionally, leading to failures for a core API our frontend relies on.</p>
<p class="mb-4 text-gray-800">Because the project canvas relies on this API, users started seeing an infinite loading state that hid the underlying error. We declared <a class="markdown-inline-link" href="https://status.railway.com/cmfv13zib12ij6f0fr7dtts3o" target="_blank" rel="noreferrer noopener">an incident</a> and deployed a fix within 15 minutes, successfully resolving the immediate issue.</p>
<p class="mb-4 text-gray-800">However, this failure cascaded globally across our backend infrastructure, resulting in:</p>
<ul class="list-disc pl-6 mb-4 space-y-4 text-gray-800">
<li>Deployment failures across all interfaces (Dashboard, GitHub Webhooks, CLI)</li>
<li>Further dashboard loading failures</li>
</ul>
<p class="mb-4 text-gray-800">After the resolution of the initial incident, a substantial backlog of queued deployments were triggered to start simultaneously. This surge overwhelmed our Control Plane, manifesting as elevated database errors on our end.</p>
<p class="mb-4 text-gray-800">We traced the root cause of this down to our Control Pane. The dashboard&#x27;s dependency on Control Plane APIs for critical data, such as service domains and private network endpoints, prevented it from loading properly during the outage. We started a <a class="markdown-inline-link" href="https://status.railway.com/cmfv2ciuy00krz1jvwfghr4gz" target="_blank" rel="noreferrer noopener">second incident</a>, re-opened the <a class="markdown-inline-link" href="https://status.railway.com/cmfv13zib12ij6f0fr7dtts3o" target="_blank" rel="noreferrer noopener">original incident</a>, and began remediation efforts immediately.</p>
<p class="mb-4 text-gray-800">To manage the increased deployment backlog, we implemented a phased response:</p>
<ol class="list-decimal pl-6 mb-4 space-y-4 text-gray-800">
<li>Initially suspended deployments for non-Pro tier users. This allowed all Pro users to deploy normally, and helped us offgas the deployment queue so it can gradually recover,</li>
<li>Temporarily suspended all deployments during peak instability. This suspended new deployments globally (including for Pro users) for up to 20 minutes to help us offgas the queue further,</li>
<li>Significantly scaled up our Control Plane&#x27;s database resources to handle increased load. This included increasing CPU and memory allocations, optimizing connection limits, and expanding the database&#x27;s IOPS capacity to ensure it could handle the anticipated traffic surge once deployments resumed.</li>
</ol>
<p class="mb-4 text-gray-800">Despite the performance optimization performed on the Control Plane’s database, performance issues persisted. Investigation revealed that the root cause was our connection pooling layer (PgBouncer). It was causing an abnormal amount of CPU and Memory usage beyond what was nominal:</p>
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/0f6307f3ae77a8bde42ad2528742dc1d7e8c53e34455c5f478f4c80e70d7f1c4.png?w=768&amp;format=webp&amp;q=90" srcSet="https://cms.railway.com/media/0f6307f3ae77a8bde42ad2528742dc1d7e8c53e34455c5f478f4c80e70d7f1c4.png?w=320&amp;format=webp&amp;q=90 320w, https://cms.railway.com/media/0f6307f3ae77a8bde42ad2528742dc1d7e8c53e34455c5f478f4c80e70d7f1c4.png?w=480&amp;format=webp&amp;q=90 480w, https://cms.railway.com/media/0f6307f3ae77a8bde42ad2528742dc1d7e8c53e34455c5f478f4c80e70d7f1c4.png?w=640&amp;format=webp&amp;q=90 640w, https://cms.railway.com/media/0f6307f3ae77a8bde42ad2528742dc1d7e8c53e34455c5f478f4c80e70d7f1c4.png?w=768&amp;format=webp&amp;q=90 768w, https://cms.railway.com/media/0f6307f3ae77a8bde42ad2528742dc1d7e8c53e34455c5f478f4c80e70d7f1c4.png?w=960&amp;format=webp&amp;q=90 960w, https://cms.railway.com/media/0f6307f3ae77a8bde42ad2528742dc1d7e8c53e34455c5f478f4c80e70d7f1c4.png?w=1200&amp;format=webp&amp;q=90 1200w, https://cms.railway.com/media/0f6307f3ae77a8bde42ad2528742dc1d7e8c53e34455c5f478f4c80e70d7f1c4.png?w=1440&amp;format=webp&amp;q=90 1440w, https://cms.railway.com/media/0f6307f3ae77a8bde42ad2528742dc1d7e8c53e34455c5f478f4c80e70d7f1c4.png?w=1600&amp;format=webp&amp;q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="Control Plane’s API Server CPU and Memory spikes during the incident" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">Control Plane’s API Server CPU and Memory spikes during the incident</figcaption></figure>
<p class="mb-4 text-gray-800">Further investigation revealed the issue stemmed from a PgBouncer version upgrade. Our Control Plane database relies on PgBouncer as a connection pooler, common in most applications for managing Postgres connections at scale.</p>
<p class="mb-4 text-gray-800">We had been using an older Bitnami image, and following <a class="markdown-inline-link" href="https://github.com/bitnami/charts/issues/35164" target="_blank" rel="noreferrer noopener">Bitnami&#x27;s deprecation of their public images</a>, we migrated to our internal mirrored repository. This migration advanced us several PgBouncer versions ahead.</p>
<p class="mb-4 text-gray-800">The upgraded version under load exhibited abnormally high CPU and memory consumption, manifesting as increased latency and database connection timeouts. We identified the likely cause as a breaking default configuration change in PgBouncer.</p>
<p class="mb-4 text-gray-800">Upon discovering this, we immediately rolled back to our previously stable PgBouncer version (rather than adjusting the configuration to prioritize stability and eliminating any risk of additional version-related issues). Following the rollback, we saw a 100x performance increase with all metrics returning to nominal values, and we re-enabled deployments for all non-Pro users.</p>
<a class="relative no-underline" href="/p/incident-report-sept-22-2025#preventative-measures"><span id="preventative-measures" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Preventative Measures</h2></a>
<p class="mb-4 text-gray-800">At the start of Q3, we started a project to re-engineer parts of our Control Plane to be more resilient. This includes moving large parts of its storage layer away from Postgres. This was intended to defend against single region Control Plane failures impacting the platform, and the upcoming system will allow for several orders higher magnitude IOPS compared to our current Postgres implementation.</p>
<p class="mb-4 text-gray-800">In the meantime, we will be making these changes to our development practices to prevent any disruption(s) of this class from happening again:</p>
<ul class="list-disc pl-6 mb-4 space-y-4 text-gray-800">
<li><strong>Enhanced CI pipeline validation</strong> to prevent disruptive schema changes from reaching production, and automated schema compatibility checks before backend deployments. We believe that if the breaking schema change did not make it to production here, we would not have this cascading failure scenario</li>
<li><strong>Implement monitoring and alerting for PgBouncer node performance metrics</strong> (CPU, memory, connection pool utilization). If we had known about the performance regression prior, we could have recovered from the incidents much faster</li>
<li><strong>Phased rollout strategy for PgBouncer upgrades</strong>. By incrementally updating PgBouncer instances rather than performing a global upgrade, combined with the enhanced monitoring above, we would have identified the version-specific performance issues before full deployment.</li>
</ul>
<hr class="my-8"/>
<p class="mb-4 text-gray-800">Railway is committed to providing the best-in-class cloud experience. Any downtime is unacceptable for us. We apologize for any inconvenience caused by this, and we are going to work towards eliminating the entire class of issues contributing to this incident.</p></div></section></article><div class="max-w-6xl mx-auto"><div><header class="flex items-center justify-between mb-8"><h3 class="text-gray-500 text-h3">Continue Reading...</h3><a class="text-pink-500 hover:underline" href="/engineering">View All <!-- -->Engineering<!-- --> -&gt;</a></header><div class="grid grid-cols-1 md:grid-cols-2 gap-8"><a class="flex flex-col bg-secondaryBg p-6 rounded-lg hover:bg-gray-100 group" href="/p/dev-new"><div class="flex gap-2"><div class="post-category-pill font-medium max-w-max text-xs px-2 py-[3px] rounded-[4px] tracking-[0.06em] uppercase" data-category-tone="plum">Engineering</div></div><div class="flex-grow"><header class="font-medium font-serif text-lg mt-2 mb-1">Your Cloud should come with an app builder </header><p class="text-base text-gray-800 line-clamp-2">Announcing: dev.new, an agent that can build, preview, test, and deploy an app from the same place. We tested the whole loop by asking it to ship a real-time multiplayer FPS game that runs in the browser</p></div><div class="flex items-center gap-3 mt-6"><div class="flex items-center"><img src="https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg" alt="Avatar of Mahmoud Abdelwahab" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/></div><span class="text-sm text-gray-600">Mahmoud Abdelwahab</span><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><span class="text-sm text-gray-600">Aug 13, 2026</span></div></a><a class="flex flex-col bg-secondaryBg p-6 rounded-lg hover:bg-gray-100 group" href="/p/feature-flags"><div class="flex gap-2"><div class="post-category-pill font-medium max-w-max text-xs px-2 py-[3px] rounded-[4px] tracking-[0.06em] uppercase" data-category-tone="plum">Engineering</div></div><div class="flex-grow"><header class="font-medium font-serif text-lg mt-2 mb-1">Roll it out, roll it back, never redeploy</header><p class="text-base text-gray-800 line-clamp-2">Merge now, release when you&#x27;re ready: turn a feature on for your team, then 10% of users, then everyone. Feature flags are now built into every Railway project, and your agents can run the rollout from the CLI, SDK, and MCP. No third-party service required.</p></div><div class="flex items-center gap-3 mt-6"><div class="flex items-center"><img src="https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg" alt="Avatar of Victor Ramirez" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/><img src="https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg" alt="Avatar of Mahmoud Abdelwahab" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:-8px" loading="lazy" decoding="async" width="24" height="24"/></div><span class="text-sm text-gray-600">Victor Ramirez &amp; Mahmoud Abdelwahab</span><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><span class="text-sm text-gray-600">Jul 17, 2026</span></div></a></div></div><section class="post-bottom-cta my-16 border rounded-[16px] px-6 py-16 sm:px-12 sm:py-24 flex flex-col items-center justify-center text-center" aria-labelledby="post-bottom-cta-title"><h3 id="post-bottom-cta-title" class="text-h2 font-serif font-medium">Ready to get started?</h3><p class="mt-4 text-base sm:text-lg text-gray-600">Join millions of developers deploying applications on Railway</p><div class="mt-8 flex w-full flex-col items-stretch justify-center gap-3 sm:w-auto sm:flex-row"><a class="post-bottom-cta-primary inline-flex min-h-[48px] items-center justify-center rounded-[8px] px-6 py-3 text-base font-medium no-underline transition-colors duration-100" href="https://railway.com" target="_blank" rel="noreferrer noopener">Deploy a new project</a><a class="post-bottom-cta-secondary inline-flex min-h-[48px] items-center justify-center rounded-[8px] border px-6 py-3 text-base font-medium no-underline transition-colors duration-100" href="https://railway.com/enterprise" target="_blank" rel="noreferrer noopener">Book a demo</a></div></section></div></div></div><footer class="px-5 md:px-8 py-12 border-t border-gray-100"><div class="grid grid-cols-2 sm:grid-cols-12 gap-8 sm:gap-4 lg:gap-16 max-w-6xl mx-auto"><div class="hidden sm:flex col-span-3 lg:col-span-4 flex-col justify-between"><a class="underline" href="https://railway.com" target="_blank" rel="noreferrer noopener"><svg data-v-423bf9ae="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 59.99945071644884 60" class="logo w-8"><g data-v-423bf9ae="" id="6f639fe5-02e1-4640-a1fb-f4b22b64e5ef" transform="matrix(0.5999641134746578,0,0,0.5999641134746578,-437.5538206497148,-266.68705118956507)" stroke="none"><path d="M729.872 487.327a50.86 50.86 0 0 0-.464 5.033h75.879c-.265-.518-.621-.985-.98-1.442-12.972-16.769-19.95-15.315-29.932-15.741-3.328-.137-5.585-.192-18.832-.192-7.09 0-14.798.018-22.304.038.737-1.746 1.6-3.419 2.525-5.061-2.885 5.106-4.865 10.771-5.805 16.789l.891-4.397c.007-.031.018-.063.024-.094h38.883v5.067h-39.885zM805.885 497.432h-76.438c.08 1.352.206 2.686.388 4.002h70.571c3.146 0 4.907-1.786 5.479-4.002zM733.851 515.257a52.226 52.226 0 0 1-1.98-4.997c6.608 19.89 25.328 34.251 47.433 34.251 20.205 0 37.566-12.007 45.452-29.254h-90.905zM729.38 492.915c-.018.531-.08 1.055-.08 1.589 0 .538.063 1.059.08 1.59v-3.179zM824.77 515.229z"></path><path d="M779.303 444.505c-18.682 0-34.939 10.265-43.524 25.439 6.709-.014 19.775-.022 19.775-.022h.003v-.005c15.444 0 16.018.069 19.035.195l1.868.069c6.507.217 14.505.916 20.798 5.68 3.416 2.584 8.348 8.287 11.288 12.35 2.718 3.758 3.5 8.078 1.652 12.217-1.701 3.804-5.361 6.073-9.793 6.073H730.85l-.884-4.201c.426 2.707 1.037 5.344 1.879 7.886h94.914a49.863 49.863 0 0 0 2.546-15.682c.001-27.611-22.386-49.999-50.002-49.999z"></path></g></svg></a><div class="text-xs font-medium text-gray-600 w-full">Copyright © <!-- -->2026<!-- --> Railway Corp. <br/>All rights reserved.</div></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Product</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/changelog" target="_blank" rel="noreferrer noopener">Changelog</a></li><li><a class="hover:text-foreground" href="https://railway.com/pricing" target="_blank" rel="noreferrer noopener">Pricing</a></li><li><a class="hover:text-foreground" href="https://railway.com/deploy" target="_blank" rel="noreferrer noopener">Templates</a></li><li><a class="hover:text-foreground" href="https://station.railway.com/feedback" target="_blank" rel="noreferrer noopener">Feedback</a></li><li><a class="hover:text-foreground" href="https://railway.com/open-source-kickback" target="_blank" rel="noreferrer noopener">OSS Kickback</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Company</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/about" target="_blank" rel="noreferrer noopener">About</a></li><li><a class="hover:text-foreground" href="https://railway.com/careers" target="_blank" rel="noreferrer noopener">Careers</a></li><li><a class="hover:text-foreground" href="/">Blog</a></li><li><a class="hover:text-foreground" href="https://shop.railway.com" target="_blank" rel="noreferrer noopener">Shop</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Contact</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://discord.gg/railway" target="_blank" rel="noreferrer noopener">Discord</a></li><li><a class="hover:text-foreground" href="https://twitter.com/railway" target="_blank" rel="noreferrer noopener">Twitter</a></li><li><a class="hover:text-foreground" href="https://github.com/railwayapp" target="_blank" rel="noreferrer noopener">GitHub</a></li><li><a class="hover:text-foreground" href="mailto:contact@railway.com">Email</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Legal</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/legal/acceptable-use" target="_blank" rel="noreferrer noopener">Acceptable Use</a></li><li><a class="hover:text-foreground" href="https://railway.com/legal/privacy" target="_blank" rel="noreferrer noopener">Privacy Policy</a></li><li><a class="hover:text-foreground" href="https://railway.com/legal/terms" target="_blank" rel="noreferrer noopener">Terms of Service</a></li></ul></div></div></footer></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"page":{"authors":[{"avatar":{"alt":"Ray Chen headshot","height":1096,"id":"1853","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-ray-chen-dd0fa1297e94.jpg","width":1152},"avatarUrl":"https://cms.railway.com/media/person-ray-chen-dd0fa1297e94.jpg","githubUrl":null,"id":"29","name":"Ray Chen","slug":"ray-chen","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":"We recently experienced an outage that impacted our dashboard and deployment pipeline.\n\nWhen a Major Outage occurs, it is Railway’s policy to share the public details of what happened.\n\n## Impact\n\nThis outage impacted our dashboard and deployment pipeline.\n\nYou may have experienced an infinite loading state when trying to load your projects’ canvas, delayed deployments, or a “Limited Access” message when initiating new deployments.\n\nAll running deployments and platform-level networking features remained online throughout this period. Users who did not access the dashboard or initiate deployments during the incident window experienced no service disruption.\n\nSome users may have experienced issues with “Invalid database URL” or missing Private Networking during this period. These issues have been fixed.\n\n## Incident Timeline\n\n- **11:05 UTC** - Initial reports received of Railway project canvas failing to load for users\n- **11:14 UTC** - Incident declared as Partial Outage affecting dashboard functionality; investigation commenced\n- **11:35 UTC** - Root cause identified and remediated; partial service restoration confirmed\n- **11:45 UTC** - Full service restoration achieved; incident marked as resolved\n- **11:48 UTC** - Control Plane exhibited abnormal resource consumption and elevated API latency; new incident initiated. Deployment capabilities suspended for non-Pro tier users (Free, Trial, Hobby), who encountered \"Limited Access\" notifications when attempting to deploy\n- **12:14 UTC** - Correlation established between dashboard and Control Plane issues; original incident reopened\n- **12:25 UTC** - Control Plane degradation intensified; all deployments suspended across all tiers (including Pro)\n- **12:44 UTC** - Deployment functionality restored for Pro users\n- **12:50 UTC** - Deployment functionality restored for non-Pro users\n- **13:05 UTC** - Rolled back deployment functionality for non-Pro users due to persistent Control Plane instability; Pro tier deployments remained operational\n- **14:01 UTC** - Full service recovery confirmed; deployment capabilities restored for all user tiers\n- **21:27 UTC** - Retroactive fixing of a rare edge case where private networks and their endpoints no longer appear to be visible on the dashboard\n\nFor further reference, please refer to the incident’s live updates on our Status Page on [dashboard may fail to load for some users](https://status.railway.com/cmfv13zib12ij6f0fr7dtts3o) and [control plane instability](https://status.railway.com/cmfv2ciuy00krz1jvwfghr4gz).\n\n## What Happened?\n\nThe Railway dashboard and project canvas relies on our API to fetch data. During a code cleanup, we made a database schema change to remove an unused column. However, a subsequent code change reverted part of this unintentionally, leading to failures for a core API our frontend relies on.\n\nBecause the project canvas relies on this API, users started seeing an infinite loading state that hid the underlying error. We declared [an incident](https://status.railway.com/cmfv13zib12ij6f0fr7dtts3o) and deployed a fix within 15 minutes, successfully resolving the immediate issue.\n\nHowever, this failure cascaded globally across our backend infrastructure, resulting in:\n\n- Deployment failures across all interfaces (Dashboard, GitHub Webhooks, CLI)\n- Further dashboard loading failures\n\nAfter the resolution of the initial incident, a substantial backlog of queued deployments were triggered to start simultaneously. This surge overwhelmed our Control Plane, manifesting as elevated database errors on our end.\n\nWe traced the root cause of this down to our Control Pane. The dashboard's dependency on Control Plane APIs for critical data, such as service domains and private network endpoints, prevented it from loading properly during the outage. We started a [second incident](https://status.railway.com/cmfv2ciuy00krz1jvwfghr4gz), re-opened the [original incident](https://status.railway.com/cmfv13zib12ij6f0fr7dtts3o), and began remediation efforts immediately.\n\nTo manage the increased deployment backlog, we implemented a phased response:\n\n1. Initially suspended deployments for non-Pro tier users. This allowed all Pro users to deploy normally, and helped us offgas the deployment queue so it can gradually recover,\n2. Temporarily suspended all deployments during peak instability. This suspended new deployments globally (including for Pro users) for up to 20 minutes to help us offgas the queue further,\n3. Significantly scaled up our Control Plane's database resources to handle increased load. This included increasing CPU and memory allocations, optimizing connection limits, and expanding the database's IOPS capacity to ensure it could handle the anticipated traffic surge once deployments resumed.\n\nDespite the performance optimization performed on the Control Plane’s database, performance issues persisted. Investigation revealed that the root cause was our connection pooling layer (PgBouncer). It was causing an abnormal amount of CPU and Memory usage beyond what was nominal:\n\n![Control Plane’s API Server CPU and Memory spikes during the incident](https://cms.railway.com/media/0f6307f3ae77a8bde42ad2528742dc1d7e8c53e34455c5f478f4c80e70d7f1c4.png)\n\nFurther investigation revealed the issue stemmed from a PgBouncer version upgrade. Our Control Plane database relies on PgBouncer as a connection pooler, common in most applications for managing Postgres connections at scale.\n\nWe had been using an older Bitnami image, and following [Bitnami's deprecation of their public images](https://github.com/bitnami/charts/issues/35164), we migrated to our internal mirrored repository. This migration advanced us several PgBouncer versions ahead.\n\nThe upgraded version under load exhibited abnormally high CPU and memory consumption, manifesting as increased latency and database connection timeouts. We identified the likely cause as a breaking default configuration change in PgBouncer.\n\nUpon discovering this, we immediately rolled back to our previously stable PgBouncer version (rather than adjusting the configuration to prioritize stability and eliminating any risk of additional version-related issues). Following the rollback, we saw a 100x performance increase with all metrics returning to nominal values, and we re-enabled deployments for all non-Pro users.\n\n## Preventative Measures\n\nAt the start of Q3, we started a project to re-engineer parts of our Control Plane to be more resilient. This includes moving large parts of its storage layer away from Postgres. This was intended to defend against single region Control Plane failures impacting the platform, and the upcoming system will allow for several orders higher magnitude IOPS compared to our current Postgres implementation.\n\nIn the meantime, we will be making these changes to our development practices to prevent any disruption(s) of this class from happening again:\n\n- **Enhanced CI pipeline validation** to prevent disruptive schema changes from reaching production, and automated schema compatibility checks before backend deployments. We believe that if the breaking schema change did not make it to production here, we would not have this cascading failure scenario\n- **Implement monitoring and alerting for PgBouncer node performance metrics** (CPU, memory, connection pool utilization). If we had known about the performance regression prior, we could have recovered from the incidents much faster\n- **Phased rollout strategy for PgBouncer upgrades**. By incrementally updating PgBouncer instances rather than performing a global upgrade, combined with the enhanced monitoring above, we would have identified the version-specific performance issues before full deployment.\n\n---\n\nRailway is committed to providing the best-in-class cloud experience. Any downtime is unacceptable for us. We apologize for any inconvenience caused by this, and we are going to work towards eliminating the entire class of issues contributing to this incident.","createdAt":"2026-06-09T12:15:51.527Z","description":"We experienced an outage affecting our dashboard and deployment pipeline, resulting in a temporary suspension of deployments across all user tiers.","externalAuthor":false,"featured":false,"featuredImage":{"alt":"Incident Report: April 23rd, 2025 social image","height":null,"id":"375","mimeType":"image/png","url":"https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png","width":null},"id":"92","publishedAt":"2025-09-22T00:00:00.000Z","seoDescription":"We experienced an outage affecting our dashboard and deployment pipeline, resulting in a temporary suspension of deployments across all user tiers.","seoTitle":"Incident Report: September 22nd, 2025","slug":"incident-report-sept-22-2025","socialImage":{"alt":"Incident Report: April 23rd, 2025 social image","height":null,"id":"375","mimeType":"image/png","url":"https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png","width":null},"title":"Incident Report: September 22nd, 2025","updatedAt":"2025-09-22T00:00:00.000Z"},"relatedPosts":[{"authors":[{"avatar":{"alt":"Mahmoud Abdelwahab headshot","height":460,"id":"1846","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","width":460},"avatarUrl":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","githubUrl":null,"id":"4","name":"Mahmoud Abdelwahab","slug":"mahmoud-abdelwahab","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":null,"createdAt":"2026-08-06T08:07:06.228Z","description":"Announcing: dev.new, an agent that can build, preview, test, and deploy an app from the same place. We tested the whole loop by asking it to ship a real-time multiplayer FPS game that runs in the browser","externalAuthor":false,"featured":true,"featuredImage":{"alt":"dev.new featured image","height":1440,"id":"1983","mimeType":"image/png","url":"https://cms.railway.com/media/dev.new-featured-image-4.png","width":3264},"id":"289","publishedAt":"2026-08-13T08:07:00.000Z","seoDescription":null,"seoTitle":null,"slug":"dev-new","socialImage":{"alt":"dev.new featured image 1","height":720,"id":"1976","mimeType":"image/png","url":"https://cms.railway.com/media/dev.new-featured-image-3.png","width":1632},"title":"Your Cloud should come with an app builder ","updatedAt":"2026-08-14T20:19:27.322Z"},{"authors":[{"avatar":{"alt":"Avatar of Victor Ramirez","height":null,"id":"1855","mimeType":"image/jpeg","url":"https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg","width":null},"avatarUrl":"https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg","githubUrl":null,"id":"34","name":"Victor Ramirez","slug":"victor-ramirez","title":null},{"avatar":{"alt":"Mahmoud Abdelwahab headshot","height":460,"id":"1846","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","width":460},"avatarUrl":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","githubUrl":null,"id":"4","name":"Mahmoud Abdelwahab","slug":"mahmoud-abdelwahab","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":null,"createdAt":"2026-07-15T22:26:48.036Z","description":"Merge now, release when you're ready: turn a feature on for your team, then 10% of users, then everyone. Feature flags are now built into every Railway project, and your agents can run the rollout from the CLI, SDK, and MCP. No third-party service required.","externalAuthor":false,"featured":false,"featuredImage":{"alt":"feature flags featured image","height":1440,"id":"1968","mimeType":"image/png","url":"https://cms.railway.com/media/feature-flags-featured-image.png","width":3264},"id":"278","publishedAt":"2026-07-17T07:00:00.000Z","seoDescription":null,"seoTitle":null,"slug":"feature-flags","socialImage":null,"title":"Roll it out, roll it back, never redeploy","updatedAt":"2026-08-13T16:53:55.126Z"}]},"__N_SSG":true},"page":"/p/[slug]","query":{"slug":"incident-report-sept-22-2025"},"buildId":"nFX7R3M0MdLMAf-Ufh9qa","isFallback":false,"isExperimentalCompile":false,"gsp":true,"scriptLoader":[]}</script></body></html>