Files
nexus/sreweekly/articles/500/02-railway-incident-report-october-28th-2025.html
2026-09-12 17:23:01 +08:00

247 lines
40 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8" data-next-head=""/><meta name="viewport" content="width=device-width" data-next-head=""/><meta name="twitter:card" content="summary_large_image" data-next-head=""/><meta name="twitter:site" content="@Railway" data-next-head=""/><meta name="twitter:creator" content="@Railway" data-next-head=""/><meta property="og:image:type" content="image/png" data-next-head=""/><meta property="og:image:width" content="1200" data-next-head=""/><meta property="og:image:height" content="630" data-next-head=""/><meta property="og:site_name" content="Railway Blog" data-next-head=""/><title data-next-head="">Incident Report: October 28th, 2025</title><meta name="robots" content="index,follow" data-next-head=""/><meta name="description" content="We recently experienced an outage that impacted our backend API, disrupting dashboard access, CLI operations, GitHub deployments, and API functionality." data-next-head=""/><meta property="og:title" content="Incident Report: October 28th, 2025" data-next-head=""/><meta property="og:description" content="We recently experienced an outage that impacted our backend API, disrupting dashboard access, CLI operations, GitHub deployments, and API functionality." data-next-head=""/><meta property="og:url" content="https://blog.railway.com/p/incident-report-oct-28th-2025" data-next-head=""/><meta property="og:type" content="article" data-next-head=""/><meta property="article:published_time" content="2025-10-28T00:00:00.000Z" data-next-head=""/><meta property="article:modified_time" content="2025-10-28T00:00:00.000Z" data-next-head=""/><meta property="article:author" content="Ray Chen" data-next-head=""/><meta property="article:section" content="Engineering" data-next-head=""/><meta property="og:image" content="https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png?w=1200&amp;q=90" data-next-head=""/><link rel="canonical" href="https://blog.railway.com/p/incident-report-oct-28th-2025" data-next-head=""/><meta name="twitter:title" content="Incident Report: October 28th, 2025" data-next-head=""/><meta name="twitter:description" content="We recently experienced an outage that impacted our backend API, disrupting dashboard access, CLI operations, GitHub deployments, and API functionality." data-next-head=""/><meta name="twitter:image" content="https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png?w=1200&amp;q=90" data-next-head=""/><link rel="alternate" type="text/markdown" href="/p/incident-report-oct-28th-2025.md" data-next-head=""/><link rel="alternate" type="application/rss+xml" title="Railway Blog" href="https://blog.railway.com/rss.xml"/><link rel="preload" href="/_next/static/media/e4af272ccee01ff0-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/91c3caac73720166-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/3359c9acc30ef440-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/aff755c8b4bb3089-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/3988c35203e0830a-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/427e4a37d3642943-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/css/619a01ff31a99955.css" as="style"/><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"BlogPosting","headline":"Incident Report: October 28th, 2025","description":"We recently experienced an outage that impacted our backend API, disrupting dashboard access, CLI operations, GitHub deployments, and API functionality.","image":["https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png?w=1200&q=90"],"datePublished":"2025-10-28T00:00:00.000Z","dateModified":"2025-10-28T00:00:00.000Z","author":{"@type":"Person","name":"Ray Chen"},"publisher":{"@type":"Organization","name":"Railway","logo":{"@type":"ImageObject","url":"https://railway.com/brand/logo-dark.png"}},"mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.railway.com/p/incident-report-oct-28th-2025"}}</script><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https://blog.railway.com"},{"@type":"ListItem","position":2,"name":"Engineering","item":"https://blog.railway.com/engineering"},{"@type":"ListItem","position":3,"name":"Incident Report: October 28th, 2025","item":"https://blog.railway.com/p/incident-report-oct-28th-2025"}]}</script><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"FAQPage","mainEntity":[{"@type":"Question","name":"What Happened?","acceptedAnswer":{"@type":"Answer","text":"Railway’s client-facing interfaces (Dashboard, CLI, Public API, etc.) depend on our backend, which relies on Postgres as its primary datastore."}}]}</script><style>:root {
--foreground: hsl(0, 0%, 100%);
--background: #181622;
--secondaryBg: hsl(250, 21%, 11%);
--blue-50: hsl(220, 55%, 10%);
--blue-100: hsl(220, 55%, 13%);
--blue-200: hsl(220, 62%, 25%);
--blue-300: hsl(220, 68%, 35%);
--blue-400: hsl(220, 72%, 45%);
--blue-500: hsl(220, 80%, 55%);
--blue-600: hsl(220, 80%, 65%);
--blue-700: hsl(220, 80%, 75%);
--blue-800: hsl(220, 80%, 85%);
--blue-900: hsl(220, 80%, 95%);
--blue-950: hsl(220, 55%, 97%);
--gray-50: hsl(248, 21%, 13%);
--gray-100: hsl(246, 18%, 15%);
--gray-200: hsl(246, 11%, 22%);
--gray-300: hsl(246, 8%, 35%);
--gray-400: hsl(246, 7%, 45%);
--gray-500: hsl(246, 6%, 55%);
--gray-600: hsl(246, 6%, 65%);
--gray-700: hsl(246, 6%, 78%);
--gray-800: hsl(246, 6%, 87%);
--gray-900: hsl(246, 6%, 95%);
--gray-950: hsl(246, 6%, 95%);
--red-50: hsl(1, 35%, 10%);
--red-100: hsl(1, 45%, 12%);
--red-200: hsl(1, 55%, 20%);
--red-300: hsl(1, 62%, 28%);
--red-400: hsl(1, 62%, 35%);
--red-500: hsl(1, 62%, 44%);
--red-600: hsl(1, 62%, 60%);
--red-700: hsl(1, 62%, 76%);
--red-800: hsl(1, 64%, 85%);
--red-900: hsl(1, 68%, 95%);
--red-950: hsl(1, 55%, 98%);
--pink-50: hsl(270, 40%, 16%);
--pink-100: hsl(270, 40%, 16%);
--pink-200: hsl(270, 45%, 24%);
--pink-300: hsl(270, 50%, 32%);
--pink-400: hsl(270, 55%, 43%);
--pink-500: hsl(270, 60%, 52%);
--pink-600: hsl(270, 70%, 65%);
--pink-700: hsl(270, 70%, 75%);
--pink-800: hsl(270, 70%, 85%);
--pink-900: hsl(270, 70%, 95%);
--pink-950: hsl(270, 70%, 95%);
--green-50: hsl(152, 15%, 10%);
--green-100: hsl(152, 26%, 11%);
--green-200: hsl(152, 32%, 16%);
--green-300: hsl(152, 38%, 24%);
--green-400: hsl(152, 38%, 34%);
--green-500: hsl(152, 38%, 42%);
--green-600: hsl(152, 38%, 60%);
--green-700: hsl(152, 38%, 70%);
--green-800: hsl(152, 38%, 80%);
--green-900: hsl(152, 38%, 91%);
--green-950: hsl(152, 40%, 97%);
--yellow-50: hsl(44, 95%, 12%);
--yellow-100: hsl(44, 95%, 12%);
--yellow-200: hsl(44, 96%, 24%);
--yellow-300: hsl(44, 95%, 36%);
--yellow-400: hsl(44, 95%, 48%);
--yellow-500: hsl(44, 95%, 60%);
--yellow-600: hsl(44, 95%, 69%);
--yellow-700: hsl(44, 95%, 78%);
--yellow-800: hsl(44, 95%, 86%);
--yellow-900: hsl(44, 95%, 95%);
--yellow-950: hsl(44, 95%, 95%);
}
.light {
--foreground: hsl(250, 24%, 9%);
--background: #F1F0EF;
--secondaryBg: hsl(0, 0%, 98%);
--blue-50: hsl(220, 55%, 97%);
--blue-100: hsl(220, 80%, 95%);
--blue-200: hsl(220, 80%, 85%);
--blue-300: hsl(220, 80%, 75%);
--blue-400: hsl(220, 80%, 65%);
--blue-500: hsl(220, 80%, 55%);
--blue-600: hsl(220, 72%, 45%);
--blue-700: hsl(220, 68%, 35%);
--blue-800: hsl(220, 62%, 25%);
--blue-900: hsl(220, 55%, 13%);
--blue-950: hsl(220, 55%, 10%);
--gray-50: hsl(246, 6%, 95%);
--gray-100: hsl(246, 6%, 95%);
--gray-200: hsl(246, 6%, 87%);
--gray-300: hsl(246, 6%, 78%);
--gray-400: hsl(246, 6%, 65%);
--gray-500: hsl(246, 6%, 55%);
--gray-600: hsl(246, 7%, 45%);
--gray-700: hsl(246, 8%, 35%);
--gray-800: hsl(246, 11%, 22%);
--gray-900: hsl(246, 18%, 15%);
--gray-950: hsl(248, 21%, 13%);
--red-50: hsl(1, 55%, 98%);
--red-100: hsl(1, 68%, 95%);
--red-200: hsl(1, 64%, 85%);
--red-300: hsl(1, 62%, 76%);
--red-400: hsl(1, 62%, 60%);
--red-500: hsl(1, 62%, 44%);
--red-600: hsl(1, 62%, 35%);
--red-700: hsl(1, 62%, 28%);
--red-800: hsl(1, 55%, 20%);
--red-900: hsl(1, 45%, 12%);
--red-950: hsl(1, 35%, 10%);
--pink-50: hsl(270, 70%, 95%);
--pink-100: hsl(270, 70%, 95%);
--pink-200: hsl(270, 70%, 85%);
--pink-300: hsl(270, 70%, 75%);
--pink-400: hsl(270, 70%, 65%);
--pink-500: hsl(270, 60%, 52%);
--pink-600: hsl(270, 55%, 43%);
--pink-700: hsl(270, 50%, 32%);
--pink-800: hsl(270, 45%, 24%);
--pink-900: hsl(270, 40%, 16%);
--pink-950: hsl(270, 40%, 16%);
--green-50: hsl(152, 40%, 97%);
--green-100: hsl(152, 38%, 91%);
--green-200: hsl(152, 38%, 80%);
--green-300: hsl(152, 38%, 70%);
--green-400: hsl(152, 38%, 60%);
--green-500: hsl(152, 38%, 42%);
--green-600: hsl(152, 38%, 34%);
--green-700: hsl(152, 38%, 24%);
--green-800: hsl(152, 32%, 16%);
--green-900: hsl(152, 26%, 11%);
--green-950: hsl(152, 15%, 10%);
--yellow-50: hsl(44, 95%, 95%);
--yellow-100: hsl(44, 95%, 95%);
--yellow-200: hsl(44, 95%, 86%);
--yellow-300: hsl(44, 95%, 78%);
--yellow-400: hsl(44, 95%, 69%);
--yellow-500: hsl(44, 95%, 60%);
--yellow-600: hsl(44, 95%, 48%);
--yellow-700: hsl(44, 95%, 36%);
--yellow-800: hsl(44, 96%, 24%);
--yellow-900: hsl(44, 95%, 12%);
--yellow-950: hsl(44, 95%, 12%);
}
.dark {
--foreground: hsl(0, 0%, 100%);
--background: #181622;
--secondaryBg: hsl(250, 21%, 11%);
--blue-50: hsl(220, 55%, 10%);
--blue-100: hsl(220, 55%, 13%);
--blue-200: hsl(220, 62%, 25%);
--blue-300: hsl(220, 68%, 35%);
--blue-400: hsl(220, 72%, 45%);
--blue-500: hsl(220, 80%, 55%);
--blue-600: hsl(220, 80%, 65%);
--blue-700: hsl(220, 80%, 75%);
--blue-800: hsl(220, 80%, 85%);
--blue-900: hsl(220, 80%, 95%);
--blue-950: hsl(220, 55%, 97%);
--gray-50: hsl(248, 21%, 13%);
--gray-100: hsl(246, 18%, 15%);
--gray-200: hsl(246, 11%, 22%);
--gray-300: hsl(246, 8%, 35%);
--gray-400: hsl(246, 7%, 45%);
--gray-500: hsl(246, 6%, 55%);
--gray-600: hsl(246, 6%, 65%);
--gray-700: hsl(246, 6%, 78%);
--gray-800: hsl(246, 6%, 87%);
--gray-900: hsl(246, 6%, 95%);
--gray-950: hsl(246, 6%, 95%);
--red-50: hsl(1, 35%, 10%);
--red-100: hsl(1, 45%, 12%);
--red-200: hsl(1, 55%, 20%);
--red-300: hsl(1, 62%, 28%);
--red-400: hsl(1, 62%, 35%);
--red-500: hsl(1, 62%, 44%);
--red-600: hsl(1, 62%, 60%);
--red-700: hsl(1, 62%, 76%);
--red-800: hsl(1, 64%, 85%);
--red-900: hsl(1, 68%, 95%);
--red-950: hsl(1, 55%, 98%);
--pink-50: hsl(270, 40%, 16%);
--pink-100: hsl(270, 40%, 16%);
--pink-200: hsl(270, 45%, 24%);
--pink-300: hsl(270, 50%, 32%);
--pink-400: hsl(270, 55%, 43%);
--pink-500: hsl(270, 60%, 52%);
--pink-600: hsl(270, 70%, 65%);
--pink-700: hsl(270, 70%, 75%);
--pink-800: hsl(270, 70%, 85%);
--pink-900: hsl(270, 70%, 95%);
--pink-950: hsl(270, 70%, 95%);
--green-50: hsl(152, 15%, 10%);
--green-100: hsl(152, 26%, 11%);
--green-200: hsl(152, 32%, 16%);
--green-300: hsl(152, 38%, 24%);
--green-400: hsl(152, 38%, 34%);
--green-500: hsl(152, 38%, 42%);
--green-600: hsl(152, 38%, 60%);
--green-700: hsl(152, 38%, 70%);
--green-800: hsl(152, 38%, 80%);
--green-900: hsl(152, 38%, 91%);
--green-950: hsl(152, 40%, 97%);
--yellow-50: hsl(44, 95%, 12%);
--yellow-100: hsl(44, 95%, 12%);
--yellow-200: hsl(44, 96%, 24%);
--yellow-300: hsl(44, 95%, 36%);
--yellow-400: hsl(44, 95%, 48%);
--yellow-500: hsl(44, 95%, 60%);
--yellow-600: hsl(44, 95%, 69%);
--yellow-700: hsl(44, 95%, 78%);
--yellow-800: hsl(44, 95%, 86%);
--yellow-900: hsl(44, 95%, 95%);
--yellow-950: hsl(44, 95%, 95%);
}</style><link rel="stylesheet" href="/_next/static/css/619a01ff31a99955.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" noModule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-0b474a604a067f92.js" defer=""></script><script src="/_next/static/chunks/framework-90fc9b0f7c4706c1.js" defer=""></script><script src="/_next/static/chunks/main-e89abd6570bbb9dc.js" defer=""></script><script src="/_next/static/chunks/pages/_app-3049b372678b389f.js" defer=""></script><script src="/_next/static/chunks/5399-d49efedd870f41de.js" defer=""></script><script src="/_next/static/chunks/3801-cdd2681d27083ad8.js" defer=""></script><script src="/_next/static/chunks/9594-1c44b12020a3af0a.js" defer=""></script><script src="/_next/static/chunks/4696-fd1712bc36a5bc50.js" defer=""></script><script src="/_next/static/chunks/pages/p/%5Bslug%5D-ddadb1ca54eaab0f.js" defer=""></script><script src="/_next/static/nFX7R3M0MdLMAf-Ufh9qa/_buildManifest.js" defer=""></script><script src="/_next/static/nFX7R3M0MdLMAf-Ufh9qa/_ssgManifest.js" defer=""></script></head><body><div id="__next"><script>!function(){try{var d=document.documentElement,c=d.classList;c.remove('light','dark');var e=localStorage.getItem('theme');if('system'===e||(!e&&true)){var t='(prefers-color-scheme: dark)',m=window.matchMedia(t);if(m.media!==t||m.matches){d.style.colorScheme = 'dark';c.add('dark')}else{d.style.colorScheme = 'light';c.add('light')}}else if(e){c.add(e|| '')}if(e==='light'||e==='dark')d.style.colorScheme=e}catch(e){}}()</script><div class="__variable_f367f3 __variable_0b668e __variable_ecea63 font-sans"><div class="px-5 md:px-8"><nav class="max-w-6xl mx-auto py-6 flex justify-between items-center border-b border-gray-100"><a class="flex items-center space-x-4" href="/"><svg data-v-423bf9ae="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 59.99945071644884 60" class="logo w-8"><g data-v-423bf9ae="" id="6f639fe5-02e1-4640-a1fb-f4b22b64e5ef" transform="matrix(0.5999641134746578,0,0,0.5999641134746578,-437.5538206497148,-266.68705118956507)" stroke="none"><path d="M729.872 487.327a50.86 50.86 0 0 0-.464 5.033h75.879c-.265-.518-.621-.985-.98-1.442-12.972-16.769-19.95-15.315-29.932-15.741-3.328-.137-5.585-.192-18.832-.192-7.09 0-14.798.018-22.304.038.737-1.746 1.6-3.419 2.525-5.061-2.885 5.106-4.865 10.771-5.805 16.789l.891-4.397c.007-.031.018-.063.024-.094h38.883v5.067h-39.885zM805.885 497.432h-76.438c.08 1.352.206 2.686.388 4.002h70.571c3.146 0 4.907-1.786 5.479-4.002zM733.851 515.257a52.226 52.226 0 0 1-1.98-4.997c6.608 19.89 25.328 34.251 47.433 34.251 20.205 0 37.566-12.007 45.452-29.254h-90.905zM729.38 492.915c-.018.531-.08 1.055-.08 1.589 0 .538.063 1.059.08 1.59v-3.179zM824.77 515.229z"></path><path d="M779.303 444.505c-18.682 0-34.939 10.265-43.524 25.439 6.709-.014 19.775-.022 19.775-.022h.003v-.005c15.444 0 16.018.069 19.035.195l1.868.069c6.507.217 14.505.916 20.798 5.68 3.416 2.584 8.348 8.287 11.288 12.35 2.718 3.758 3.5 8.078 1.652 12.217-1.701 3.804-5.361 6.073-9.793 6.073H730.85l-.884-4.201c.426 2.707 1.037 5.344 1.879 7.886h94.914a49.863 49.863 0 0 0 2.546-15.682c.001-27.611-22.386-49.999-50.002-49.999z"></path></g></svg><span class="text-xl font-semibold">Blog</span></a><div class="text-gray-600 flex items-center space-x-6"><a class="text-sm hover:text-pink-600" href="https://railway.com" target="_blank" rel="noreferrer noopener">Go to Homepage</a><a class="hover:text-pink-600" href="/rss.xml" title="RSS Feed"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4 11a9 9 0 0 1 9 9"></path><path d="M4 4a16 16 0 0 1 16 16"></path><circle cx="5" cy="19" r="1"></circle></svg></a></div></nav></div><div class="min-h-screen overflow-x-hidden relative"><div class="mt-10 mb-5 px-5 md:px-8 mx-auto"><article class="max-w-6xl mx-auto mt-24 mb-12 border-b border-gray-100 pb-32"><div class="max-w-[704px] mx-auto flex items-center text-sm text-gray-600 space-x-3"><div class="flex items-center space-x-3"><div class="flex items-center"><img src="https://cms.railway.com/media/person-ray-chen-dd0fa1297e94.jpg" alt="Avatar of Ray Chen" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/></div><span>Ray Chen</span></div><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><time dateTime="2025-10-28T00:00:00.000Z">Oct 28, 2025</time></div><header class="mt-5 mb-16 max-w-[704px] mx-auto"><h1 class="text-post-title font-medium font-serif">Incident Report: October 28th, 2025</h1></header><section class="max-w-[704px] mx-auto text-base sm:text-lg"><nav style="position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border-width:0" aria-label="Table of Contents"><h2>Table of Contents</h2><ol><li style="margin-left:1.5rem"><a href="#impact">Impact</a></li><li style="margin-left:1.5rem"><a href="#incident-timeline">Incident Timeline</a></li><li style="margin-left:1.5rem"><a href="#what-happened">What Happened?</a></li><li style="margin-left:1.5rem"><a href="#preventative-measures">Preventative Measures</a></li></ol></nav><div><p class="mb-4 text-gray-800">We recently experienced an outage that impacted our backend API.</p>
<p class="mb-4 text-gray-800">When a Major Outage occurs, it is Railway’s policy to share the public details of what happened.</p>
<a class="relative no-underline" href="/p/incident-report-oct-28th-2025#impact"><span id="impact" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Impact</h2></a>
<p class="mb-4 text-gray-800">This outage rendered the dashboard inaccessible, caused CLI operations relying on our API to fail (e.g. <code class="inline-code">railway up</code>), delayed GitHub-based deployments due to webhook processing failures, and disrupted all Railway Public API operations.</p>
<p class="mb-4 text-gray-800">All running deployments and platform-level features remained online throughout this period. Users who did not access the dashboard, use the CLI, or initiate a new GitHub-based deploy during the incident window experienced no service disruption.</p>
<a class="relative no-underline" href="/p/incident-report-oct-28th-2025#incident-timeline"><span id="incident-timeline" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Incident Timeline</h2></a>
<p class="mb-4 text-gray-800">On October 28th, 2025:</p>
<ul class="list-disc pl-6 mb-4 space-y-4 text-gray-800">
<li><strong>18:23 UTC</strong> - A database change containing a schema modification was introduced</li>
<li><strong>18:34 UTC</strong> - Database change is live in production</li>
<li><strong>18:36 UTC</strong> - Internal monitoring triggered alerts, engineering team paged</li>
<li><strong>18:41 UTC</strong> - Incident declared</li>
<li><strong>18:56 UTC</strong> - Root caused identified</li>
<li><strong>19:00 UTC</strong> - Gradual service recovery confirmed</li>
<li><strong>19:15 UTC</strong> - Full service restoration confirmed</li>
</ul>
<p class="mb-4 text-gray-800">For further reference, please refer to the incident’s live updates on our Status Page on <a class="markdown-inline-link" href="https://status.railway.com/cmhawy13n00ctxzoxqbcch0no" target="_blank" rel="noreferrer noopener">Dashboard and API Service Disruption</a>.</p>
<a class="relative no-underline" href="/p/incident-report-oct-28th-2025#what-happened"><span id="what-happened" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">What Happened?</h2></a>
<p class="mb-4 text-gray-800">Railway’s client-facing interfaces (Dashboard, CLI, Public API, etc.) depend on our backend, which relies on Postgres as its primary datastore.</p>
<p class="mb-4 text-gray-800">A routine change to this Postgres database introduced a new column with an index to a table containing approximately 1 billion records. This table is critical in our backend API’s infrastructure, used by nearly all API operations.</p>
<p class="mb-4 text-gray-800">The index creation did not use Postgres’ <code class="inline-code">CONCURRENTLY</code> option, causing an exclusive lock on the entire table. During the lock period, all queries against the database were queued behind the index operation. Our API servers continued accepting requests, each attempting database connections that immediately blocked on the locked table.</p>
<p class="mb-4 text-gray-800">This triggered a cascading failure where our PgBouncer configuration exceeded the underlying Postgres connection limit, exhausting all available connection slots including administrative ones. Manual intervention attempts to terminate the index creation failed with:</p>
<pre class="mb-6"><code>FATAL: remaining connection slots are reserved for roles with privileges of the &quot;pg_use_reserved_connections&quot; role</code></pre>
<p class="mb-4 text-gray-800">With no ability to establish an administrative connection to Postgres, we could not execute commands to terminate the problematic index creation. While exploring other recovery options, the migration completed successfully after approximately 30 minutes, automatically releasing the table lock. This allowed all queued operations to process and connection pools to return to normal levels.</p>
<a class="relative no-underline" href="/p/incident-report-oct-28th-2025#preventative-measures"><span id="preventative-measures" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Preventative Measures</h2></a>
<p class="mb-4 text-gray-800">We’re going to introduce several changes to prevent errors of this class from happening again:</p>
<ul class="list-disc pl-6 mb-4 space-y-4 text-gray-800">
<li>In CI, we will enforce <code class="inline-code">CONCURRENTLY</code> usage for all index creation operations, blocking non-compliant pull requests before merge.</li>
<li>PgBouncer connection pool limits will be adjusted to prevent overwhelming the underlying Postgres instance&#x27;s capacity.</li>
<li>Database user connection limits will be configured to guarantee administrative access during incidents, ensuring maintenance operations remain possible under all conditions.</li>
</ul>
<hr class="my-8"/>
<p class="mb-4 text-gray-800">Railway is committed to providing the best-in-class cloud experience. Any downtime is unacceptable for us. We apologize for any inconvenience caused by this, and we are going to work towards eliminating the entire class of issues contributing to this incident.</p></div></section></article><div class="max-w-6xl mx-auto"><div><header class="flex items-center justify-between mb-8"><h3 class="text-gray-500 text-h3">Continue Reading...</h3><a class="text-pink-500 hover:underline" href="/engineering">View All <!-- -->Engineering<!-- --> -&gt;</a></header><div class="grid grid-cols-1 md:grid-cols-2 gap-8"><a class="flex flex-col bg-secondaryBg p-6 rounded-lg hover:bg-gray-100 group" href="/p/dev-new"><div class="flex gap-2"><div class="post-category-pill font-medium max-w-max text-xs px-2 py-[3px] rounded-[4px] tracking-[0.06em] uppercase" data-category-tone="plum">Engineering</div></div><div class="flex-grow"><header class="font-medium font-serif text-lg mt-2 mb-1">Your Cloud should come with an app builder </header><p class="text-base text-gray-800 line-clamp-2">Announcing: dev.new, an agent that can build, preview, test, and deploy an app from the same place. We tested the whole loop by asking it to ship a real-time multiplayer FPS game that runs in the browser</p></div><div class="flex items-center gap-3 mt-6"><div class="flex items-center"><img src="https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg" alt="Avatar of Mahmoud Abdelwahab" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/></div><span class="text-sm text-gray-600">Mahmoud Abdelwahab</span><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><span class="text-sm text-gray-600">Aug 13, 2026</span></div></a><a class="flex flex-col bg-secondaryBg p-6 rounded-lg hover:bg-gray-100 group" href="/p/feature-flags"><div class="flex gap-2"><div class="post-category-pill font-medium max-w-max text-xs px-2 py-[3px] rounded-[4px] tracking-[0.06em] uppercase" data-category-tone="plum">Engineering</div></div><div class="flex-grow"><header class="font-medium font-serif text-lg mt-2 mb-1">Roll it out, roll it back, never redeploy</header><p class="text-base text-gray-800 line-clamp-2">Merge now, release when you&#x27;re ready: turn a feature on for your team, then 10% of users, then everyone. Feature flags are now built into every Railway project, and your agents can run the rollout from the CLI, SDK, and MCP. No third-party service required.</p></div><div class="flex items-center gap-3 mt-6"><div class="flex items-center"><img src="https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg" alt="Avatar of Victor Ramirez" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/><img src="https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg" alt="Avatar of Mahmoud Abdelwahab" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:-8px" loading="lazy" decoding="async" width="24" height="24"/></div><span class="text-sm text-gray-600">Victor Ramirez &amp; Mahmoud Abdelwahab</span><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><span class="text-sm text-gray-600">Jul 17, 2026</span></div></a></div></div><section class="post-bottom-cta my-16 border rounded-[16px] px-6 py-16 sm:px-12 sm:py-24 flex flex-col items-center justify-center text-center" aria-labelledby="post-bottom-cta-title"><h3 id="post-bottom-cta-title" class="text-h2 font-serif font-medium">Ready to get started?</h3><p class="mt-4 text-base sm:text-lg text-gray-600">Join millions of developers deploying applications on Railway</p><div class="mt-8 flex w-full flex-col items-stretch justify-center gap-3 sm:w-auto sm:flex-row"><a class="post-bottom-cta-primary inline-flex min-h-[48px] items-center justify-center rounded-[8px] px-6 py-3 text-base font-medium no-underline transition-colors duration-100" href="https://railway.com" target="_blank" rel="noreferrer noopener">Deploy a new project</a><a class="post-bottom-cta-secondary inline-flex min-h-[48px] items-center justify-center rounded-[8px] border px-6 py-3 text-base font-medium no-underline transition-colors duration-100" href="https://railway.com/enterprise" target="_blank" rel="noreferrer noopener">Book a demo</a></div></section></div></div></div><footer class="px-5 md:px-8 py-12 border-t border-gray-100"><div class="grid grid-cols-2 sm:grid-cols-12 gap-8 sm:gap-4 lg:gap-16 max-w-6xl mx-auto"><div class="hidden sm:flex col-span-3 lg:col-span-4 flex-col justify-between"><a class="underline" href="https://railway.com" target="_blank" rel="noreferrer noopener"><svg data-v-423bf9ae="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 59.99945071644884 60" class="logo w-8"><g data-v-423bf9ae="" id="6f639fe5-02e1-4640-a1fb-f4b22b64e5ef" transform="matrix(0.5999641134746578,0,0,0.5999641134746578,-437.5538206497148,-266.68705118956507)" stroke="none"><path d="M729.872 487.327a50.86 50.86 0 0 0-.464 5.033h75.879c-.265-.518-.621-.985-.98-1.442-12.972-16.769-19.95-15.315-29.932-15.741-3.328-.137-5.585-.192-18.832-.192-7.09 0-14.798.018-22.304.038.737-1.746 1.6-3.419 2.525-5.061-2.885 5.106-4.865 10.771-5.805 16.789l.891-4.397c.007-.031.018-.063.024-.094h38.883v5.067h-39.885zM805.885 497.432h-76.438c.08 1.352.206 2.686.388 4.002h70.571c3.146 0 4.907-1.786 5.479-4.002zM733.851 515.257a52.226 52.226 0 0 1-1.98-4.997c6.608 19.89 25.328 34.251 47.433 34.251 20.205 0 37.566-12.007 45.452-29.254h-90.905zM729.38 492.915c-.018.531-.08 1.055-.08 1.589 0 .538.063 1.059.08 1.59v-3.179zM824.77 515.229z"></path><path d="M779.303 444.505c-18.682 0-34.939 10.265-43.524 25.439 6.709-.014 19.775-.022 19.775-.022h.003v-.005c15.444 0 16.018.069 19.035.195l1.868.069c6.507.217 14.505.916 20.798 5.68 3.416 2.584 8.348 8.287 11.288 12.35 2.718 3.758 3.5 8.078 1.652 12.217-1.701 3.804-5.361 6.073-9.793 6.073H730.85l-.884-4.201c.426 2.707 1.037 5.344 1.879 7.886h94.914a49.863 49.863 0 0 0 2.546-15.682c.001-27.611-22.386-49.999-50.002-49.999z"></path></g></svg></a><div class="text-xs font-medium text-gray-600 w-full">Copyright © <!-- -->2026<!-- --> Railway Corp. <br/>All rights reserved.</div></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Product</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/changelog" target="_blank" rel="noreferrer noopener">Changelog</a></li><li><a class="hover:text-foreground" href="https://railway.com/pricing" target="_blank" rel="noreferrer noopener">Pricing</a></li><li><a class="hover:text-foreground" href="https://railway.com/deploy" target="_blank" rel="noreferrer noopener">Templates</a></li><li><a class="hover:text-foreground" href="https://station.railway.com/feedback" target="_blank" rel="noreferrer noopener">Feedback</a></li><li><a class="hover:text-foreground" href="https://railway.com/open-source-kickback" target="_blank" rel="noreferrer noopener">OSS Kickback</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Company</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/about" target="_blank" rel="noreferrer noopener">About</a></li><li><a class="hover:text-foreground" href="https://railway.com/careers" target="_blank" rel="noreferrer noopener">Careers</a></li><li><a class="hover:text-foreground" href="/">Blog</a></li><li><a class="hover:text-foreground" href="https://shop.railway.com" target="_blank" rel="noreferrer noopener">Shop</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Contact</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://discord.gg/railway" target="_blank" rel="noreferrer noopener">Discord</a></li><li><a class="hover:text-foreground" href="https://twitter.com/railway" target="_blank" rel="noreferrer noopener">Twitter</a></li><li><a class="hover:text-foreground" href="https://github.com/railwayapp" target="_blank" rel="noreferrer noopener">GitHub</a></li><li><a class="hover:text-foreground" href="mailto:contact@railway.com">Email</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Legal</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/legal/acceptable-use" target="_blank" rel="noreferrer noopener">Acceptable Use</a></li><li><a class="hover:text-foreground" href="https://railway.com/legal/privacy" target="_blank" rel="noreferrer noopener">Privacy Policy</a></li><li><a class="hover:text-foreground" href="https://railway.com/legal/terms" target="_blank" rel="noreferrer noopener">Terms of Service</a></li></ul></div></div></footer></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"page":{"authors":[{"avatar":{"alt":"Ray Chen headshot","height":1096,"id":"1853","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-ray-chen-dd0fa1297e94.jpg","width":1152},"avatarUrl":"https://cms.railway.com/media/person-ray-chen-dd0fa1297e94.jpg","githubUrl":null,"id":"29","name":"Ray Chen","slug":"ray-chen","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":"We recently experienced an outage that impacted our backend API.\n\nWhen a Major Outage occurs, it is Railway’s policy to share the public details of what happened.\n\n## Impact\n\nThis outage rendered the dashboard inaccessible, caused CLI operations relying on our API to fail (e.g. `railway up`), delayed GitHub-based deployments due to webhook processing failures, and disrupted all Railway Public API operations.\n\nAll running deployments and platform-level features remained online throughout this period. Users who did not access the dashboard, use the CLI, or initiate a new GitHub-based deploy during the incident window experienced no service disruption.\n\n## Incident Timeline\n\nOn October 28th, 2025:\n\n- **18:23 UTC** - A database change containing a schema modification was introduced\n- **18:34 UTC** - Database change is live in production\n- **18:36 UTC** - Internal monitoring triggered alerts, engineering team paged\n- **18:41 UTC** - Incident declared\n- **18:56 UTC** - Root caused identified\n- **19:00 UTC** - Gradual service recovery confirmed\n- **19:15 UTC** - Full service restoration confirmed\n\nFor further reference, please refer to the incident’s live updates on our Status Page on [Dashboard and API Service Disruption](https://status.railway.com/cmhawy13n00ctxzoxqbcch0no).\n\n## What Happened?\n\nRailway’s client-facing interfaces (Dashboard, CLI, Public API, etc.) depend on our backend, which relies on Postgres as its primary datastore.\n\nA routine change to this Postgres database introduced a new column with an index to a table containing approximately 1 billion records. This table is critical in our backend API’s infrastructure, used by nearly all API operations.\n\nThe index creation did not use Postgres’ `CONCURRENTLY` option, causing an exclusive lock on the entire table. During the lock period, all queries against the database were queued behind the index operation. Our API servers continued accepting requests, each attempting database connections that immediately blocked on the locked table.\n\nThis triggered a cascading failure where our PgBouncer configuration exceeded the underlying Postgres connection limit, exhausting all available connection slots including administrative ones. Manual intervention attempts to terminate the index creation failed with:\n\n```diff\nFATAL: remaining connection slots are reserved for roles with privileges of the \"pg_use_reserved_connections\" role\n```\n\nWith no ability to establish an administrative connection to Postgres, we could not execute commands to terminate the problematic index creation. While exploring other recovery options, the migration completed successfully after approximately 30 minutes, automatically releasing the table lock. This allowed all queued operations to process and connection pools to return to normal levels.\n\n## Preventative Measures\n\nWe’re going to introduce several changes to prevent errors of this class from happening again:\n\n- In CI, we will enforce `CONCURRENTLY` usage for all index creation operations, blocking non-compliant pull requests before merge.\n- PgBouncer connection pool limits will be adjusted to prevent overwhelming the underlying Postgres instance's capacity.\n- Database user connection limits will be configured to guarantee administrative access during incidents, ensuring maintenance operations remain possible under all conditions.\n\n---\n\nRailway is committed to providing the best-in-class cloud experience. Any downtime is unacceptable for us. We apologize for any inconvenience caused by this, and we are going to work towards eliminating the entire class of issues contributing to this incident.","createdAt":"2026-06-09T12:15:49.449Z","description":"We recently experienced an outage that impacted our backend API. This disrupted dashboard access, CLI operations, GitHub-based deployment processing, and API functionality.","externalAuthor":false,"featured":false,"featuredImage":{"alt":"Incident Report: April 23rd, 2025 social image","height":null,"id":"375","mimeType":"image/png","url":"https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png","width":null},"id":"91","publishedAt":"2025-10-28T00:00:00.000Z","seoDescription":"We recently experienced an outage that impacted our backend API, disrupting dashboard access, CLI operations, GitHub deployments, and API functionality.","seoTitle":"Incident Report: October 28th, 2025","slug":"incident-report-oct-28th-2025","socialImage":{"alt":"Incident Report: April 23rd, 2025 social image","height":null,"id":"375","mimeType":"image/png","url":"https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png","width":null},"title":"Incident Report: October 28th, 2025","updatedAt":"2025-10-28T00:00:00.000Z"},"relatedPosts":[{"authors":[{"avatar":{"alt":"Mahmoud Abdelwahab headshot","height":460,"id":"1846","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","width":460},"avatarUrl":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","githubUrl":null,"id":"4","name":"Mahmoud Abdelwahab","slug":"mahmoud-abdelwahab","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":null,"createdAt":"2026-08-06T08:07:06.228Z","description":"Announcing: dev.new, an agent that can build, preview, test, and deploy an app from the same place. We tested the whole loop by asking it to ship a real-time multiplayer FPS game that runs in the browser","externalAuthor":false,"featured":true,"featuredImage":{"alt":"dev.new featured image","height":1440,"id":"1983","mimeType":"image/png","url":"https://cms.railway.com/media/dev.new-featured-image-4.png","width":3264},"id":"289","publishedAt":"2026-08-13T08:07:00.000Z","seoDescription":null,"seoTitle":null,"slug":"dev-new","socialImage":{"alt":"dev.new featured image 1","height":720,"id":"1976","mimeType":"image/png","url":"https://cms.railway.com/media/dev.new-featured-image-3.png","width":1632},"title":"Your Cloud should come with an app builder ","updatedAt":"2026-08-14T20:19:27.322Z"},{"authors":[{"avatar":{"alt":"Avatar of Victor Ramirez","height":null,"id":"1855","mimeType":"image/jpeg","url":"https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg","width":null},"avatarUrl":"https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg","githubUrl":null,"id":"34","name":"Victor Ramirez","slug":"victor-ramirez","title":null},{"avatar":{"alt":"Mahmoud Abdelwahab headshot","height":460,"id":"1846","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","width":460},"avatarUrl":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","githubUrl":null,"id":"4","name":"Mahmoud Abdelwahab","slug":"mahmoud-abdelwahab","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":null,"createdAt":"2026-07-15T22:26:48.036Z","description":"Merge now, release when you're ready: turn a feature on for your team, then 10% of users, then everyone. Feature flags are now built into every Railway project, and your agents can run the rollout from the CLI, SDK, and MCP. No third-party service required.","externalAuthor":false,"featured":false,"featuredImage":{"alt":"feature flags featured image","height":1440,"id":"1968","mimeType":"image/png","url":"https://cms.railway.com/media/feature-flags-featured-image.png","width":3264},"id":"278","publishedAt":"2026-07-17T07:00:00.000Z","seoDescription":null,"seoTitle":null,"slug":"feature-flags","socialImage":null,"title":"Roll it out, roll it back, never redeploy","updatedAt":"2026-08-13T16:53:55.126Z"}]},"__N_SSG":true},"page":"/p/[slug]","query":{"slug":"incident-report-oct-28th-2025"},"buildId":"nFX7R3M0MdLMAf-Ufh9qa","isFallback":false,"isExperimentalCompile":false,"gsp":true,"scriptLoader":[]}</script></body></html>