247 lines
46 KiB
HTML
247 lines
46 KiB
HTML
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8" data-next-head=""/><meta name="viewport" content="width=device-width" data-next-head=""/><meta name="twitter:card" content="summary_large_image" data-next-head=""/><meta name="twitter:site" content="@Railway" data-next-head=""/><meta name="twitter:creator" content="@Railway" data-next-head=""/><meta property="og:image:type" content="image/png" data-next-head=""/><meta property="og:image:width" content="1200" data-next-head=""/><meta property="og:image:height" content="630" data-next-head=""/><meta property="og:site_name" content="Railway Blog" data-next-head=""/><title data-next-head="">Incident Report: Dec 1st, 2023</title><meta name="robots" content="index,follow" data-next-head=""/><meta name="description" content="We recently experienced an outage that affected 30% of our US-West compute infrastructure and caused workloads to be unreachable." data-next-head=""/><meta property="og:title" content="Incident Report: Dec 1st, 2023" data-next-head=""/><meta property="og:description" content="We recently experienced an outage that affected 30% of our US-West compute infrastructure and caused workloads to be unreachable." data-next-head=""/><meta property="og:url" content="https://blog.railway.com/p/2023-12-01-incident-report" data-next-head=""/><meta property="og:type" content="article" data-next-head=""/><meta property="article:published_time" content="2023-12-01T20:00:00.000Z" data-next-head=""/><meta property="article:modified_time" content="2023-12-01T20:00:00.000Z" data-next-head=""/><meta property="article:author" content="Angelo Saraceno" data-next-head=""/><meta property="article:section" content="Engineering" data-next-head=""/><meta property="og:image" content="https://cms.railway.com/media/37247bde529d43cd1c3d4274f8fe1d51af25ad9ae5f752f6f9b83f5e65218ee4.png?w=1200&q=90" data-next-head=""/><link rel="canonical" href="https://blog.railway.com/p/2023-12-01-incident-report" data-next-head=""/><meta name="twitter:title" content="Incident Report: Dec 1st, 2023" data-next-head=""/><meta name="twitter:description" content="We recently experienced an outage that affected 30% of our US-West compute infrastructure and caused workloads to be unreachable." data-next-head=""/><meta name="twitter:image" content="https://cms.railway.com/media/37247bde529d43cd1c3d4274f8fe1d51af25ad9ae5f752f6f9b83f5e65218ee4.png?w=1200&q=90" data-next-head=""/><link rel="alternate" type="text/markdown" href="/p/2023-12-01-incident-report.md" data-next-head=""/><link rel="alternate" type="application/rss+xml" title="Railway Blog" href="https://blog.railway.com/rss.xml"/><link rel="preload" href="/_next/static/media/e4af272ccee01ff0-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/91c3caac73720166-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/3359c9acc30ef440-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/aff755c8b4bb3089-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/3988c35203e0830a-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/427e4a37d3642943-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/css/619a01ff31a99955.css" as="style"/><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"BlogPosting","headline":"Incident Report: Dec 1st, 2023","description":"We recently experienced an outage that affected 30% of our US-West compute infrastructure and caused workloads to be unreachable.","image":["https://cms.railway.com/media/37247bde529d43cd1c3d4274f8fe1d51af25ad9ae5f752f6f9b83f5e65218ee4.png?w=1200&q=90"],"datePublished":"2023-12-01T20:00:00.000Z","dateModified":"2023-12-01T20:00:00.000Z","author":{"@type":"Person","name":"Angelo Saraceno"},"publisher":{"@type":"Organization","name":"Railway","logo":{"@type":"ImageObject","url":"https://railway.com/brand/logo-dark.png"}},"mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.railway.com/p/2023-12-01-incident-report"}}</script><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https://blog.railway.com"},{"@type":"ListItem","position":2,"name":"Engineering","item":"https://blog.railway.com/engineering"},{"@type":"ListItem","position":3,"name":"Incident Report: Dec 1st, 2023","item":"https://blog.railway.com/p/2023-12-01-incident-report"}]}</script><style>:root {
|
||
--foreground: hsl(0, 0%, 100%);
|
||
--background: #181622;
|
||
--secondaryBg: hsl(250, 21%, 11%);
|
||
--blue-50: hsl(220, 55%, 10%);
|
||
--blue-100: hsl(220, 55%, 13%);
|
||
--blue-200: hsl(220, 62%, 25%);
|
||
--blue-300: hsl(220, 68%, 35%);
|
||
--blue-400: hsl(220, 72%, 45%);
|
||
--blue-500: hsl(220, 80%, 55%);
|
||
--blue-600: hsl(220, 80%, 65%);
|
||
--blue-700: hsl(220, 80%, 75%);
|
||
--blue-800: hsl(220, 80%, 85%);
|
||
--blue-900: hsl(220, 80%, 95%);
|
||
--blue-950: hsl(220, 55%, 97%);
|
||
--gray-50: hsl(248, 21%, 13%);
|
||
--gray-100: hsl(246, 18%, 15%);
|
||
--gray-200: hsl(246, 11%, 22%);
|
||
--gray-300: hsl(246, 8%, 35%);
|
||
--gray-400: hsl(246, 7%, 45%);
|
||
--gray-500: hsl(246, 6%, 55%);
|
||
--gray-600: hsl(246, 6%, 65%);
|
||
--gray-700: hsl(246, 6%, 78%);
|
||
--gray-800: hsl(246, 6%, 87%);
|
||
--gray-900: hsl(246, 6%, 95%);
|
||
--gray-950: hsl(246, 6%, 95%);
|
||
--red-50: hsl(1, 35%, 10%);
|
||
--red-100: hsl(1, 45%, 12%);
|
||
--red-200: hsl(1, 55%, 20%);
|
||
--red-300: hsl(1, 62%, 28%);
|
||
--red-400: hsl(1, 62%, 35%);
|
||
--red-500: hsl(1, 62%, 44%);
|
||
--red-600: hsl(1, 62%, 60%);
|
||
--red-700: hsl(1, 62%, 76%);
|
||
--red-800: hsl(1, 64%, 85%);
|
||
--red-900: hsl(1, 68%, 95%);
|
||
--red-950: hsl(1, 55%, 98%);
|
||
--pink-50: hsl(270, 40%, 16%);
|
||
--pink-100: hsl(270, 40%, 16%);
|
||
--pink-200: hsl(270, 45%, 24%);
|
||
--pink-300: hsl(270, 50%, 32%);
|
||
--pink-400: hsl(270, 55%, 43%);
|
||
--pink-500: hsl(270, 60%, 52%);
|
||
--pink-600: hsl(270, 70%, 65%);
|
||
--pink-700: hsl(270, 70%, 75%);
|
||
--pink-800: hsl(270, 70%, 85%);
|
||
--pink-900: hsl(270, 70%, 95%);
|
||
--pink-950: hsl(270, 70%, 95%);
|
||
--green-50: hsl(152, 15%, 10%);
|
||
--green-100: hsl(152, 26%, 11%);
|
||
--green-200: hsl(152, 32%, 16%);
|
||
--green-300: hsl(152, 38%, 24%);
|
||
--green-400: hsl(152, 38%, 34%);
|
||
--green-500: hsl(152, 38%, 42%);
|
||
--green-600: hsl(152, 38%, 60%);
|
||
--green-700: hsl(152, 38%, 70%);
|
||
--green-800: hsl(152, 38%, 80%);
|
||
--green-900: hsl(152, 38%, 91%);
|
||
--green-950: hsl(152, 40%, 97%);
|
||
--yellow-50: hsl(44, 95%, 12%);
|
||
--yellow-100: hsl(44, 95%, 12%);
|
||
--yellow-200: hsl(44, 96%, 24%);
|
||
--yellow-300: hsl(44, 95%, 36%);
|
||
--yellow-400: hsl(44, 95%, 48%);
|
||
--yellow-500: hsl(44, 95%, 60%);
|
||
--yellow-600: hsl(44, 95%, 69%);
|
||
--yellow-700: hsl(44, 95%, 78%);
|
||
--yellow-800: hsl(44, 95%, 86%);
|
||
--yellow-900: hsl(44, 95%, 95%);
|
||
--yellow-950: hsl(44, 95%, 95%);
|
||
}
|
||
|
||
.light {
|
||
--foreground: hsl(250, 24%, 9%);
|
||
--background: #F1F0EF;
|
||
--secondaryBg: hsl(0, 0%, 98%);
|
||
--blue-50: hsl(220, 55%, 97%);
|
||
--blue-100: hsl(220, 80%, 95%);
|
||
--blue-200: hsl(220, 80%, 85%);
|
||
--blue-300: hsl(220, 80%, 75%);
|
||
--blue-400: hsl(220, 80%, 65%);
|
||
--blue-500: hsl(220, 80%, 55%);
|
||
--blue-600: hsl(220, 72%, 45%);
|
||
--blue-700: hsl(220, 68%, 35%);
|
||
--blue-800: hsl(220, 62%, 25%);
|
||
--blue-900: hsl(220, 55%, 13%);
|
||
--blue-950: hsl(220, 55%, 10%);
|
||
--gray-50: hsl(246, 6%, 95%);
|
||
--gray-100: hsl(246, 6%, 95%);
|
||
--gray-200: hsl(246, 6%, 87%);
|
||
--gray-300: hsl(246, 6%, 78%);
|
||
--gray-400: hsl(246, 6%, 65%);
|
||
--gray-500: hsl(246, 6%, 55%);
|
||
--gray-600: hsl(246, 7%, 45%);
|
||
--gray-700: hsl(246, 8%, 35%);
|
||
--gray-800: hsl(246, 11%, 22%);
|
||
--gray-900: hsl(246, 18%, 15%);
|
||
--gray-950: hsl(248, 21%, 13%);
|
||
--red-50: hsl(1, 55%, 98%);
|
||
--red-100: hsl(1, 68%, 95%);
|
||
--red-200: hsl(1, 64%, 85%);
|
||
--red-300: hsl(1, 62%, 76%);
|
||
--red-400: hsl(1, 62%, 60%);
|
||
--red-500: hsl(1, 62%, 44%);
|
||
--red-600: hsl(1, 62%, 35%);
|
||
--red-700: hsl(1, 62%, 28%);
|
||
--red-800: hsl(1, 55%, 20%);
|
||
--red-900: hsl(1, 45%, 12%);
|
||
--red-950: hsl(1, 35%, 10%);
|
||
--pink-50: hsl(270, 70%, 95%);
|
||
--pink-100: hsl(270, 70%, 95%);
|
||
--pink-200: hsl(270, 70%, 85%);
|
||
--pink-300: hsl(270, 70%, 75%);
|
||
--pink-400: hsl(270, 70%, 65%);
|
||
--pink-500: hsl(270, 60%, 52%);
|
||
--pink-600: hsl(270, 55%, 43%);
|
||
--pink-700: hsl(270, 50%, 32%);
|
||
--pink-800: hsl(270, 45%, 24%);
|
||
--pink-900: hsl(270, 40%, 16%);
|
||
--pink-950: hsl(270, 40%, 16%);
|
||
--green-50: hsl(152, 40%, 97%);
|
||
--green-100: hsl(152, 38%, 91%);
|
||
--green-200: hsl(152, 38%, 80%);
|
||
--green-300: hsl(152, 38%, 70%);
|
||
--green-400: hsl(152, 38%, 60%);
|
||
--green-500: hsl(152, 38%, 42%);
|
||
--green-600: hsl(152, 38%, 34%);
|
||
--green-700: hsl(152, 38%, 24%);
|
||
--green-800: hsl(152, 32%, 16%);
|
||
--green-900: hsl(152, 26%, 11%);
|
||
--green-950: hsl(152, 15%, 10%);
|
||
--yellow-50: hsl(44, 95%, 95%);
|
||
--yellow-100: hsl(44, 95%, 95%);
|
||
--yellow-200: hsl(44, 95%, 86%);
|
||
--yellow-300: hsl(44, 95%, 78%);
|
||
--yellow-400: hsl(44, 95%, 69%);
|
||
--yellow-500: hsl(44, 95%, 60%);
|
||
--yellow-600: hsl(44, 95%, 48%);
|
||
--yellow-700: hsl(44, 95%, 36%);
|
||
--yellow-800: hsl(44, 96%, 24%);
|
||
--yellow-900: hsl(44, 95%, 12%);
|
||
--yellow-950: hsl(44, 95%, 12%);
|
||
}
|
||
|
||
.dark {
|
||
--foreground: hsl(0, 0%, 100%);
|
||
--background: #181622;
|
||
--secondaryBg: hsl(250, 21%, 11%);
|
||
--blue-50: hsl(220, 55%, 10%);
|
||
--blue-100: hsl(220, 55%, 13%);
|
||
--blue-200: hsl(220, 62%, 25%);
|
||
--blue-300: hsl(220, 68%, 35%);
|
||
--blue-400: hsl(220, 72%, 45%);
|
||
--blue-500: hsl(220, 80%, 55%);
|
||
--blue-600: hsl(220, 80%, 65%);
|
||
--blue-700: hsl(220, 80%, 75%);
|
||
--blue-800: hsl(220, 80%, 85%);
|
||
--blue-900: hsl(220, 80%, 95%);
|
||
--blue-950: hsl(220, 55%, 97%);
|
||
--gray-50: hsl(248, 21%, 13%);
|
||
--gray-100: hsl(246, 18%, 15%);
|
||
--gray-200: hsl(246, 11%, 22%);
|
||
--gray-300: hsl(246, 8%, 35%);
|
||
--gray-400: hsl(246, 7%, 45%);
|
||
--gray-500: hsl(246, 6%, 55%);
|
||
--gray-600: hsl(246, 6%, 65%);
|
||
--gray-700: hsl(246, 6%, 78%);
|
||
--gray-800: hsl(246, 6%, 87%);
|
||
--gray-900: hsl(246, 6%, 95%);
|
||
--gray-950: hsl(246, 6%, 95%);
|
||
--red-50: hsl(1, 35%, 10%);
|
||
--red-100: hsl(1, 45%, 12%);
|
||
--red-200: hsl(1, 55%, 20%);
|
||
--red-300: hsl(1, 62%, 28%);
|
||
--red-400: hsl(1, 62%, 35%);
|
||
--red-500: hsl(1, 62%, 44%);
|
||
--red-600: hsl(1, 62%, 60%);
|
||
--red-700: hsl(1, 62%, 76%);
|
||
--red-800: hsl(1, 64%, 85%);
|
||
--red-900: hsl(1, 68%, 95%);
|
||
--red-950: hsl(1, 55%, 98%);
|
||
--pink-50: hsl(270, 40%, 16%);
|
||
--pink-100: hsl(270, 40%, 16%);
|
||
--pink-200: hsl(270, 45%, 24%);
|
||
--pink-300: hsl(270, 50%, 32%);
|
||
--pink-400: hsl(270, 55%, 43%);
|
||
--pink-500: hsl(270, 60%, 52%);
|
||
--pink-600: hsl(270, 70%, 65%);
|
||
--pink-700: hsl(270, 70%, 75%);
|
||
--pink-800: hsl(270, 70%, 85%);
|
||
--pink-900: hsl(270, 70%, 95%);
|
||
--pink-950: hsl(270, 70%, 95%);
|
||
--green-50: hsl(152, 15%, 10%);
|
||
--green-100: hsl(152, 26%, 11%);
|
||
--green-200: hsl(152, 32%, 16%);
|
||
--green-300: hsl(152, 38%, 24%);
|
||
--green-400: hsl(152, 38%, 34%);
|
||
--green-500: hsl(152, 38%, 42%);
|
||
--green-600: hsl(152, 38%, 60%);
|
||
--green-700: hsl(152, 38%, 70%);
|
||
--green-800: hsl(152, 38%, 80%);
|
||
--green-900: hsl(152, 38%, 91%);
|
||
--green-950: hsl(152, 40%, 97%);
|
||
--yellow-50: hsl(44, 95%, 12%);
|
||
--yellow-100: hsl(44, 95%, 12%);
|
||
--yellow-200: hsl(44, 96%, 24%);
|
||
--yellow-300: hsl(44, 95%, 36%);
|
||
--yellow-400: hsl(44, 95%, 48%);
|
||
--yellow-500: hsl(44, 95%, 60%);
|
||
--yellow-600: hsl(44, 95%, 69%);
|
||
--yellow-700: hsl(44, 95%, 78%);
|
||
--yellow-800: hsl(44, 95%, 86%);
|
||
--yellow-900: hsl(44, 95%, 95%);
|
||
--yellow-950: hsl(44, 95%, 95%);
|
||
}</style><link rel="stylesheet" href="/_next/static/css/619a01ff31a99955.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" noModule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-0b474a604a067f92.js" defer=""></script><script src="/_next/static/chunks/framework-90fc9b0f7c4706c1.js" defer=""></script><script src="/_next/static/chunks/main-e89abd6570bbb9dc.js" defer=""></script><script src="/_next/static/chunks/pages/_app-3049b372678b389f.js" defer=""></script><script src="/_next/static/chunks/5399-d49efedd870f41de.js" defer=""></script><script src="/_next/static/chunks/3801-cdd2681d27083ad8.js" defer=""></script><script src="/_next/static/chunks/9594-1c44b12020a3af0a.js" defer=""></script><script src="/_next/static/chunks/4696-fd1712bc36a5bc50.js" defer=""></script><script src="/_next/static/chunks/pages/p/%5Bslug%5D-ddadb1ca54eaab0f.js" defer=""></script><script src="/_next/static/nFX7R3M0MdLMAf-Ufh9qa/_buildManifest.js" defer=""></script><script src="/_next/static/nFX7R3M0MdLMAf-Ufh9qa/_ssgManifest.js" defer=""></script></head><body><div id="__next"><script>!function(){try{var d=document.documentElement,c=d.classList;c.remove('light','dark');var e=localStorage.getItem('theme');if('system'===e||(!e&&true)){var t='(prefers-color-scheme: dark)',m=window.matchMedia(t);if(m.media!==t||m.matches){d.style.colorScheme = 'dark';c.add('dark')}else{d.style.colorScheme = 'light';c.add('light')}}else if(e){c.add(e|| '')}if(e==='light'||e==='dark')d.style.colorScheme=e}catch(e){}}()</script><div class="__variable_f367f3 __variable_0b668e __variable_ecea63 font-sans"><div class="px-5 md:px-8"><nav class="max-w-6xl mx-auto py-6 flex justify-between items-center border-b border-gray-100"><a class="flex items-center space-x-4" href="/"><svg data-v-423bf9ae="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 59.99945071644884 60" class="logo w-8"><g data-v-423bf9ae="" id="6f639fe5-02e1-4640-a1fb-f4b22b64e5ef" transform="matrix(0.5999641134746578,0,0,0.5999641134746578,-437.5538206497148,-266.68705118956507)" stroke="none"><path d="M729.872 487.327a50.86 50.86 0 0 0-.464 5.033h75.879c-.265-.518-.621-.985-.98-1.442-12.972-16.769-19.95-15.315-29.932-15.741-3.328-.137-5.585-.192-18.832-.192-7.09 0-14.798.018-22.304.038.737-1.746 1.6-3.419 2.525-5.061-2.885 5.106-4.865 10.771-5.805 16.789l.891-4.397c.007-.031.018-.063.024-.094h38.883v5.067h-39.885zM805.885 497.432h-76.438c.08 1.352.206 2.686.388 4.002h70.571c3.146 0 4.907-1.786 5.479-4.002zM733.851 515.257a52.226 52.226 0 0 1-1.98-4.997c6.608 19.89 25.328 34.251 47.433 34.251 20.205 0 37.566-12.007 45.452-29.254h-90.905zM729.38 492.915c-.018.531-.08 1.055-.08 1.589 0 .538.063 1.059.08 1.59v-3.179zM824.77 515.229z"></path><path d="M779.303 444.505c-18.682 0-34.939 10.265-43.524 25.439 6.709-.014 19.775-.022 19.775-.022h.003v-.005c15.444 0 16.018.069 19.035.195l1.868.069c6.507.217 14.505.916 20.798 5.68 3.416 2.584 8.348 8.287 11.288 12.35 2.718 3.758 3.5 8.078 1.652 12.217-1.701 3.804-5.361 6.073-9.793 6.073H730.85l-.884-4.201c.426 2.707 1.037 5.344 1.879 7.886h94.914a49.863 49.863 0 0 0 2.546-15.682c.001-27.611-22.386-49.999-50.002-49.999z"></path></g></svg><span class="text-xl font-semibold">Blog</span></a><div class="text-gray-600 flex items-center space-x-6"><a class="text-sm hover:text-pink-600" href="https://railway.com" target="_blank" rel="noreferrer noopener">Go to Homepage</a><a class="hover:text-pink-600" href="/rss.xml" title="RSS Feed"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4 11a9 9 0 0 1 9 9"></path><path d="M4 4a16 16 0 0 1 16 16"></path><circle cx="5" cy="19" r="1"></circle></svg></a></div></nav></div><div class="min-h-screen overflow-x-hidden relative"><div class="mt-10 mb-5 px-5 md:px-8 mx-auto"><article class="max-w-6xl mx-auto mt-24 mb-12 border-b border-gray-100 pb-32"><div class="max-w-[704px] mx-auto flex items-center text-sm text-gray-600 space-x-3"><div class="flex items-center space-x-3"><div class="flex items-center"><img src="https://cms.railway.com/media/6b99ad48a4bd2571569575090858205c1e2697f5308e4a0781c1073c3d87cc31.jpg" alt="Avatar of Angelo Saraceno" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/></div><span>Angelo Saraceno</span></div><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><time dateTime="2023-12-01T20:00:00.000Z">Dec 1, 2023</time></div><header class="mt-5 mb-16 max-w-[704px] mx-auto"><h1 class="text-post-title font-medium font-serif">Incident Report: Dec 1st, 2023</h1></header><section class="max-w-[704px] mx-auto text-base sm:text-lg"><nav style="position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border-width:0" aria-label="Table of Contents"><h2>Table of Contents</h2><ol><li style="margin-left:0rem"><a href="#summary">Summary</a></li><li style="margin-left:0rem"><a href="#incident-details">Incident Details</a></li><li style="margin-left:1.5rem"><a href="#response-and-resolution">Response and Resolution</a></li><li style="margin-left:0rem"><a href="#preventative-measures">Preventative Measures</a></li><li style="margin-left:1.5rem"><a href="#incoming-mitigations">Incoming Mitigations</a></li><li style="margin-left:0rem"><a href="#moving-forward">Moving Forward</a></li></ol></nav><div><p class="mb-4 text-gray-800">We recently experienced an outage on our platform that affected 30% of our US-West compute infrastructure and caused workloads to be unreachable. When production outages occur, it is Railway’s policy to share the public details of what occurred.</p>
|
||
<a class="relative no-underline" href="/p/2023-12-01-incident-report#summary"><span id="summary" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-16 mb-8">Summary</h2></a>
|
||
<p class="mb-4 text-gray-800">Between 16:40 UTC and 20:53 UTC, Railway’s engineers received alerts from internal monitoring that users were unable to connect to their workloads. This was due to high resource usage leading to a soft lock of all cores on the affected hosts. This led to rolling restarts of hosts as we attempted recovery on machines hosting customer workloads.</p>
|
||
<p class="mb-4 text-gray-800">Railway engineers began intervention immediately after notification. Due to the similarity with prior incidents the Railway team has had with our underlying compute platform, we believed it was related to our vendor infrastructure — but potentially triggered by a code change we deployed.</p>
|
||
<p class="mb-4 text-gray-800">30% of the total machine fleet within US-West was affected due to the outage. Some customer workloads were offline for periods ranging from 10 minutes to 1 hour as we worked to mitigate the issue.</p>
|
||
<p class="mb-4 text-gray-800">The Railway infrastructure engineering team were able to attribute the root cause of this incident to a newly deployed metrics collection agent that appeared to trigger a CPU core soft lock on these hosts, despite them being well below resource thresholds. We believe the underlying issue is potentially within the Google Cloud Hypervisor. We’ve documented our findings in <a class="markdown-inline-link" href="https://blog.railway.com/p/gcp-incidents" target="_blank" rel="noreferrer noopener">this blogpost</a>.</p>
|
||
<p class="mb-4 text-gray-800">After diagnosing and remediating the issue, the Infrastructure team was able to restore all workloads with no data loss reported.</p>
|
||
<a class="relative no-underline" href="/p/2023-12-01-incident-report#incident-details"><span id="incident-details" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-16 mb-8">Incident Details</h2></a>
|
||
<p class="mb-4 text-gray-800">Railway’s engineering team is undertaking a migration to a new metrics collection service which will deliver significant benefit to Railway’s customer base.</p>
|
||
<p class="mb-4 text-gray-800">Earlier on Dec 1st, at 16:02 UTC, the Railway engineering began rolling out the newer metrics collection service on Railway infrastructure. This rollout had been previously validated on a subset of Railway’s infrastructure and had not triggered any warnings at that point in time.</p>
|
||
<p class="mb-4 text-gray-800">However, during the rollout, our deployment scripts began reporting that certain hosts were unavailable and the update could not be applied. After we noticed the first couple boxes coming offline, we went back to check the deployment logs. Since our first priority was to bring workloads back, by the time we identified the issue in the logs, the incident was well under way.</p>
|
||
<p class="mb-4 text-gray-800">As the metrics collector started up, a configuration error meant it began analyzing processes beyond its original mandate — essentially sampling all processes on a host at a high frequency. This added monitoring resulted in large transfers of memory between the kernelspace and userspace on our hosts, and even though overall resource pressure on the hosts were within nominal thresholds, triggered cascading CPU core soft locks which brought the host down. These reports are inline with a bug Google acknowledged <a class="markdown-inline-link" href="https://github.com/kubernetes/kubernetes/issues/37853#issuecomment-264587949" target="_blank" rel="noreferrer noopener">here</a>.</p>
|
||
<p class="mb-4 text-gray-800">This issue only affected a subset of machines randomly throughout the fleet. The metrics collector was operating perfectly on 70% of the fleet. This meant we initially didn’t suspect it as a cause. Furthermore, the failure scenario wasn’t immediate — the impacted machines failed unpredictably in bursts with no discernible pattern in the metrics, workload patterns, or log data that we had been able to pull from the unaffected machines.</p>
|
||
<p class="mb-4 text-gray-800">The misconfigured agent caused CPU core locks across all cores on the affected machines, which made workloads (including our own daemons through which we manage the machine) un-schedulable on the CPU.</p>
|
||
<p class="mb-4 text-gray-800">At 17:05 UTC, Railway’s engineering team began manual restarts to recover affected hosts. Because we needed to do these restarts manually — especially after confirming that the host CPU was soft-locked by way of checking serial console logs — the process took between 10-30 minutes per host. However, though we recovered most hosts before 18:30 UTC, several others failed during this time and thus took longer to recover. Railway’s engineering team began engaging additional members of the Customer Success and Support teams to communicate the impact to affected customers across the Community Forum, Email, Twitter, and Discord.</p>
|
||
<a class="relative no-underline" href="/p/2023-12-01-incident-report#response-and-resolution"><span id="response-and-resolution" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Response and Resolution</h2></a>
|
||
<p class="mb-4 text-gray-800">Around 18:30 UTC, customers started reporting that active resolution measures were successful. The Railway engineering team then catalogued all failed machines and failed over additional hosts that we suspected to be impacted. While this action was ongoing, additional monitoring was conducted to ensure that the machines with customer workloads were all coming back online appropriately.</p>
|
||
<p class="mb-4 text-gray-800">By 20:23 UTC all hosts were back online and we had total confirmation from customers that workloads were restored.</p>
|
||
<p class="mb-4 text-gray-800">We declared the incident to be over at 20:53 UTC.</p>
|
||
<a class="relative no-underline" href="/p/2023-12-01-incident-report#preventative-measures"><span id="preventative-measures" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-16 mb-8"><strong>Preventative Measures</strong></h2></a>
|
||
<p class="mb-4 text-gray-800">This incident resulted in the following takeaways:</p>
|
||
<ol class="list-decimal pl-6 mb-4 space-y-4 text-gray-800">
|
||
<li><strong>Improve internal deployment tooling:</strong> We need to shorten feedback time on deployments, helping us identify issues faster.</li>
|
||
<li><strong>Build more resilience:</strong> We need to build better mechanisms to detect and automatically recover from OS/hardware/hypervisor level failures by isolating the blast radius of changes.</li>
|
||
<li><strong>Removing the black box</strong>: We’re already building our cloud independent distributed storage system and making forays into bare metal. We can only deliver the level of service we intend to by controlling, and having visibility into, the entire stack — from silicon to hypervisor.</li>
|
||
</ol>
|
||
<a class="relative no-underline" href="/p/2023-12-01-incident-report#incoming-mitigations"><span id="incoming-mitigations" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Incoming Mitigations</h2></a>
|
||
<ol class="list-decimal pl-6 mb-4 space-y-4 text-gray-800">
|
||
<li><strong>Better System Checks:</strong> We're improving how we check the health and status of our systems to catch issues faster.</li>
|
||
<li><strong>Mandate Staged Rollouts for all Services:</strong> For riskier changes, we already stage rollouts. If this incident has taught us anything, it’s that there’s risk in even the most mundane of changes at our scale. Moving forward, for all services, we’ll start with 10% of our systems, then move to 25%, 50%, and finally 100%, carefully monitoring at each stage</li>
|
||
</ol>
|
||
<a class="relative no-underline" href="/p/2023-12-01-incident-report#moving-forward"><span id="moving-forward" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-16 mb-8">Moving Forward</h2></a>
|
||
<p class="mb-4 text-gray-800">The Railway team is looking to continually improve our processes with our customers and our community partners. Although we are glad that our internal systems caught it before our customers did, we truly regret the impact. We are in active contact with our customers who faced business impact from this outage.</p></div></section></article><div class="max-w-6xl mx-auto"><div><header class="flex items-center justify-between mb-8"><h3 class="text-gray-500 text-h3">Continue Reading...</h3><a class="text-pink-500 hover:underline" href="/engineering">View All <!-- -->Engineering<!-- --> -></a></header><div class="grid grid-cols-1 md:grid-cols-2 gap-8"><a class="flex flex-col bg-secondaryBg p-6 rounded-lg hover:bg-gray-100 group" href="/p/dev-new"><div class="flex gap-2"><div class="post-category-pill font-medium max-w-max text-xs px-2 py-[3px] rounded-[4px] tracking-[0.06em] uppercase" data-category-tone="plum">Engineering</div></div><div class="flex-grow"><header class="font-medium font-serif text-lg mt-2 mb-1">Your Cloud should come with an app builder </header><p class="text-base text-gray-800 line-clamp-2">Announcing: dev.new, an agent that can build, preview, test, and deploy an app from the same place. We tested the whole loop by asking it to ship a real-time multiplayer FPS game that runs in the browser</p></div><div class="flex items-center gap-3 mt-6"><div class="flex items-center"><img src="https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg" alt="Avatar of Mahmoud Abdelwahab" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/></div><span class="text-sm text-gray-600">Mahmoud Abdelwahab</span><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><span class="text-sm text-gray-600">Aug 13, 2026</span></div></a><a class="flex flex-col bg-secondaryBg p-6 rounded-lg hover:bg-gray-100 group" href="/p/feature-flags"><div class="flex gap-2"><div class="post-category-pill font-medium max-w-max text-xs px-2 py-[3px] rounded-[4px] tracking-[0.06em] uppercase" data-category-tone="plum">Engineering</div></div><div class="flex-grow"><header class="font-medium font-serif text-lg mt-2 mb-1">Roll it out, roll it back, never redeploy</header><p class="text-base text-gray-800 line-clamp-2">Merge now, release when you're ready: turn a feature on for your team, then 10% of users, then everyone. Feature flags are now built into every Railway project, and your agents can run the rollout from the CLI, SDK, and MCP. No third-party service required.</p></div><div class="flex items-center gap-3 mt-6"><div class="flex items-center"><img src="https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg" alt="Avatar of Victor Ramirez" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/><img src="https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg" alt="Avatar of Mahmoud Abdelwahab" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:-8px" loading="lazy" decoding="async" width="24" height="24"/></div><span class="text-sm text-gray-600">Victor Ramirez & Mahmoud Abdelwahab</span><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><span class="text-sm text-gray-600">Jul 17, 2026</span></div></a></div></div><section class="post-bottom-cta my-16 border rounded-[16px] px-6 py-16 sm:px-12 sm:py-24 flex flex-col items-center justify-center text-center" aria-labelledby="post-bottom-cta-title"><h3 id="post-bottom-cta-title" class="text-h2 font-serif font-medium">Ready to get started?</h3><p class="mt-4 text-base sm:text-lg text-gray-600">Join millions of developers deploying applications on Railway</p><div class="mt-8 flex w-full flex-col items-stretch justify-center gap-3 sm:w-auto sm:flex-row"><a class="post-bottom-cta-primary inline-flex min-h-[48px] items-center justify-center rounded-[8px] px-6 py-3 text-base font-medium no-underline transition-colors duration-100" href="https://railway.com" target="_blank" rel="noreferrer noopener">Deploy a new project</a><a class="post-bottom-cta-secondary inline-flex min-h-[48px] items-center justify-center rounded-[8px] border px-6 py-3 text-base font-medium no-underline transition-colors duration-100" href="https://railway.com/enterprise" target="_blank" rel="noreferrer noopener">Book a demo</a></div></section></div></div></div><footer class="px-5 md:px-8 py-12 border-t border-gray-100"><div class="grid grid-cols-2 sm:grid-cols-12 gap-8 sm:gap-4 lg:gap-16 max-w-6xl mx-auto"><div class="hidden sm:flex col-span-3 lg:col-span-4 flex-col justify-between"><a class="underline" href="https://railway.com" target="_blank" rel="noreferrer noopener"><svg data-v-423bf9ae="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 59.99945071644884 60" class="logo w-8"><g data-v-423bf9ae="" id="6f639fe5-02e1-4640-a1fb-f4b22b64e5ef" transform="matrix(0.5999641134746578,0,0,0.5999641134746578,-437.5538206497148,-266.68705118956507)" stroke="none"><path d="M729.872 487.327a50.86 50.86 0 0 0-.464 5.033h75.879c-.265-.518-.621-.985-.98-1.442-12.972-16.769-19.95-15.315-29.932-15.741-3.328-.137-5.585-.192-18.832-.192-7.09 0-14.798.018-22.304.038.737-1.746 1.6-3.419 2.525-5.061-2.885 5.106-4.865 10.771-5.805 16.789l.891-4.397c.007-.031.018-.063.024-.094h38.883v5.067h-39.885zM805.885 497.432h-76.438c.08 1.352.206 2.686.388 4.002h70.571c3.146 0 4.907-1.786 5.479-4.002zM733.851 515.257a52.226 52.226 0 0 1-1.98-4.997c6.608 19.89 25.328 34.251 47.433 34.251 20.205 0 37.566-12.007 45.452-29.254h-90.905zM729.38 492.915c-.018.531-.08 1.055-.08 1.589 0 .538.063 1.059.08 1.59v-3.179zM824.77 515.229z"></path><path d="M779.303 444.505c-18.682 0-34.939 10.265-43.524 25.439 6.709-.014 19.775-.022 19.775-.022h.003v-.005c15.444 0 16.018.069 19.035.195l1.868.069c6.507.217 14.505.916 20.798 5.68 3.416 2.584 8.348 8.287 11.288 12.35 2.718 3.758 3.5 8.078 1.652 12.217-1.701 3.804-5.361 6.073-9.793 6.073H730.85l-.884-4.201c.426 2.707 1.037 5.344 1.879 7.886h94.914a49.863 49.863 0 0 0 2.546-15.682c.001-27.611-22.386-49.999-50.002-49.999z"></path></g></svg></a><div class="text-xs font-medium text-gray-600 w-full">Copyright © <!-- -->2026<!-- --> Railway Corp. <br/>All rights reserved.</div></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Product</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/changelog" target="_blank" rel="noreferrer noopener">Changelog</a></li><li><a class="hover:text-foreground" href="https://railway.com/pricing" target="_blank" rel="noreferrer noopener">Pricing</a></li><li><a class="hover:text-foreground" href="https://railway.com/deploy" target="_blank" rel="noreferrer noopener">Templates</a></li><li><a class="hover:text-foreground" href="https://station.railway.com/feedback" target="_blank" rel="noreferrer noopener">Feedback</a></li><li><a class="hover:text-foreground" href="https://railway.com/open-source-kickback" target="_blank" rel="noreferrer noopener">OSS Kickback</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Company</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/about" target="_blank" rel="noreferrer noopener">About</a></li><li><a class="hover:text-foreground" href="https://railway.com/careers" target="_blank" rel="noreferrer noopener">Careers</a></li><li><a class="hover:text-foreground" href="/">Blog</a></li><li><a class="hover:text-foreground" href="https://shop.railway.com" target="_blank" rel="noreferrer noopener">Shop</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Contact</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://discord.gg/railway" target="_blank" rel="noreferrer noopener">Discord</a></li><li><a class="hover:text-foreground" href="https://twitter.com/railway" target="_blank" rel="noreferrer noopener">Twitter</a></li><li><a class="hover:text-foreground" href="https://github.com/railwayapp" target="_blank" rel="noreferrer noopener">GitHub</a></li><li><a class="hover:text-foreground" href="mailto:contact@railway.com">Email</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Legal</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/legal/acceptable-use" target="_blank" rel="noreferrer noopener">Acceptable Use</a></li><li><a class="hover:text-foreground" href="https://railway.com/legal/privacy" target="_blank" rel="noreferrer noopener">Privacy Policy</a></li><li><a class="hover:text-foreground" href="https://railway.com/legal/terms" target="_blank" rel="noreferrer noopener">Terms of Service</a></li></ul></div></div></footer></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"page":{"authors":[{"avatar":{"alt":"Avatar of Angelo Saraceno","height":null,"id":"1831","mimeType":"image/jpeg","url":"https://cms.railway.com/media/6b99ad48a4bd2571569575090858205c1e2697f5308e4a0781c1073c3d87cc31.jpg","width":null},"avatarUrl":"https://cms.railway.com/media/6b99ad48a4bd2571569575090858205c1e2697f5308e4a0781c1073c3d87cc31.jpg","githubUrl":null,"id":"6","name":"Angelo Saraceno","slug":"angelo-saraceno","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":"We recently experienced an outage on our platform that affected 30% of our US-West compute infrastructure and caused workloads to be unreachable. When production outages occur, it is Railway’s policy to share the public details of what occurred. \n\n# Summary\n\nBetween 16:40 UTC and 20:53 UTC, Railway’s engineers received alerts from internal monitoring that users were unable to connect to their workloads. This was due to high resource usage leading to a soft lock of all cores on the affected hosts. This led to rolling restarts of hosts as we attempted recovery on machines hosting customer workloads.\n\nRailway engineers began intervention immediately after notification. Due to the similarity with prior incidents the Railway team has had with our underlying compute platform, we believed it was related to our vendor infrastructure — but potentially triggered by a code change we deployed.\n\n30% of the total machine fleet within US-West was affected due to the outage. Some customer workloads were offline for periods ranging from 10 minutes to 1 hour as we worked to mitigate the issue.\n\nThe Railway infrastructure engineering team were able to attribute the root cause of this incident to a newly deployed metrics collection agent that appeared to trigger a CPU core soft lock on these hosts, despite them being well below resource thresholds. We believe the underlying issue is potentially within the Google Cloud Hypervisor. We’ve documented our findings in [this blogpost](https://blog.railway.com/p/gcp-incidents).\n\nAfter diagnosing and remediating the issue, the Infrastructure team was able to restore all workloads with no data loss reported.\n\n# Incident Details\n\nRailway’s engineering team is undertaking a migration to a new metrics collection service which will deliver significant benefit to Railway’s customer base.\n\nEarlier on Dec 1st, at 16:02 UTC, the Railway engineering began rolling out the newer metrics collection service on Railway infrastructure. This rollout had been previously validated on a subset of Railway’s infrastructure and had not triggered any warnings at that point in time.\n\nHowever, during the rollout, our deployment scripts began reporting that certain hosts were unavailable and the update could not be applied. After we noticed the first couple boxes coming offline, we went back to check the deployment logs. Since our first priority was to bring workloads back, by the time we identified the issue in the logs, the incident was well under way.\n\nAs the metrics collector started up, a configuration error meant it began analyzing processes beyond its original mandate — essentially sampling all processes on a host at a high frequency. This added monitoring resulted in large transfers of memory between the kernelspace and userspace on our hosts, and even though overall resource pressure on the hosts were within nominal thresholds, triggered cascading CPU core soft locks which brought the host down. These reports are inline with a bug Google acknowledged [here](https://github.com/kubernetes/kubernetes/issues/37853#issuecomment-264587949).\n\nThis issue only affected a subset of machines randomly throughout the fleet. The metrics collector was operating perfectly on 70% of the fleet. This meant we initially didn’t suspect it as a cause. Furthermore, the failure scenario wasn’t immediate — the impacted machines failed unpredictably in bursts with no discernible pattern in the metrics, workload patterns, or log data that we had been able to pull from the unaffected machines.\n\nThe misconfigured agent caused CPU core locks across all cores on the affected machines, which made workloads (including our own daemons through which we manage the machine) un-schedulable on the CPU.\n\nAt 17:05 UTC, Railway’s engineering team began manual restarts to recover affected hosts. Because we needed to do these restarts manually — especially after confirming that the host CPU was soft-locked by way of checking serial console logs — the process took between 10-30 minutes per host. However, though we recovered most hosts before 18:30 UTC, several others failed during this time and thus took longer to recover. Railway’s engineering team began engaging additional members of the Customer Success and Support teams to communicate the impact to affected customers across the Community Forum, Email, Twitter, and Discord.\n\n## Response and Resolution\n\nAround 18:30 UTC, customers started reporting that active resolution measures were successful. The Railway engineering team then catalogued all failed machines and failed over additional hosts that we suspected to be impacted. While this action was ongoing, additional monitoring was conducted to ensure that the machines with customer workloads were all coming back online appropriately. \n\nBy 20:23 UTC all hosts were back online and we had total confirmation from customers that workloads were restored. \n\nWe declared the incident to be over at 20:53 UTC.\n\n# **Preventative Measures**\n\nThis incident resulted in the following takeaways:\n\n1. **Improve internal deployment tooling:** We need to shorten feedback time on deployments, helping us identify issues faster.\n2. **Build more resilience:** We need to build better mechanisms to detect and automatically recover from OS/hardware/hypervisor level failures by isolating the blast radius of changes.\n3. **Removing the black box**: We’re already building our cloud independent distributed storage system and making forays into bare metal. We can only deliver the level of service we intend to by controlling, and having visibility into, the entire stack — from silicon to hypervisor.\n\n## Incoming Mitigations\n\n1. **Better System Checks:** We're improving how we check the health and status of our systems to catch issues faster.\n2. **Mandate Staged Rollouts for all Services:** For riskier changes, we already stage rollouts. If this incident has taught us anything, it’s that there’s risk in even the most mundane of changes at our scale. Moving forward, for all services, we’ll start with 10% of our systems, then move to 25%, 50%, and finally 100%, carefully monitoring at each stage\n\n# Moving Forward\n\nThe Railway team is looking to continually improve our processes with our customers and our community partners. Although we are glad that our internal systems caught it before our customers did, we truly regret the impact. We are in active contact with our customers who faced business impact from this outage.","createdAt":"2026-06-09T12:15:20.644Z","description":"We recently experienced an outage on our platform that affected 30% of our US-West compute infrastructure and caused workloads to be unreachable. When production outages occur, it is Railway’s policy to share the public details of what occurred.","externalAuthor":false,"featured":false,"featuredImage":{"alt":"Incident Report: Dec 1st, 2023 featured image","height":null,"id":"386","mimeType":"image/png","url":"https://cms.railway.com/media/cad308b0e24c03c57293e9101d3c32e6dbc4005c597b259e5643bffb3113c9fe.png","width":null},"id":"74","publishedAt":"2023-12-01T20:00:00.000Z","seoDescription":"We recently experienced an outage that affected 30% of our US-West compute infrastructure and caused workloads to be unreachable.","seoTitle":"Incident Report: Dec 1st, 2023","slug":"2023-12-01-incident-report","socialImage":{"alt":"Incident Report: Dec 1st, 2023 social image","height":null,"id":"381","mimeType":"image/png","url":"https://cms.railway.com/media/37247bde529d43cd1c3d4274f8fe1d51af25ad9ae5f752f6f9b83f5e65218ee4.png","width":null},"title":"Incident Report: Dec 1st, 2023","updatedAt":"2023-12-01T20:00:00.000Z"},"relatedPosts":[{"authors":[{"avatar":{"alt":"Mahmoud Abdelwahab headshot","height":460,"id":"1846","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","width":460},"avatarUrl":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","githubUrl":null,"id":"4","name":"Mahmoud Abdelwahab","slug":"mahmoud-abdelwahab","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":null,"createdAt":"2026-08-06T08:07:06.228Z","description":"Announcing: dev.new, an agent that can build, preview, test, and deploy an app from the same place. We tested the whole loop by asking it to ship a real-time multiplayer FPS game that runs in the browser","externalAuthor":false,"featured":true,"featuredImage":{"alt":"dev.new featured image","height":1440,"id":"1983","mimeType":"image/png","url":"https://cms.railway.com/media/dev.new-featured-image-4.png","width":3264},"id":"289","publishedAt":"2026-08-13T08:07:00.000Z","seoDescription":null,"seoTitle":null,"slug":"dev-new","socialImage":{"alt":"dev.new featured image 1","height":720,"id":"1976","mimeType":"image/png","url":"https://cms.railway.com/media/dev.new-featured-image-3.png","width":1632},"title":"Your Cloud should come with an app builder ","updatedAt":"2026-08-14T20:19:27.322Z"},{"authors":[{"avatar":{"alt":"Avatar of Victor Ramirez","height":null,"id":"1855","mimeType":"image/jpeg","url":"https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg","width":null},"avatarUrl":"https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg","githubUrl":null,"id":"34","name":"Victor Ramirez","slug":"victor-ramirez","title":null},{"avatar":{"alt":"Mahmoud Abdelwahab headshot","height":460,"id":"1846","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","width":460},"avatarUrl":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","githubUrl":null,"id":"4","name":"Mahmoud Abdelwahab","slug":"mahmoud-abdelwahab","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":null,"createdAt":"2026-07-15T22:26:48.036Z","description":"Merge now, release when you're ready: turn a feature on for your team, then 10% of users, then everyone. Feature flags are now built into every Railway project, and your agents can run the rollout from the CLI, SDK, and MCP. No third-party service required.","externalAuthor":false,"featured":false,"featuredImage":{"alt":"feature flags featured image","height":1440,"id":"1968","mimeType":"image/png","url":"https://cms.railway.com/media/feature-flags-featured-image.png","width":3264},"id":"278","publishedAt":"2026-07-17T07:00:00.000Z","seoDescription":null,"seoTitle":null,"slug":"feature-flags","socialImage":null,"title":"Roll it out, roll it back, never redeploy","updatedAt":"2026-08-13T16:53:55.126Z"}]},"__N_SSG":true},"page":"/p/[slug]","query":{"slug":"2023-12-01-incident-report"},"buildId":"nFX7R3M0MdLMAf-Ufh9qa","isFallback":false,"isExperimentalCompile":false,"gsp":true,"scriptLoader":[]}</script></body></html> |