251 lines
44 KiB
HTML
251 lines
44 KiB
HTML
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8" data-next-head=""/><meta name="viewport" content="width=device-width" data-next-head=""/><meta name="twitter:card" content="summary_large_image" data-next-head=""/><meta name="twitter:site" content="@Railway" data-next-head=""/><meta name="twitter:creator" content="@Railway" data-next-head=""/><meta property="og:image:type" content="image/png" data-next-head=""/><meta property="og:image:width" content="1200" data-next-head=""/><meta property="og:image:height" content="630" data-next-head=""/><meta property="og:site_name" content="Railway Blog" data-next-head=""/><title data-next-head="">Incident Report: December 16th, 2024</title><meta name="robots" content="index,follow" data-next-head=""/><meta name="description" content="A roughly 10-minute outage on our Google Cloud ingress networking halted inbound requests across all regions; private networking was unaffected." data-next-head=""/><meta property="og:title" content="Incident Report: December 16th, 2024" data-next-head=""/><meta property="og:description" content="A roughly 10-minute outage on our Google Cloud ingress networking halted inbound requests across all regions; private networking was unaffected." data-next-head=""/><meta property="og:url" content="https://blog.railway.com/p/incident-december-16-2024" data-next-head=""/><meta property="og:type" content="article" data-next-head=""/><meta property="article:published_time" content="2024-12-16T00:00:00.000Z" data-next-head=""/><meta property="article:modified_time" content="2024-12-16T00:00:00.000Z" data-next-head=""/><meta property="article:author" content="Jacob Cooper" data-next-head=""/><meta property="article:section" content="Engineering" data-next-head=""/><meta property="og:image" content="https://cms.railway.com/media/2cdcb187f45e610bb6e30467bc080f921b7d75a1871c92651be119fc63996764.png?w=1200&q=90" data-next-head=""/><link rel="canonical" href="https://blog.railway.com/p/incident-december-16-2024" data-next-head=""/><meta name="twitter:title" content="Incident Report: December 16th, 2024" data-next-head=""/><meta name="twitter:description" content="A roughly 10-minute outage on our Google Cloud ingress networking halted inbound requests across all regions; private networking was unaffected." data-next-head=""/><meta name="twitter:image" content="https://cms.railway.com/media/2cdcb187f45e610bb6e30467bc080f921b7d75a1871c92651be119fc63996764.png?w=1200&q=90" data-next-head=""/><link rel="alternate" type="text/markdown" href="/p/incident-december-16-2024.md" data-next-head=""/><link rel="alternate" type="application/rss+xml" title="Railway Blog" href="https://blog.railway.com/rss.xml"/><link rel="preload" href="/_next/static/media/e4af272ccee01ff0-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/91c3caac73720166-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/3359c9acc30ef440-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/aff755c8b4bb3089-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/3988c35203e0830a-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/427e4a37d3642943-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/css/619a01ff31a99955.css" as="style"/><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"BlogPosting","headline":"Incident Report: December 16th, 2024","description":"A roughly 10-minute outage on our Google Cloud ingress networking halted inbound requests across all regions; private networking was unaffected.","image":["https://cms.railway.com/media/2cdcb187f45e610bb6e30467bc080f921b7d75a1871c92651be119fc63996764.png?w=1200&q=90"],"datePublished":"2024-12-16T00:00:00.000Z","dateModified":"2024-12-16T00:00:00.000Z","author":{"@type":"Person","name":"Jacob Cooper"},"publisher":{"@type":"Organization","name":"Railway","logo":{"@type":"ImageObject","url":"https://railway.com/brand/logo-dark.png"}},"mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.railway.com/p/incident-december-16-2024"}}</script><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https://blog.railway.com"},{"@type":"ListItem","position":2,"name":"Engineering","item":"https://blog.railway.com/engineering"},{"@type":"ListItem","position":3,"name":"Incident Report: December 16th, 2024","item":"https://blog.railway.com/p/incident-december-16-2024"}]}</script><style>:root {
|
||
--foreground: hsl(0, 0%, 100%);
|
||
--background: #181622;
|
||
--secondaryBg: hsl(250, 21%, 11%);
|
||
--blue-50: hsl(220, 55%, 10%);
|
||
--blue-100: hsl(220, 55%, 13%);
|
||
--blue-200: hsl(220, 62%, 25%);
|
||
--blue-300: hsl(220, 68%, 35%);
|
||
--blue-400: hsl(220, 72%, 45%);
|
||
--blue-500: hsl(220, 80%, 55%);
|
||
--blue-600: hsl(220, 80%, 65%);
|
||
--blue-700: hsl(220, 80%, 75%);
|
||
--blue-800: hsl(220, 80%, 85%);
|
||
--blue-900: hsl(220, 80%, 95%);
|
||
--blue-950: hsl(220, 55%, 97%);
|
||
--gray-50: hsl(248, 21%, 13%);
|
||
--gray-100: hsl(246, 18%, 15%);
|
||
--gray-200: hsl(246, 11%, 22%);
|
||
--gray-300: hsl(246, 8%, 35%);
|
||
--gray-400: hsl(246, 7%, 45%);
|
||
--gray-500: hsl(246, 6%, 55%);
|
||
--gray-600: hsl(246, 6%, 65%);
|
||
--gray-700: hsl(246, 6%, 78%);
|
||
--gray-800: hsl(246, 6%, 87%);
|
||
--gray-900: hsl(246, 6%, 95%);
|
||
--gray-950: hsl(246, 6%, 95%);
|
||
--red-50: hsl(1, 35%, 10%);
|
||
--red-100: hsl(1, 45%, 12%);
|
||
--red-200: hsl(1, 55%, 20%);
|
||
--red-300: hsl(1, 62%, 28%);
|
||
--red-400: hsl(1, 62%, 35%);
|
||
--red-500: hsl(1, 62%, 44%);
|
||
--red-600: hsl(1, 62%, 60%);
|
||
--red-700: hsl(1, 62%, 76%);
|
||
--red-800: hsl(1, 64%, 85%);
|
||
--red-900: hsl(1, 68%, 95%);
|
||
--red-950: hsl(1, 55%, 98%);
|
||
--pink-50: hsl(270, 40%, 16%);
|
||
--pink-100: hsl(270, 40%, 16%);
|
||
--pink-200: hsl(270, 45%, 24%);
|
||
--pink-300: hsl(270, 50%, 32%);
|
||
--pink-400: hsl(270, 55%, 43%);
|
||
--pink-500: hsl(270, 60%, 52%);
|
||
--pink-600: hsl(270, 70%, 65%);
|
||
--pink-700: hsl(270, 70%, 75%);
|
||
--pink-800: hsl(270, 70%, 85%);
|
||
--pink-900: hsl(270, 70%, 95%);
|
||
--pink-950: hsl(270, 70%, 95%);
|
||
--green-50: hsl(152, 15%, 10%);
|
||
--green-100: hsl(152, 26%, 11%);
|
||
--green-200: hsl(152, 32%, 16%);
|
||
--green-300: hsl(152, 38%, 24%);
|
||
--green-400: hsl(152, 38%, 34%);
|
||
--green-500: hsl(152, 38%, 42%);
|
||
--green-600: hsl(152, 38%, 60%);
|
||
--green-700: hsl(152, 38%, 70%);
|
||
--green-800: hsl(152, 38%, 80%);
|
||
--green-900: hsl(152, 38%, 91%);
|
||
--green-950: hsl(152, 40%, 97%);
|
||
--yellow-50: hsl(44, 95%, 12%);
|
||
--yellow-100: hsl(44, 95%, 12%);
|
||
--yellow-200: hsl(44, 96%, 24%);
|
||
--yellow-300: hsl(44, 95%, 36%);
|
||
--yellow-400: hsl(44, 95%, 48%);
|
||
--yellow-500: hsl(44, 95%, 60%);
|
||
--yellow-600: hsl(44, 95%, 69%);
|
||
--yellow-700: hsl(44, 95%, 78%);
|
||
--yellow-800: hsl(44, 95%, 86%);
|
||
--yellow-900: hsl(44, 95%, 95%);
|
||
--yellow-950: hsl(44, 95%, 95%);
|
||
}
|
||
|
||
.light {
|
||
--foreground: hsl(250, 24%, 9%);
|
||
--background: #F1F0EF;
|
||
--secondaryBg: hsl(0, 0%, 98%);
|
||
--blue-50: hsl(220, 55%, 97%);
|
||
--blue-100: hsl(220, 80%, 95%);
|
||
--blue-200: hsl(220, 80%, 85%);
|
||
--blue-300: hsl(220, 80%, 75%);
|
||
--blue-400: hsl(220, 80%, 65%);
|
||
--blue-500: hsl(220, 80%, 55%);
|
||
--blue-600: hsl(220, 72%, 45%);
|
||
--blue-700: hsl(220, 68%, 35%);
|
||
--blue-800: hsl(220, 62%, 25%);
|
||
--blue-900: hsl(220, 55%, 13%);
|
||
--blue-950: hsl(220, 55%, 10%);
|
||
--gray-50: hsl(246, 6%, 95%);
|
||
--gray-100: hsl(246, 6%, 95%);
|
||
--gray-200: hsl(246, 6%, 87%);
|
||
--gray-300: hsl(246, 6%, 78%);
|
||
--gray-400: hsl(246, 6%, 65%);
|
||
--gray-500: hsl(246, 6%, 55%);
|
||
--gray-600: hsl(246, 7%, 45%);
|
||
--gray-700: hsl(246, 8%, 35%);
|
||
--gray-800: hsl(246, 11%, 22%);
|
||
--gray-900: hsl(246, 18%, 15%);
|
||
--gray-950: hsl(248, 21%, 13%);
|
||
--red-50: hsl(1, 55%, 98%);
|
||
--red-100: hsl(1, 68%, 95%);
|
||
--red-200: hsl(1, 64%, 85%);
|
||
--red-300: hsl(1, 62%, 76%);
|
||
--red-400: hsl(1, 62%, 60%);
|
||
--red-500: hsl(1, 62%, 44%);
|
||
--red-600: hsl(1, 62%, 35%);
|
||
--red-700: hsl(1, 62%, 28%);
|
||
--red-800: hsl(1, 55%, 20%);
|
||
--red-900: hsl(1, 45%, 12%);
|
||
--red-950: hsl(1, 35%, 10%);
|
||
--pink-50: hsl(270, 70%, 95%);
|
||
--pink-100: hsl(270, 70%, 95%);
|
||
--pink-200: hsl(270, 70%, 85%);
|
||
--pink-300: hsl(270, 70%, 75%);
|
||
--pink-400: hsl(270, 70%, 65%);
|
||
--pink-500: hsl(270, 60%, 52%);
|
||
--pink-600: hsl(270, 55%, 43%);
|
||
--pink-700: hsl(270, 50%, 32%);
|
||
--pink-800: hsl(270, 45%, 24%);
|
||
--pink-900: hsl(270, 40%, 16%);
|
||
--pink-950: hsl(270, 40%, 16%);
|
||
--green-50: hsl(152, 40%, 97%);
|
||
--green-100: hsl(152, 38%, 91%);
|
||
--green-200: hsl(152, 38%, 80%);
|
||
--green-300: hsl(152, 38%, 70%);
|
||
--green-400: hsl(152, 38%, 60%);
|
||
--green-500: hsl(152, 38%, 42%);
|
||
--green-600: hsl(152, 38%, 34%);
|
||
--green-700: hsl(152, 38%, 24%);
|
||
--green-800: hsl(152, 32%, 16%);
|
||
--green-900: hsl(152, 26%, 11%);
|
||
--green-950: hsl(152, 15%, 10%);
|
||
--yellow-50: hsl(44, 95%, 95%);
|
||
--yellow-100: hsl(44, 95%, 95%);
|
||
--yellow-200: hsl(44, 95%, 86%);
|
||
--yellow-300: hsl(44, 95%, 78%);
|
||
--yellow-400: hsl(44, 95%, 69%);
|
||
--yellow-500: hsl(44, 95%, 60%);
|
||
--yellow-600: hsl(44, 95%, 48%);
|
||
--yellow-700: hsl(44, 95%, 36%);
|
||
--yellow-800: hsl(44, 96%, 24%);
|
||
--yellow-900: hsl(44, 95%, 12%);
|
||
--yellow-950: hsl(44, 95%, 12%);
|
||
}
|
||
|
||
.dark {
|
||
--foreground: hsl(0, 0%, 100%);
|
||
--background: #181622;
|
||
--secondaryBg: hsl(250, 21%, 11%);
|
||
--blue-50: hsl(220, 55%, 10%);
|
||
--blue-100: hsl(220, 55%, 13%);
|
||
--blue-200: hsl(220, 62%, 25%);
|
||
--blue-300: hsl(220, 68%, 35%);
|
||
--blue-400: hsl(220, 72%, 45%);
|
||
--blue-500: hsl(220, 80%, 55%);
|
||
--blue-600: hsl(220, 80%, 65%);
|
||
--blue-700: hsl(220, 80%, 75%);
|
||
--blue-800: hsl(220, 80%, 85%);
|
||
--blue-900: hsl(220, 80%, 95%);
|
||
--blue-950: hsl(220, 55%, 97%);
|
||
--gray-50: hsl(248, 21%, 13%);
|
||
--gray-100: hsl(246, 18%, 15%);
|
||
--gray-200: hsl(246, 11%, 22%);
|
||
--gray-300: hsl(246, 8%, 35%);
|
||
--gray-400: hsl(246, 7%, 45%);
|
||
--gray-500: hsl(246, 6%, 55%);
|
||
--gray-600: hsl(246, 6%, 65%);
|
||
--gray-700: hsl(246, 6%, 78%);
|
||
--gray-800: hsl(246, 6%, 87%);
|
||
--gray-900: hsl(246, 6%, 95%);
|
||
--gray-950: hsl(246, 6%, 95%);
|
||
--red-50: hsl(1, 35%, 10%);
|
||
--red-100: hsl(1, 45%, 12%);
|
||
--red-200: hsl(1, 55%, 20%);
|
||
--red-300: hsl(1, 62%, 28%);
|
||
--red-400: hsl(1, 62%, 35%);
|
||
--red-500: hsl(1, 62%, 44%);
|
||
--red-600: hsl(1, 62%, 60%);
|
||
--red-700: hsl(1, 62%, 76%);
|
||
--red-800: hsl(1, 64%, 85%);
|
||
--red-900: hsl(1, 68%, 95%);
|
||
--red-950: hsl(1, 55%, 98%);
|
||
--pink-50: hsl(270, 40%, 16%);
|
||
--pink-100: hsl(270, 40%, 16%);
|
||
--pink-200: hsl(270, 45%, 24%);
|
||
--pink-300: hsl(270, 50%, 32%);
|
||
--pink-400: hsl(270, 55%, 43%);
|
||
--pink-500: hsl(270, 60%, 52%);
|
||
--pink-600: hsl(270, 70%, 65%);
|
||
--pink-700: hsl(270, 70%, 75%);
|
||
--pink-800: hsl(270, 70%, 85%);
|
||
--pink-900: hsl(270, 70%, 95%);
|
||
--pink-950: hsl(270, 70%, 95%);
|
||
--green-50: hsl(152, 15%, 10%);
|
||
--green-100: hsl(152, 26%, 11%);
|
||
--green-200: hsl(152, 32%, 16%);
|
||
--green-300: hsl(152, 38%, 24%);
|
||
--green-400: hsl(152, 38%, 34%);
|
||
--green-500: hsl(152, 38%, 42%);
|
||
--green-600: hsl(152, 38%, 60%);
|
||
--green-700: hsl(152, 38%, 70%);
|
||
--green-800: hsl(152, 38%, 80%);
|
||
--green-900: hsl(152, 38%, 91%);
|
||
--green-950: hsl(152, 40%, 97%);
|
||
--yellow-50: hsl(44, 95%, 12%);
|
||
--yellow-100: hsl(44, 95%, 12%);
|
||
--yellow-200: hsl(44, 96%, 24%);
|
||
--yellow-300: hsl(44, 95%, 36%);
|
||
--yellow-400: hsl(44, 95%, 48%);
|
||
--yellow-500: hsl(44, 95%, 60%);
|
||
--yellow-600: hsl(44, 95%, 69%);
|
||
--yellow-700: hsl(44, 95%, 78%);
|
||
--yellow-800: hsl(44, 95%, 86%);
|
||
--yellow-900: hsl(44, 95%, 95%);
|
||
--yellow-950: hsl(44, 95%, 95%);
|
||
}</style><link rel="stylesheet" href="/_next/static/css/619a01ff31a99955.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" noModule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-0b474a604a067f92.js" defer=""></script><script src="/_next/static/chunks/framework-90fc9b0f7c4706c1.js" defer=""></script><script src="/_next/static/chunks/main-e89abd6570bbb9dc.js" defer=""></script><script src="/_next/static/chunks/pages/_app-3049b372678b389f.js" defer=""></script><script src="/_next/static/chunks/5399-d49efedd870f41de.js" defer=""></script><script src="/_next/static/chunks/3801-cdd2681d27083ad8.js" defer=""></script><script src="/_next/static/chunks/9594-1c44b12020a3af0a.js" defer=""></script><script src="/_next/static/chunks/4696-fd1712bc36a5bc50.js" defer=""></script><script src="/_next/static/chunks/pages/p/%5Bslug%5D-ddadb1ca54eaab0f.js" defer=""></script><script src="/_next/static/nFX7R3M0MdLMAf-Ufh9qa/_buildManifest.js" defer=""></script><script src="/_next/static/nFX7R3M0MdLMAf-Ufh9qa/_ssgManifest.js" defer=""></script></head><body><div id="__next"><script>!function(){try{var d=document.documentElement,c=d.classList;c.remove('light','dark');var e=localStorage.getItem('theme');if('system'===e||(!e&&true)){var t='(prefers-color-scheme: dark)',m=window.matchMedia(t);if(m.media!==t||m.matches){d.style.colorScheme = 'dark';c.add('dark')}else{d.style.colorScheme = 'light';c.add('light')}}else if(e){c.add(e|| '')}if(e==='light'||e==='dark')d.style.colorScheme=e}catch(e){}}()</script><div class="__variable_f367f3 __variable_0b668e __variable_ecea63 font-sans"><div class="px-5 md:px-8"><nav class="max-w-6xl mx-auto py-6 flex justify-between items-center border-b border-gray-100"><a class="flex items-center space-x-4" href="/"><svg data-v-423bf9ae="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 59.99945071644884 60" class="logo w-8"><g data-v-423bf9ae="" id="6f639fe5-02e1-4640-a1fb-f4b22b64e5ef" transform="matrix(0.5999641134746578,0,0,0.5999641134746578,-437.5538206497148,-266.68705118956507)" stroke="none"><path d="M729.872 487.327a50.86 50.86 0 0 0-.464 5.033h75.879c-.265-.518-.621-.985-.98-1.442-12.972-16.769-19.95-15.315-29.932-15.741-3.328-.137-5.585-.192-18.832-.192-7.09 0-14.798.018-22.304.038.737-1.746 1.6-3.419 2.525-5.061-2.885 5.106-4.865 10.771-5.805 16.789l.891-4.397c.007-.031.018-.063.024-.094h38.883v5.067h-39.885zM805.885 497.432h-76.438c.08 1.352.206 2.686.388 4.002h70.571c3.146 0 4.907-1.786 5.479-4.002zM733.851 515.257a52.226 52.226 0 0 1-1.98-4.997c6.608 19.89 25.328 34.251 47.433 34.251 20.205 0 37.566-12.007 45.452-29.254h-90.905zM729.38 492.915c-.018.531-.08 1.055-.08 1.589 0 .538.063 1.059.08 1.59v-3.179zM824.77 515.229z"></path><path d="M779.303 444.505c-18.682 0-34.939 10.265-43.524 25.439 6.709-.014 19.775-.022 19.775-.022h.003v-.005c15.444 0 16.018.069 19.035.195l1.868.069c6.507.217 14.505.916 20.798 5.68 3.416 2.584 8.348 8.287 11.288 12.35 2.718 3.758 3.5 8.078 1.652 12.217-1.701 3.804-5.361 6.073-9.793 6.073H730.85l-.884-4.201c.426 2.707 1.037 5.344 1.879 7.886h94.914a49.863 49.863 0 0 0 2.546-15.682c.001-27.611-22.386-49.999-50.002-49.999z"></path></g></svg><span class="text-xl font-semibold">Blog</span></a><div class="text-gray-600 flex items-center space-x-6"><a class="text-sm hover:text-pink-600" href="https://railway.com" target="_blank" rel="noreferrer noopener">Go to Homepage</a><a class="hover:text-pink-600" href="/rss.xml" title="RSS Feed"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4 11a9 9 0 0 1 9 9"></path><path d="M4 4a16 16 0 0 1 16 16"></path><circle cx="5" cy="19" r="1"></circle></svg></a></div></nav></div><div class="min-h-screen overflow-x-hidden relative"><div class="mt-10 mb-5 px-5 md:px-8 mx-auto"><article class="max-w-6xl mx-auto mt-24 mb-12 border-b border-gray-100 pb-32"><div class="max-w-[704px] mx-auto flex items-center text-sm text-gray-600 space-x-3"><div class="flex items-center space-x-3"><div class="flex items-center"><img src="https://cms.railway.com/media/b30ff69ce3b376e5ad838a6f790e4fa609559334c48e5661bf4ef80c7d25b107.png" alt="Avatar of Jacob Cooper" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/></div><span>Jacob Cooper</span></div><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><time dateTime="2024-12-16T00:00:00.000Z">Dec 16, 2024</time></div><header class="mt-5 mb-16 max-w-[704px] mx-auto"><h1 class="text-post-title font-medium font-serif">Incident Report: December 16th, 2024</h1></header><section class="max-w-[704px] mx-auto text-base sm:text-lg"><nav style="position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border-width:0" aria-label="Table of Contents"><h2>Table of Contents</h2><ol><li style="margin-left:0rem"><a href="#incident-response-timeline">Incident Response Timeline</a></li><li style="margin-left:0rem"><a href="#what-happened">What Happened</a></li><li style="margin-left:0rem"><a href="#short-term-resolution">Short Term Resolution</a></li><li style="margin-left:0rem"><a href="#long-term-mitigation">Long Term Mitigation</a></li></ol></nav><div><p class="mb-4 text-gray-800">We recently experienced an outage which affected inbound traffic, on Google Cloud, on all regions of our network.</p>
|
||
<p class="mb-4 text-gray-800">During this outage, inbound requests on Google Cloud Edge servers were unable to route requests to their applications. This outage lasted approximately 10 minutes.</p>
|
||
<p class="mb-4 text-gray-800">Private networking, outbound requests, Metal Hosts, and Dedicated Instances were unaffected.</p>
|
||
<p class="mb-4 text-gray-800">When production outages occur, it is Railway’s policy to share the public details of what occurred.</p>
|
||
<a class="relative no-underline" href="/p/incident-december-16-2024#incident-response-timeline"><span id="incident-response-timeline" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-16 mb-8">Incident Response Timeline</h2></a>
|
||
<p class="mb-4 text-gray-800">NOTE: All dates are December 17th (UTC)</p>
|
||
<p class="mb-4 text-gray-800">[12:47am] - A change is merged modifying IP allow-list rules</p>
|
||
<p class="mb-4 text-gray-800">[12:49am] - Railway engineer is notified via automated alerting</p>
|
||
<p class="mb-4 text-gray-800">[12:54am] - Change reverted</p>
|
||
<p class="mb-4 text-gray-800">[12:58am] - Change applied</p>
|
||
<p class="mb-4 text-gray-800">[12:59am] - Our automated monitoring recovered</p>
|
||
<a class="relative no-underline" href="/p/incident-december-16-2024#what-happened"><span id="what-happened" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-16 mb-8">What Happened</h2></a>
|
||
<p class="mb-4 text-gray-800">Over the last 2 years, Railway has maintained both a static and dynamic block list.</p>
|
||
<p class="mb-4 text-gray-800">Over this period, as we’ve gotten better at dealing with DDoS, we’ve transitioned entirely to the dynamic solution.</p>
|
||
<p class="mb-4 text-gray-800">A customer reached out to Railway that their application was having intermittent timeouts communicating with Railway.</p>
|
||
<p class="mb-4 text-gray-800">It turned out their IP was being statically blocked on a list from our “pre-dynamic” era</p>
|
||
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/a5ed71a003dcad6029cdf07793cf98e5e7bfb8e9da5ec97840112a86595323b5.png?w=768&format=webp&q=90" srcSet="https://cms.railway.com/media/a5ed71a003dcad6029cdf07793cf98e5e7bfb8e9da5ec97840112a86595323b5.png?w=320&format=webp&q=90 320w, https://cms.railway.com/media/a5ed71a003dcad6029cdf07793cf98e5e7bfb8e9da5ec97840112a86595323b5.png?w=480&format=webp&q=90 480w, https://cms.railway.com/media/a5ed71a003dcad6029cdf07793cf98e5e7bfb8e9da5ec97840112a86595323b5.png?w=640&format=webp&q=90 640w, https://cms.railway.com/media/a5ed71a003dcad6029cdf07793cf98e5e7bfb8e9da5ec97840112a86595323b5.png?w=768&format=webp&q=90 768w, https://cms.railway.com/media/a5ed71a003dcad6029cdf07793cf98e5e7bfb8e9da5ec97840112a86595323b5.png?w=960&format=webp&q=90 960w, https://cms.railway.com/media/a5ed71a003dcad6029cdf07793cf98e5e7bfb8e9da5ec97840112a86595323b5.png?w=1200&format=webp&q=90 1200w, https://cms.railway.com/media/a5ed71a003dcad6029cdf07793cf98e5e7bfb8e9da5ec97840112a86595323b5.png?w=1440&format=webp&q=90 1440w, https://cms.railway.com/media/a5ed71a003dcad6029cdf07793cf98e5e7bfb8e9da5ec97840112a86595323b5.png?w=1600&format=webp&q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="CleanShot 2024-12-16 at 17.43.33@2x.png" class="w-full rounded-[8px]" loading="lazy" decoding="async"/></figure>
|
||
<p class="mb-4 text-gray-800">We began the process of removing this IP list from the blocked list of IPs, by emptying the array</p>
|
||
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/dfd71bd384311ac35eb573cd3a7b113a5f23811685b5c66adfd066b1190a2239.png?w=768&format=webp&q=90" srcSet="https://cms.railway.com/media/dfd71bd384311ac35eb573cd3a7b113a5f23811685b5c66adfd066b1190a2239.png?w=320&format=webp&q=90 320w, https://cms.railway.com/media/dfd71bd384311ac35eb573cd3a7b113a5f23811685b5c66adfd066b1190a2239.png?w=480&format=webp&q=90 480w, https://cms.railway.com/media/dfd71bd384311ac35eb573cd3a7b113a5f23811685b5c66adfd066b1190a2239.png?w=640&format=webp&q=90 640w, https://cms.railway.com/media/dfd71bd384311ac35eb573cd3a7b113a5f23811685b5c66adfd066b1190a2239.png?w=768&format=webp&q=90 768w, https://cms.railway.com/media/dfd71bd384311ac35eb573cd3a7b113a5f23811685b5c66adfd066b1190a2239.png?w=960&format=webp&q=90 960w, https://cms.railway.com/media/dfd71bd384311ac35eb573cd3a7b113a5f23811685b5c66adfd066b1190a2239.png?w=1200&format=webp&q=90 1200w, https://cms.railway.com/media/dfd71bd384311ac35eb573cd3a7b113a5f23811685b5c66adfd066b1190a2239.png?w=1440&format=webp&q=90 1440w, https://cms.railway.com/media/dfd71bd384311ac35eb573cd3a7b113a5f23811685b5c66adfd066b1190a2239.png?w=1600&format=webp&q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="CleanShot 2024-12-16 at 17.45.20@2x.png" class="w-full rounded-[8px]" loading="lazy" decoding="async"/></figure>
|
||
<p class="mb-4 text-gray-800">However, the Google Cloud Terraform Provider consumes the source_ranges as an optional list. When nothing is provided, it defaults to applying the action to everything.</p>
|
||
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/b6cfc07bdea091224d57fccc025991205cb1973323f616a2712065e8a24f530f.png?w=768&format=webp&q=90" srcSet="https://cms.railway.com/media/b6cfc07bdea091224d57fccc025991205cb1973323f616a2712065e8a24f530f.png?w=320&format=webp&q=90 320w, https://cms.railway.com/media/b6cfc07bdea091224d57fccc025991205cb1973323f616a2712065e8a24f530f.png?w=480&format=webp&q=90 480w, https://cms.railway.com/media/b6cfc07bdea091224d57fccc025991205cb1973323f616a2712065e8a24f530f.png?w=640&format=webp&q=90 640w, https://cms.railway.com/media/b6cfc07bdea091224d57fccc025991205cb1973323f616a2712065e8a24f530f.png?w=768&format=webp&q=90 768w, https://cms.railway.com/media/b6cfc07bdea091224d57fccc025991205cb1973323f616a2712065e8a24f530f.png?w=960&format=webp&q=90 960w, https://cms.railway.com/media/b6cfc07bdea091224d57fccc025991205cb1973323f616a2712065e8a24f530f.png?w=1200&format=webp&q=90 1200w, https://cms.railway.com/media/b6cfc07bdea091224d57fccc025991205cb1973323f616a2712065e8a24f530f.png?w=1440&format=webp&q=90 1440w, https://cms.railway.com/media/b6cfc07bdea091224d57fccc025991205cb1973323f616a2712065e8a24f530f.png?w=1600&format=webp&q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="CleanShot 2024-12-16 at 17.36.05@2x.png" class="w-full rounded-[8px]" loading="lazy" decoding="async"/></figure><div class="flex w-full p-4 my-8 rounded border border-transparent bg-blue-100"><div class="text-yellow-500">💡</div><div class="flex flex-col w-full"><div class="ml-4 text-foreground"><p class="mb-4 text-gray-800">Terraform, being written in Golang, makes no such distinction between “empty array” and nil.</p></div></div></div><p class="mb-4 text-gray-800">When a GCP Firewall rule does not specify a source IP range, by default it selects all IPs. Terraform, when supplied with an empty list, changes the underlying GCP firewall rule to remove the source IP list - essentially changing the meaning of the rule as read in Terraform from “block this empty list of IPs” to “block everything”.</p>
|
||
<a class="relative no-underline" href="/p/incident-december-16-2024#short-term-resolution"><span id="short-term-resolution" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-16 mb-8">Short Term Resolution</h2></a>
|
||
<p class="mb-4 text-gray-800">Railway’s policy is to “Revert first, then figure out the underlying issue”</p>
|
||
<p class="mb-4 text-gray-800">We could have been faster in a couple places</p>
|
||
<ol class="list-decimal pl-6 mb-4 space-y-4 text-gray-800">
|
||
<li>Our page fired to the current oncall, waking them up, instead of the engineer who merged the change</li>
|
||
<li>Our bastion Terraform runner ran a plan before the apply, causing 3 minutes of “delay” to resolve</li>
|
||
</ol>
|
||
<p class="mb-4 text-gray-800">As such, our short term resolution:</p>
|
||
<ol class="list-decimal pl-6 mb-4 space-y-4 text-gray-800">
|
||
<li>Automatically apply “page override” for merging changes (for faster notification)</li>
|
||
<li>Skip the “plan” phase during a revert for Terraform</li>
|
||
<li>Safely removing the offending firewall rule and audit all remaining legacy Google Cloud firewall rules for loose specificity</li>
|
||
</ol>
|
||
<a class="relative no-underline" href="/p/incident-december-16-2024#long-term-mitigation"><span id="long-term-mitigation" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-16 mb-8">Long Term Mitigation</h2></a>
|
||
<p class="mb-4 text-gray-800">Our move to metal is aimed at building new primitives that we fully control, and building predictable behaviour into those primitives. Every layer of this system - from the diverse mix of ISP we contract with, to the BGP based L3 fabric running across our switches to the design of our virtualisation stack is aimed towards resiliency and fault isolation.</p>
|
||
<p class="mb-4 text-gray-800">By maintaining support for GCP, adding AWS (currently trialing with limited customers) and investing in multiple metal datacenters in each region - we are building not just resilient new infrastructure, but adding support for a diverse mix of compute while simultaneously making this incident a thing of the past.</p></div></section></article><div class="max-w-6xl mx-auto"><div><header class="flex items-center justify-between mb-8"><h3 class="text-gray-500 text-h3">Continue Reading...</h3><a class="text-pink-500 hover:underline" href="/engineering">View All <!-- -->Engineering<!-- --> -></a></header><div class="grid grid-cols-1 md:grid-cols-2 gap-8"><a class="flex flex-col bg-secondaryBg p-6 rounded-lg hover:bg-gray-100 group" href="/p/dev-new"><div class="flex gap-2"><div class="post-category-pill font-medium max-w-max text-xs px-2 py-[3px] rounded-[4px] tracking-[0.06em] uppercase" data-category-tone="plum">Engineering</div></div><div class="flex-grow"><header class="font-medium font-serif text-lg mt-2 mb-1">Your Cloud should come with an app builder </header><p class="text-base text-gray-800 line-clamp-2">Announcing: dev.new, an agent that can build, preview, test, and deploy an app from the same place. We tested the whole loop by asking it to ship a real-time multiplayer FPS game that runs in the browser</p></div><div class="flex items-center gap-3 mt-6"><div class="flex items-center"><img src="https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg" alt="Avatar of Mahmoud Abdelwahab" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/></div><span class="text-sm text-gray-600">Mahmoud Abdelwahab</span><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><span class="text-sm text-gray-600">Aug 13, 2026</span></div></a><a class="flex flex-col bg-secondaryBg p-6 rounded-lg hover:bg-gray-100 group" href="/p/feature-flags"><div class="flex gap-2"><div class="post-category-pill font-medium max-w-max text-xs px-2 py-[3px] rounded-[4px] tracking-[0.06em] uppercase" data-category-tone="plum">Engineering</div></div><div class="flex-grow"><header class="font-medium font-serif text-lg mt-2 mb-1">Roll it out, roll it back, never redeploy</header><p class="text-base text-gray-800 line-clamp-2">Merge now, release when you're ready: turn a feature on for your team, then 10% of users, then everyone. Feature flags are now built into every Railway project, and your agents can run the rollout from the CLI, SDK, and MCP. No third-party service required.</p></div><div class="flex items-center gap-3 mt-6"><div class="flex items-center"><img src="https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg" alt="Avatar of Victor Ramirez" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/><img src="https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg" alt="Avatar of Mahmoud Abdelwahab" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:-8px" loading="lazy" decoding="async" width="24" height="24"/></div><span class="text-sm text-gray-600">Victor Ramirez & Mahmoud Abdelwahab</span><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><span class="text-sm text-gray-600">Jul 17, 2026</span></div></a></div></div><section class="post-bottom-cta my-16 border rounded-[16px] px-6 py-16 sm:px-12 sm:py-24 flex flex-col items-center justify-center text-center" aria-labelledby="post-bottom-cta-title"><h3 id="post-bottom-cta-title" class="text-h2 font-serif font-medium">Ready to get started?</h3><p class="mt-4 text-base sm:text-lg text-gray-600">Join millions of developers deploying applications on Railway</p><div class="mt-8 flex w-full flex-col items-stretch justify-center gap-3 sm:w-auto sm:flex-row"><a class="post-bottom-cta-primary inline-flex min-h-[48px] items-center justify-center rounded-[8px] px-6 py-3 text-base font-medium no-underline transition-colors duration-100" href="https://railway.com" target="_blank" rel="noreferrer noopener">Deploy a new project</a><a class="post-bottom-cta-secondary inline-flex min-h-[48px] items-center justify-center rounded-[8px] border px-6 py-3 text-base font-medium no-underline transition-colors duration-100" href="https://railway.com/enterprise" target="_blank" rel="noreferrer noopener">Book a demo</a></div></section></div></div></div><footer class="px-5 md:px-8 py-12 border-t border-gray-100"><div class="grid grid-cols-2 sm:grid-cols-12 gap-8 sm:gap-4 lg:gap-16 max-w-6xl mx-auto"><div class="hidden sm:flex col-span-3 lg:col-span-4 flex-col justify-between"><a class="underline" href="https://railway.com" target="_blank" rel="noreferrer noopener"><svg data-v-423bf9ae="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 59.99945071644884 60" class="logo w-8"><g data-v-423bf9ae="" id="6f639fe5-02e1-4640-a1fb-f4b22b64e5ef" transform="matrix(0.5999641134746578,0,0,0.5999641134746578,-437.5538206497148,-266.68705118956507)" stroke="none"><path d="M729.872 487.327a50.86 50.86 0 0 0-.464 5.033h75.879c-.265-.518-.621-.985-.98-1.442-12.972-16.769-19.95-15.315-29.932-15.741-3.328-.137-5.585-.192-18.832-.192-7.09 0-14.798.018-22.304.038.737-1.746 1.6-3.419 2.525-5.061-2.885 5.106-4.865 10.771-5.805 16.789l.891-4.397c.007-.031.018-.063.024-.094h38.883v5.067h-39.885zM805.885 497.432h-76.438c.08 1.352.206 2.686.388 4.002h70.571c3.146 0 4.907-1.786 5.479-4.002zM733.851 515.257a52.226 52.226 0 0 1-1.98-4.997c6.608 19.89 25.328 34.251 47.433 34.251 20.205 0 37.566-12.007 45.452-29.254h-90.905zM729.38 492.915c-.018.531-.08 1.055-.08 1.589 0 .538.063 1.059.08 1.59v-3.179zM824.77 515.229z"></path><path d="M779.303 444.505c-18.682 0-34.939 10.265-43.524 25.439 6.709-.014 19.775-.022 19.775-.022h.003v-.005c15.444 0 16.018.069 19.035.195l1.868.069c6.507.217 14.505.916 20.798 5.68 3.416 2.584 8.348 8.287 11.288 12.35 2.718 3.758 3.5 8.078 1.652 12.217-1.701 3.804-5.361 6.073-9.793 6.073H730.85l-.884-4.201c.426 2.707 1.037 5.344 1.879 7.886h94.914a49.863 49.863 0 0 0 2.546-15.682c.001-27.611-22.386-49.999-50.002-49.999z"></path></g></svg></a><div class="text-xs font-medium text-gray-600 w-full">Copyright © <!-- -->2026<!-- --> Railway Corp. <br/>All rights reserved.</div></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Product</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/changelog" target="_blank" rel="noreferrer noopener">Changelog</a></li><li><a class="hover:text-foreground" href="https://railway.com/pricing" target="_blank" rel="noreferrer noopener">Pricing</a></li><li><a class="hover:text-foreground" href="https://railway.com/deploy" target="_blank" rel="noreferrer noopener">Templates</a></li><li><a class="hover:text-foreground" href="https://station.railway.com/feedback" target="_blank" rel="noreferrer noopener">Feedback</a></li><li><a class="hover:text-foreground" href="https://railway.com/open-source-kickback" target="_blank" rel="noreferrer noopener">OSS Kickback</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Company</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/about" target="_blank" rel="noreferrer noopener">About</a></li><li><a class="hover:text-foreground" href="https://railway.com/careers" target="_blank" rel="noreferrer noopener">Careers</a></li><li><a class="hover:text-foreground" href="/">Blog</a></li><li><a class="hover:text-foreground" href="https://shop.railway.com" target="_blank" rel="noreferrer noopener">Shop</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Contact</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://discord.gg/railway" target="_blank" rel="noreferrer noopener">Discord</a></li><li><a class="hover:text-foreground" href="https://twitter.com/railway" target="_blank" rel="noreferrer noopener">Twitter</a></li><li><a class="hover:text-foreground" href="https://github.com/railwayapp" target="_blank" rel="noreferrer noopener">GitHub</a></li><li><a class="hover:text-foreground" href="mailto:contact@railway.com">Email</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Legal</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/legal/acceptable-use" target="_blank" rel="noreferrer noopener">Acceptable Use</a></li><li><a class="hover:text-foreground" href="https://railway.com/legal/privacy" target="_blank" rel="noreferrer noopener">Privacy Policy</a></li><li><a class="hover:text-foreground" href="https://railway.com/legal/terms" target="_blank" rel="noreferrer noopener">Terms of Service</a></li></ul></div></div></footer></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"page":{"authors":[{"avatar":{"alt":"Avatar of Jacob Cooper","height":null,"id":"1841","mimeType":"image/png","url":"https://cms.railway.com/media/b30ff69ce3b376e5ad838a6f790e4fa609559334c48e5661bf4ef80c7d25b107.png","width":null},"avatarUrl":"https://cms.railway.com/media/b30ff69ce3b376e5ad838a6f790e4fa609559334c48e5661bf4ef80c7d25b107.png","githubUrl":null,"id":"3","name":"Jacob Cooper","slug":"jacob-cooper","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":"We recently experienced an outage which affected inbound traffic, on Google Cloud, on all regions of our network.\n\nDuring this outage, inbound requests on Google Cloud Edge servers were unable to route requests to their applications. This outage lasted approximately 10 minutes.\n\nPrivate networking, outbound requests, Metal Hosts, and Dedicated Instances were unaffected. \n\nWhen production outages occur, it is Railway’s policy to share the public details of what occurred.\n\n# Incident Response Timeline\n\nNOTE: All dates are December 17th (UTC)\n\n[12:47am] - A change is merged modifying IP allow-list rules\n\n[12:49am] - Railway engineer is notified via automated alerting\n\n[12:54am] - Change reverted\n\n[12:58am] - Change applied\n\n[12:59am] - Our automated monitoring recovered\n\n# What Happened\n\nOver the last 2 years, Railway has maintained both a static and dynamic block list. \n\nOver this period, as we’ve gotten better at dealing with DDoS, we’ve transitioned entirely to the dynamic solution.\n\nA customer reached out to Railway that their application was having intermittent timeouts communicating with Railway.\n\nIt turned out their IP was being statically blocked on a list from our “pre-dynamic” era\n\n\n\nWe began the process of removing this IP list from the blocked list of IPs, by emptying the array\n\n\n\nHowever, the Google Cloud Terraform Provider consumes the source_ranges as an optional list. When nothing is provided, it defaults to applying the action to everything.\n\n\n\n\u003caside\u003e\n💡\n\nTerraform, being written in Golang, makes no such distinction between “empty array” and nil.\n\n\u003c/aside\u003e\n\nWhen a GCP Firewall rule does not specify a source IP range, by default it selects all IPs. Terraform, when supplied with an empty list, changes the underlying GCP firewall rule to remove the source IP list - essentially changing the meaning of the rule as read in Terraform from “block this empty list of IPs” to “block everything”.\n\n# Short Term Resolution\n\nRailway’s policy is to “Revert first, then figure out the underlying issue”\n\nWe could have been faster in a couple places\n\n1. Our page fired to the current oncall, waking them up, instead of the engineer who merged the change\n2. Our bastion Terraform runner ran a plan before the apply, causing 3 minutes of “delay” to resolve\n\nAs such, our short term resolution:\n\n1. Automatically apply “page override” for merging changes (for faster notification)\n2. Skip the “plan” phase during a revert for Terraform\n3. Safely removing the offending firewall rule and audit all remaining legacy Google Cloud firewall rules for loose specificity\n\n# Long Term Mitigation\n\nOur move to metal is aimed at building new primitives that we fully control, and building predictable behaviour into those primitives. Every layer of this system - from the diverse mix of ISP we contract with, to the BGP based L3 fabric running across our switches to the design of our virtualisation stack is aimed towards resiliency and fault isolation.\n\nBy maintaining support for GCP, adding AWS (currently trialing with limited customers) and investing in multiple metal datacenters in each region - we are building not just resilient new infrastructure, but adding support for a diverse mix of compute while simultaneously making this incident a thing of the past.","createdAt":"2026-06-09T12:15:19.644Z","description":"We recently experienced an outage on our Google Cloud ingress networking which briefly halted inbound requests. When production outages occur, it is Railway’s policy to share the public details of what occured.","externalAuthor":false,"featured":false,"featuredImage":{"alt":"Incident Report: April 23rd, 2025 social image","height":null,"id":"375","mimeType":"image/png","url":"https://cms.railway.com/media/8c86f6d8e363dbc9c2690b2bedb7111a65186679c97ba99120b37c8724a29331.png","width":null},"id":"73","publishedAt":"2024-12-16T00:00:00.000Z","seoDescription":"A roughly 10-minute outage on our Google Cloud ingress networking halted inbound requests across all regions; private networking was unaffected.","seoTitle":"Incident Report: December 16th, 2024","slug":"incident-december-16-2024","socialImage":{"alt":"Incident Report: December 16th, 2024 social image","height":null,"id":"385","mimeType":"image/png","url":"https://cms.railway.com/media/2cdcb187f45e610bb6e30467bc080f921b7d75a1871c92651be119fc63996764.png","width":null},"title":"Incident Report: December 16th, 2024","updatedAt":"2024-12-16T00:00:00.000Z"},"relatedPosts":[{"authors":[{"avatar":{"alt":"Mahmoud Abdelwahab headshot","height":460,"id":"1846","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","width":460},"avatarUrl":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","githubUrl":null,"id":"4","name":"Mahmoud Abdelwahab","slug":"mahmoud-abdelwahab","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":null,"createdAt":"2026-08-06T08:07:06.228Z","description":"Announcing: dev.new, an agent that can build, preview, test, and deploy an app from the same place. We tested the whole loop by asking it to ship a real-time multiplayer FPS game that runs in the browser","externalAuthor":false,"featured":true,"featuredImage":{"alt":"dev.new featured image","height":1440,"id":"1983","mimeType":"image/png","url":"https://cms.railway.com/media/dev.new-featured-image-4.png","width":3264},"id":"289","publishedAt":"2026-08-13T08:07:00.000Z","seoDescription":null,"seoTitle":null,"slug":"dev-new","socialImage":{"alt":"dev.new featured image 1","height":720,"id":"1976","mimeType":"image/png","url":"https://cms.railway.com/media/dev.new-featured-image-3.png","width":1632},"title":"Your Cloud should come with an app builder ","updatedAt":"2026-08-14T20:19:27.322Z"},{"authors":[{"avatar":{"alt":"Avatar of Victor Ramirez","height":null,"id":"1855","mimeType":"image/jpeg","url":"https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg","width":null},"avatarUrl":"https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg","githubUrl":null,"id":"34","name":"Victor Ramirez","slug":"victor-ramirez","title":null},{"avatar":{"alt":"Mahmoud Abdelwahab headshot","height":460,"id":"1846","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","width":460},"avatarUrl":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","githubUrl":null,"id":"4","name":"Mahmoud Abdelwahab","slug":"mahmoud-abdelwahab","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":null,"createdAt":"2026-07-15T22:26:48.036Z","description":"Merge now, release when you're ready: turn a feature on for your team, then 10% of users, then everyone. Feature flags are now built into every Railway project, and your agents can run the rollout from the CLI, SDK, and MCP. No third-party service required.","externalAuthor":false,"featured":false,"featuredImage":{"alt":"feature flags featured image","height":1440,"id":"1968","mimeType":"image/png","url":"https://cms.railway.com/media/feature-flags-featured-image.png","width":3264},"id":"278","publishedAt":"2026-07-17T07:00:00.000Z","seoDescription":null,"seoTitle":null,"slug":"feature-flags","socialImage":null,"title":"Roll it out, roll it back, never redeploy","updatedAt":"2026-08-13T16:53:55.126Z"}]},"__N_SSG":true},"page":"/p/[slug]","query":{"slug":"incident-december-16-2024"},"buildId":"nFX7R3M0MdLMAf-Ufh9qa","isFallback":false,"isExperimentalCompile":false,"gsp":true,"scriptLoader":[]}</script></body></html> |