280 lines
82 KiB
HTML
280 lines
82 KiB
HTML
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8" data-next-head=""/><meta name="viewport" content="width=device-width" data-next-head=""/><meta name="twitter:card" content="summary_large_image" data-next-head=""/><meta name="twitter:site" content="@Railway" data-next-head=""/><meta name="twitter:creator" content="@Railway" data-next-head=""/><meta property="og:image:type" content="image/png" data-next-head=""/><meta property="og:image:width" content="1200" data-next-head=""/><meta property="og:image:height" content="630" data-next-head=""/><meta property="og:site_name" content="Railway Blog" data-next-head=""/><title data-next-head="">Zero-Touch Bare Metal at Scale</title><meta name="robots" content="index,follow" data-next-head=""/><meta name="description" content="We’re used to clicking a button and getting a Linux machine in the cloud. When you’re building your own cloud, you’ve got to build the button first." data-next-head=""/><meta property="og:title" content="Zero-Touch Bare Metal at Scale" data-next-head=""/><meta property="og:description" content="We’re used to clicking a button and getting a Linux machine in the cloud. When you’re building your own cloud, you’ve got to build the button first." data-next-head=""/><meta property="og:url" content="https://blog.railway.com/p/data-center-build-part-two" data-next-head=""/><meta property="og:type" content="article" data-next-head=""/><meta property="article:published_time" content="2025-03-21T00:00:00.000Z" data-next-head=""/><meta property="article:modified_time" content="2026-06-23T17:35:28.656Z" data-next-head=""/><meta property="article:author" content="Charith Amarasinghe" data-next-head=""/><meta property="article:section" content="Engineering" data-next-head=""/><meta property="og:image" content="https://cms.railway.com/media/11072737e2dc057b4e9bea2042e030aa7f20352ee34ae755a197c5c00fcad880.png?w=1200&q=90" data-next-head=""/><link rel="canonical" href="https://blog.railway.com/p/data-center-build-part-two" data-next-head=""/><meta name="twitter:title" content="Zero-Touch Bare Metal at Scale" data-next-head=""/><meta name="twitter:description" content="We’re used to clicking a button and getting a Linux machine in the cloud. When you’re building your own cloud, you’ve got to build the button first." data-next-head=""/><meta name="twitter:image" content="https://cms.railway.com/media/11072737e2dc057b4e9bea2042e030aa7f20352ee34ae755a197c5c00fcad880.png?w=1200&q=90" data-next-head=""/><link rel="alternate" type="text/markdown" href="/p/data-center-build-part-two.md" data-next-head=""/><link rel="alternate" type="application/rss+xml" title="Railway Blog" href="https://blog.railway.com/rss.xml"/><link rel="preload" href="/_next/static/media/e4af272ccee01ff0-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/91c3caac73720166-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/3359c9acc30ef440-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/aff755c8b4bb3089-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/3988c35203e0830a-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/427e4a37d3642943-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/css/619a01ff31a99955.css" as="style"/><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"BlogPosting","headline":"Zero-Touch Bare Metal at Scale","description":"We’re used to clicking a button and getting a Linux machine in the cloud. When you’re building your own cloud, you’ve got to build the button first.","image":["https://cms.railway.com/media/11072737e2dc057b4e9bea2042e030aa7f20352ee34ae755a197c5c00fcad880.png?w=1200&q=90"],"datePublished":"2025-03-21T00:00:00.000Z","dateModified":"2026-06-23T17:35:28.656Z","author":{"@type":"Person","name":"Charith Amarasinghe"},"publisher":{"@type":"Organization","name":"Railway","logo":{"@type":"ImageObject","url":"https://railway.com/brand/logo-dark.png"}},"mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.railway.com/p/data-center-build-part-two"}}</script><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https://blog.railway.com"},{"@type":"ListItem","position":2,"name":"Engineering","item":"https://blog.railway.com/engineering"},{"@type":"ListItem","position":3,"name":"Zero-Touch Bare Metal at Scale","item":"https://blog.railway.com/p/data-center-build-part-two"}]}</script><style>:root {
|
||
--foreground: hsl(0, 0%, 100%);
|
||
--background: #181622;
|
||
--secondaryBg: hsl(250, 21%, 11%);
|
||
--blue-50: hsl(220, 55%, 10%);
|
||
--blue-100: hsl(220, 55%, 13%);
|
||
--blue-200: hsl(220, 62%, 25%);
|
||
--blue-300: hsl(220, 68%, 35%);
|
||
--blue-400: hsl(220, 72%, 45%);
|
||
--blue-500: hsl(220, 80%, 55%);
|
||
--blue-600: hsl(220, 80%, 65%);
|
||
--blue-700: hsl(220, 80%, 75%);
|
||
--blue-800: hsl(220, 80%, 85%);
|
||
--blue-900: hsl(220, 80%, 95%);
|
||
--blue-950: hsl(220, 55%, 97%);
|
||
--gray-50: hsl(248, 21%, 13%);
|
||
--gray-100: hsl(246, 18%, 15%);
|
||
--gray-200: hsl(246, 11%, 22%);
|
||
--gray-300: hsl(246, 8%, 35%);
|
||
--gray-400: hsl(246, 7%, 45%);
|
||
--gray-500: hsl(246, 6%, 55%);
|
||
--gray-600: hsl(246, 6%, 65%);
|
||
--gray-700: hsl(246, 6%, 78%);
|
||
--gray-800: hsl(246, 6%, 87%);
|
||
--gray-900: hsl(246, 6%, 95%);
|
||
--gray-950: hsl(246, 6%, 95%);
|
||
--red-50: hsl(1, 35%, 10%);
|
||
--red-100: hsl(1, 45%, 12%);
|
||
--red-200: hsl(1, 55%, 20%);
|
||
--red-300: hsl(1, 62%, 28%);
|
||
--red-400: hsl(1, 62%, 35%);
|
||
--red-500: hsl(1, 62%, 44%);
|
||
--red-600: hsl(1, 62%, 60%);
|
||
--red-700: hsl(1, 62%, 76%);
|
||
--red-800: hsl(1, 64%, 85%);
|
||
--red-900: hsl(1, 68%, 95%);
|
||
--red-950: hsl(1, 55%, 98%);
|
||
--pink-50: hsl(270, 40%, 16%);
|
||
--pink-100: hsl(270, 40%, 16%);
|
||
--pink-200: hsl(270, 45%, 24%);
|
||
--pink-300: hsl(270, 50%, 32%);
|
||
--pink-400: hsl(270, 55%, 43%);
|
||
--pink-500: hsl(270, 60%, 52%);
|
||
--pink-600: hsl(270, 70%, 65%);
|
||
--pink-700: hsl(270, 70%, 75%);
|
||
--pink-800: hsl(270, 70%, 85%);
|
||
--pink-900: hsl(270, 70%, 95%);
|
||
--pink-950: hsl(270, 70%, 95%);
|
||
--green-50: hsl(152, 15%, 10%);
|
||
--green-100: hsl(152, 26%, 11%);
|
||
--green-200: hsl(152, 32%, 16%);
|
||
--green-300: hsl(152, 38%, 24%);
|
||
--green-400: hsl(152, 38%, 34%);
|
||
--green-500: hsl(152, 38%, 42%);
|
||
--green-600: hsl(152, 38%, 60%);
|
||
--green-700: hsl(152, 38%, 70%);
|
||
--green-800: hsl(152, 38%, 80%);
|
||
--green-900: hsl(152, 38%, 91%);
|
||
--green-950: hsl(152, 40%, 97%);
|
||
--yellow-50: hsl(44, 95%, 12%);
|
||
--yellow-100: hsl(44, 95%, 12%);
|
||
--yellow-200: hsl(44, 96%, 24%);
|
||
--yellow-300: hsl(44, 95%, 36%);
|
||
--yellow-400: hsl(44, 95%, 48%);
|
||
--yellow-500: hsl(44, 95%, 60%);
|
||
--yellow-600: hsl(44, 95%, 69%);
|
||
--yellow-700: hsl(44, 95%, 78%);
|
||
--yellow-800: hsl(44, 95%, 86%);
|
||
--yellow-900: hsl(44, 95%, 95%);
|
||
--yellow-950: hsl(44, 95%, 95%);
|
||
}
|
||
|
||
.light {
|
||
--foreground: hsl(250, 24%, 9%);
|
||
--background: #F1F0EF;
|
||
--secondaryBg: hsl(0, 0%, 98%);
|
||
--blue-50: hsl(220, 55%, 97%);
|
||
--blue-100: hsl(220, 80%, 95%);
|
||
--blue-200: hsl(220, 80%, 85%);
|
||
--blue-300: hsl(220, 80%, 75%);
|
||
--blue-400: hsl(220, 80%, 65%);
|
||
--blue-500: hsl(220, 80%, 55%);
|
||
--blue-600: hsl(220, 72%, 45%);
|
||
--blue-700: hsl(220, 68%, 35%);
|
||
--blue-800: hsl(220, 62%, 25%);
|
||
--blue-900: hsl(220, 55%, 13%);
|
||
--blue-950: hsl(220, 55%, 10%);
|
||
--gray-50: hsl(246, 6%, 95%);
|
||
--gray-100: hsl(246, 6%, 95%);
|
||
--gray-200: hsl(246, 6%, 87%);
|
||
--gray-300: hsl(246, 6%, 78%);
|
||
--gray-400: hsl(246, 6%, 65%);
|
||
--gray-500: hsl(246, 6%, 55%);
|
||
--gray-600: hsl(246, 7%, 45%);
|
||
--gray-700: hsl(246, 8%, 35%);
|
||
--gray-800: hsl(246, 11%, 22%);
|
||
--gray-900: hsl(246, 18%, 15%);
|
||
--gray-950: hsl(248, 21%, 13%);
|
||
--red-50: hsl(1, 55%, 98%);
|
||
--red-100: hsl(1, 68%, 95%);
|
||
--red-200: hsl(1, 64%, 85%);
|
||
--red-300: hsl(1, 62%, 76%);
|
||
--red-400: hsl(1, 62%, 60%);
|
||
--red-500: hsl(1, 62%, 44%);
|
||
--red-600: hsl(1, 62%, 35%);
|
||
--red-700: hsl(1, 62%, 28%);
|
||
--red-800: hsl(1, 55%, 20%);
|
||
--red-900: hsl(1, 45%, 12%);
|
||
--red-950: hsl(1, 35%, 10%);
|
||
--pink-50: hsl(270, 70%, 95%);
|
||
--pink-100: hsl(270, 70%, 95%);
|
||
--pink-200: hsl(270, 70%, 85%);
|
||
--pink-300: hsl(270, 70%, 75%);
|
||
--pink-400: hsl(270, 70%, 65%);
|
||
--pink-500: hsl(270, 60%, 52%);
|
||
--pink-600: hsl(270, 55%, 43%);
|
||
--pink-700: hsl(270, 50%, 32%);
|
||
--pink-800: hsl(270, 45%, 24%);
|
||
--pink-900: hsl(270, 40%, 16%);
|
||
--pink-950: hsl(270, 40%, 16%);
|
||
--green-50: hsl(152, 40%, 97%);
|
||
--green-100: hsl(152, 38%, 91%);
|
||
--green-200: hsl(152, 38%, 80%);
|
||
--green-300: hsl(152, 38%, 70%);
|
||
--green-400: hsl(152, 38%, 60%);
|
||
--green-500: hsl(152, 38%, 42%);
|
||
--green-600: hsl(152, 38%, 34%);
|
||
--green-700: hsl(152, 38%, 24%);
|
||
--green-800: hsl(152, 32%, 16%);
|
||
--green-900: hsl(152, 26%, 11%);
|
||
--green-950: hsl(152, 15%, 10%);
|
||
--yellow-50: hsl(44, 95%, 95%);
|
||
--yellow-100: hsl(44, 95%, 95%);
|
||
--yellow-200: hsl(44, 95%, 86%);
|
||
--yellow-300: hsl(44, 95%, 78%);
|
||
--yellow-400: hsl(44, 95%, 69%);
|
||
--yellow-500: hsl(44, 95%, 60%);
|
||
--yellow-600: hsl(44, 95%, 48%);
|
||
--yellow-700: hsl(44, 95%, 36%);
|
||
--yellow-800: hsl(44, 96%, 24%);
|
||
--yellow-900: hsl(44, 95%, 12%);
|
||
--yellow-950: hsl(44, 95%, 12%);
|
||
}
|
||
|
||
.dark {
|
||
--foreground: hsl(0, 0%, 100%);
|
||
--background: #181622;
|
||
--secondaryBg: hsl(250, 21%, 11%);
|
||
--blue-50: hsl(220, 55%, 10%);
|
||
--blue-100: hsl(220, 55%, 13%);
|
||
--blue-200: hsl(220, 62%, 25%);
|
||
--blue-300: hsl(220, 68%, 35%);
|
||
--blue-400: hsl(220, 72%, 45%);
|
||
--blue-500: hsl(220, 80%, 55%);
|
||
--blue-600: hsl(220, 80%, 65%);
|
||
--blue-700: hsl(220, 80%, 75%);
|
||
--blue-800: hsl(220, 80%, 85%);
|
||
--blue-900: hsl(220, 80%, 95%);
|
||
--blue-950: hsl(220, 55%, 97%);
|
||
--gray-50: hsl(248, 21%, 13%);
|
||
--gray-100: hsl(246, 18%, 15%);
|
||
--gray-200: hsl(246, 11%, 22%);
|
||
--gray-300: hsl(246, 8%, 35%);
|
||
--gray-400: hsl(246, 7%, 45%);
|
||
--gray-500: hsl(246, 6%, 55%);
|
||
--gray-600: hsl(246, 6%, 65%);
|
||
--gray-700: hsl(246, 6%, 78%);
|
||
--gray-800: hsl(246, 6%, 87%);
|
||
--gray-900: hsl(246, 6%, 95%);
|
||
--gray-950: hsl(246, 6%, 95%);
|
||
--red-50: hsl(1, 35%, 10%);
|
||
--red-100: hsl(1, 45%, 12%);
|
||
--red-200: hsl(1, 55%, 20%);
|
||
--red-300: hsl(1, 62%, 28%);
|
||
--red-400: hsl(1, 62%, 35%);
|
||
--red-500: hsl(1, 62%, 44%);
|
||
--red-600: hsl(1, 62%, 60%);
|
||
--red-700: hsl(1, 62%, 76%);
|
||
--red-800: hsl(1, 64%, 85%);
|
||
--red-900: hsl(1, 68%, 95%);
|
||
--red-950: hsl(1, 55%, 98%);
|
||
--pink-50: hsl(270, 40%, 16%);
|
||
--pink-100: hsl(270, 40%, 16%);
|
||
--pink-200: hsl(270, 45%, 24%);
|
||
--pink-300: hsl(270, 50%, 32%);
|
||
--pink-400: hsl(270, 55%, 43%);
|
||
--pink-500: hsl(270, 60%, 52%);
|
||
--pink-600: hsl(270, 70%, 65%);
|
||
--pink-700: hsl(270, 70%, 75%);
|
||
--pink-800: hsl(270, 70%, 85%);
|
||
--pink-900: hsl(270, 70%, 95%);
|
||
--pink-950: hsl(270, 70%, 95%);
|
||
--green-50: hsl(152, 15%, 10%);
|
||
--green-100: hsl(152, 26%, 11%);
|
||
--green-200: hsl(152, 32%, 16%);
|
||
--green-300: hsl(152, 38%, 24%);
|
||
--green-400: hsl(152, 38%, 34%);
|
||
--green-500: hsl(152, 38%, 42%);
|
||
--green-600: hsl(152, 38%, 60%);
|
||
--green-700: hsl(152, 38%, 70%);
|
||
--green-800: hsl(152, 38%, 80%);
|
||
--green-900: hsl(152, 38%, 91%);
|
||
--green-950: hsl(152, 40%, 97%);
|
||
--yellow-50: hsl(44, 95%, 12%);
|
||
--yellow-100: hsl(44, 95%, 12%);
|
||
--yellow-200: hsl(44, 96%, 24%);
|
||
--yellow-300: hsl(44, 95%, 36%);
|
||
--yellow-400: hsl(44, 95%, 48%);
|
||
--yellow-500: hsl(44, 95%, 60%);
|
||
--yellow-600: hsl(44, 95%, 69%);
|
||
--yellow-700: hsl(44, 95%, 78%);
|
||
--yellow-800: hsl(44, 95%, 86%);
|
||
--yellow-900: hsl(44, 95%, 95%);
|
||
--yellow-950: hsl(44, 95%, 95%);
|
||
}</style><link rel="stylesheet" href="/_next/static/css/619a01ff31a99955.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" noModule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-0b474a604a067f92.js" defer=""></script><script src="/_next/static/chunks/framework-90fc9b0f7c4706c1.js" defer=""></script><script src="/_next/static/chunks/main-e89abd6570bbb9dc.js" defer=""></script><script src="/_next/static/chunks/pages/_app-3049b372678b389f.js" defer=""></script><script src="/_next/static/chunks/5399-d49efedd870f41de.js" defer=""></script><script src="/_next/static/chunks/3801-cdd2681d27083ad8.js" defer=""></script><script src="/_next/static/chunks/9594-1c44b12020a3af0a.js" defer=""></script><script src="/_next/static/chunks/4696-fd1712bc36a5bc50.js" defer=""></script><script src="/_next/static/chunks/pages/p/%5Bslug%5D-ddadb1ca54eaab0f.js" defer=""></script><script src="/_next/static/nFX7R3M0MdLMAf-Ufh9qa/_buildManifest.js" defer=""></script><script src="/_next/static/nFX7R3M0MdLMAf-Ufh9qa/_ssgManifest.js" defer=""></script></head><body><div id="__next"><script>!function(){try{var d=document.documentElement,c=d.classList;c.remove('light','dark');var e=localStorage.getItem('theme');if('system'===e||(!e&&true)){var t='(prefers-color-scheme: dark)',m=window.matchMedia(t);if(m.media!==t||m.matches){d.style.colorScheme = 'dark';c.add('dark')}else{d.style.colorScheme = 'light';c.add('light')}}else if(e){c.add(e|| '')}if(e==='light'||e==='dark')d.style.colorScheme=e}catch(e){}}()</script><div class="__variable_f367f3 __variable_0b668e __variable_ecea63 font-sans"><div class="px-5 md:px-8"><nav class="max-w-6xl mx-auto py-6 flex justify-between items-center border-b border-gray-100"><a class="flex items-center space-x-4" href="/"><svg data-v-423bf9ae="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 59.99945071644884 60" class="logo w-8"><g data-v-423bf9ae="" id="6f639fe5-02e1-4640-a1fb-f4b22b64e5ef" transform="matrix(0.5999641134746578,0,0,0.5999641134746578,-437.5538206497148,-266.68705118956507)" stroke="none"><path d="M729.872 487.327a50.86 50.86 0 0 0-.464 5.033h75.879c-.265-.518-.621-.985-.98-1.442-12.972-16.769-19.95-15.315-29.932-15.741-3.328-.137-5.585-.192-18.832-.192-7.09 0-14.798.018-22.304.038.737-1.746 1.6-3.419 2.525-5.061-2.885 5.106-4.865 10.771-5.805 16.789l.891-4.397c.007-.031.018-.063.024-.094h38.883v5.067h-39.885zM805.885 497.432h-76.438c.08 1.352.206 2.686.388 4.002h70.571c3.146 0 4.907-1.786 5.479-4.002zM733.851 515.257a52.226 52.226 0 0 1-1.98-4.997c6.608 19.89 25.328 34.251 47.433 34.251 20.205 0 37.566-12.007 45.452-29.254h-90.905zM729.38 492.915c-.018.531-.08 1.055-.08 1.589 0 .538.063 1.059.08 1.59v-3.179zM824.77 515.229z"></path><path d="M779.303 444.505c-18.682 0-34.939 10.265-43.524 25.439 6.709-.014 19.775-.022 19.775-.022h.003v-.005c15.444 0 16.018.069 19.035.195l1.868.069c6.507.217 14.505.916 20.798 5.68 3.416 2.584 8.348 8.287 11.288 12.35 2.718 3.758 3.5 8.078 1.652 12.217-1.701 3.804-5.361 6.073-9.793 6.073H730.85l-.884-4.201c.426 2.707 1.037 5.344 1.879 7.886h94.914a49.863 49.863 0 0 0 2.546-15.682c.001-27.611-22.386-49.999-50.002-49.999z"></path></g></svg><span class="text-xl font-semibold">Blog</span></a><div class="text-gray-600 flex items-center space-x-6"><a class="text-sm hover:text-pink-600" href="https://railway.com" target="_blank" rel="noreferrer noopener">Go to Homepage</a><a class="hover:text-pink-600" href="/rss.xml" title="RSS Feed"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4 11a9 9 0 0 1 9 9"></path><path d="M4 4a16 16 0 0 1 16 16"></path><circle cx="5" cy="19" r="1"></circle></svg></a></div></nav></div><div class="min-h-screen overflow-x-hidden relative"><div class="mt-10 mb-5 px-5 md:px-8 mx-auto"><article class="max-w-6xl mx-auto mt-24 mb-12 border-b border-gray-100 pb-32"><div class="max-w-[704px] mx-auto flex items-center text-sm text-gray-600 space-x-3"><div class="flex items-center space-x-3"><div class="flex items-center"><img src="https://cms.railway.com/media/452039afd9d5078059203ebfb4028e92717d03b94e10832c814d2ca23c5bce39.png" alt="Avatar of Charith Amarasinghe" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/></div><span>Charith Amarasinghe</span></div><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><time dateTime="2025-03-21T00:00:00.000Z">Mar 21, 2025</time></div><header class="mt-5 mb-16 max-w-[704px] mx-auto"><h1 class="text-post-title font-medium font-serif">Zero-Touch Bare Metal at Scale</h1></header><section class="max-w-[704px] mx-auto text-base sm:text-lg"><nav style="position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border-width:0" aria-label="Table of Contents"><h2>Table of Contents</h2><ol><li style="margin-left:0rem"><a href="#sorting-your-lego-pieces">Sorting your LEGO pieces</a></li><li style="margin-left:0rem"><a href="#who-needs-webhooks-when-we’ve-got-claude">Who needs webhooks when we’ve got Claude</a></li><li style="margin-left:0rem"><a href="#low-config-networking-with-bgp-unnumbered">Low Config Networking with BGP Unnumbered</a></li><li style="margin-left:0rem"><a href="#building-software-to-run-hardware-to-run-software">Building Software to Run Hardware to Run Software</a></li></ol></nav><div><p class="mb-4 text-gray-800">We’ve all gotten used to clicking a button and getting a Linux machine running in the cloud. But when you’re building your own cloud, you’ve got to build the button first.</p>
|
||
<p class="mb-4 text-gray-800">Lately we’ve been <a class="markdown-inline-link" href="https://blog.railway.com/p/data-center-build-part-one" target="_blank" rel="noreferrer noopener">writing</a> about building out our <a class="markdown-inline-link" href="https://blog.railway.com/p/launch-week-02-welcome" target="_blank" rel="noreferrer noopener">Metal infrastructure</a> one rack at a time.</p>
|
||
<p class="mb-4 text-gray-800">In our <a class="markdown-inline-link" href="https://blog.railway.com/p/data-center-build-part-one" target="_blank" rel="noreferrer noopener">last blog</a>, we spoke about the trials of building out the physical infrastructure. In this episode, we talk about how we operationalize the hardware once it’s installed.</p>
|
||
<a class="relative no-underline" href="/p/data-center-build-part-two#sorting-your-lego-pieces"><span id="sorting-your-lego-pieces" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-16 mb-8">Sorting your LEGO pieces</h2></a>
|
||
<p class="mb-4 text-gray-800">You’ve built your dream server with dual-redundant NICs and multiple redundant NVMe drives for resilience. You’ve ordered 100 units and got them all racked up and wired to your detailed diagrams. You go to the DC with your USB stick and reboot into your favorite Linux distro’s installer only to be greeted by the “Choose your Network Interface” screen with a dozen or so incomprehensible interface names.</p>
|
||
<p class="mb-4 text-gray-800">Herein lies our first hurdle — how do you map the physical arrangement of hardware to what your operating system sees?</p>
|
||
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/d3ff11f80e2fac1abd85a2b07b841c389163eefb6b3228c15e93f22bb720cc97.png?w=768&format=webp&q=90" srcSet="https://cms.railway.com/media/d3ff11f80e2fac1abd85a2b07b841c389163eefb6b3228c15e93f22bb720cc97.png?w=320&format=webp&q=90 320w, https://cms.railway.com/media/d3ff11f80e2fac1abd85a2b07b841c389163eefb6b3228c15e93f22bb720cc97.png?w=480&format=webp&q=90 480w, https://cms.railway.com/media/d3ff11f80e2fac1abd85a2b07b841c389163eefb6b3228c15e93f22bb720cc97.png?w=640&format=webp&q=90 640w, https://cms.railway.com/media/d3ff11f80e2fac1abd85a2b07b841c389163eefb6b3228c15e93f22bb720cc97.png?w=768&format=webp&q=90 768w, https://cms.railway.com/media/d3ff11f80e2fac1abd85a2b07b841c389163eefb6b3228c15e93f22bb720cc97.png?w=960&format=webp&q=90 960w, https://cms.railway.com/media/d3ff11f80e2fac1abd85a2b07b841c389163eefb6b3228c15e93f22bb720cc97.png?w=1200&format=webp&q=90 1200w, https://cms.railway.com/media/d3ff11f80e2fac1abd85a2b07b841c389163eefb6b3228c15e93f22bb720cc97.png?w=1440&format=webp&q=90 1440w, https://cms.railway.com/media/d3ff11f80e2fac1abd85a2b07b841c389163eefb6b3228c15e93f22bb720cc97.png?w=1600&format=webp&q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="A Supermicro server with 12 NVMe Bays - Guess which bay is /dev/nvme0n2 as seen by Linux? (That’s a trick question)." class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">A Supermicro server with 12 NVMe Bays - Guess which bay is /dev/nvme0n2 as seen by Linux? (That’s a trick question).</figcaption></figure><div class="flex w-full p-4 my-8 rounded border border-transparent bg-blue-100"><div class="text-yellow-500">💡</div><div class="flex flex-col w-full"><div class="ml-4 text-foreground"><p class="mb-4 text-gray-800">When buying build-to-order servers, it’s essential to include instructions specifying exactly where each Network Card or NVMe Drive should get installed. Otherwise you might end up with multiple different configurations across different orders.</p></div></div></div><p class="mb-4 text-gray-800">It helps first to take a step back and discuss how Linux names devices.</p>
|
||
<p class="mb-4 text-gray-800">When a host boots Linux, the OS enumerates the attached hardware. Most commonly, devices are attached to the PCIe bus and Linux begins enumerating these according to the hierarchical structure of the bus. When Linux encounters a device during this traversal, the <a class="markdown-inline-link" href="https://en.wikipedia.org/wiki/Udev" target="_blank" rel="noreferrer noopener">udev</a> daemon will get an event and associate a number of identifiers with the device - it’ll then use these identifiers to formulate a name which it then assigns to the device nodes it creates in <code class="inline-code">/dev</code> and elsewhere.</p>
|
||
<p class="mb-4 text-gray-800">The consequence of this approach is that device names can be very unstable, especially if the hardware layout changes between boots or if the enumeration order is non-deterministic. If you used Linux in the olden days, you’d know the pain of plugging in a new PCIe card and booting only to figure out that your networking broke. Despite the many critiques that could be leveled against SystemD, it does succeed in <a class="markdown-inline-link" href="https://github.com/systemd/systemd/blob/main/docs/PREDICTABLE_INTERFACE_NAMES.md" target="_blank" rel="noreferrer noopener">addressing these problems for network interfaces since v197</a>. But storage device naming is still a crapshoot and better achieved by device serial number.</p>
|
||
<p class="mb-4 text-gray-800">Our approach to addressing this unpredictability is to lean on Redfish - a HTTP API for Board Management Controllers (BMCs) attached to server motherboards. Redfish APIs can enumerate the hardware on a board, detailing PCIe cards, NVMe drives, their serial numbers and/or MAC addresses and their physical locations.</p>
|
||
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/cd29ad9d15caa024ebf91e314c7bb6056b751ef00abbde4c5783d5ab14c60a53.png?w=768&format=webp&q=90" srcSet="https://cms.railway.com/media/cd29ad9d15caa024ebf91e314c7bb6056b751ef00abbde4c5783d5ab14c60a53.png?w=320&format=webp&q=90 320w, https://cms.railway.com/media/cd29ad9d15caa024ebf91e314c7bb6056b751ef00abbde4c5783d5ab14c60a53.png?w=480&format=webp&q=90 480w, https://cms.railway.com/media/cd29ad9d15caa024ebf91e314c7bb6056b751ef00abbde4c5783d5ab14c60a53.png?w=640&format=webp&q=90 640w, https://cms.railway.com/media/cd29ad9d15caa024ebf91e314c7bb6056b751ef00abbde4c5783d5ab14c60a53.png?w=768&format=webp&q=90 768w, https://cms.railway.com/media/cd29ad9d15caa024ebf91e314c7bb6056b751ef00abbde4c5783d5ab14c60a53.png?w=960&format=webp&q=90 960w, https://cms.railway.com/media/cd29ad9d15caa024ebf91e314c7bb6056b751ef00abbde4c5783d5ab14c60a53.png?w=1200&format=webp&q=90 1200w, https://cms.railway.com/media/cd29ad9d15caa024ebf91e314c7bb6056b751ef00abbde4c5783d5ab14c60a53.png?w=1440&format=webp&q=90 1440w, https://cms.railway.com/media/cd29ad9d15caa024ebf91e314c7bb6056b751ef00abbde4c5783d5ab14c60a53.png?w=1600&format=webp&q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="An Extract of a Redfish System object scraped from one of our storage nodes" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">An Extract of a Redfish System object scraped from one of our storage nodes</figcaption></figure>
|
||
<p class="mb-4 text-gray-800">Our very first step once a rack is installed is to build a CSV of identifiers for the equipment in the rack - hostname, BMC MAC address, BMC password, and a few other details. We then push this data via gRPC to an internal control plane called MetalCP.</p>
|
||
<p class="mb-4 text-gray-800">MetalCP runs a Temporal worker which implements a Host Import workflow. For each server, we then kick off a workflow that runs through the following steps:</p>
|
||
<ol class="list-decimal pl-6 mb-4 space-y-4 text-gray-800">
|
||
<li>Match the device to its representation in our internal DCIM tool (Railyard)</li>
|
||
<li>Connect to the datacenter's management network via Tailscale</li>
|
||
<li>Connect to the management router at the datacenter and identify the DHCP lease assigned to the BMC (via its MAC)</li>
|
||
<li>Connect to the BMC of the server via this IP and scrape all available data</li>
|
||
<li>Create an internal Protobuf representation of the hardware layout</li>
|
||
<li>Create static DHCP leases for the BMC and for the management NIC on the host using discovered MAC addresses from the scrape</li>
|
||
<li>Update a DB with all the details about the server</li>
|
||
</ol>
|
||
<p class="mb-4 text-gray-800">An import workflow takes less that a minute to complete in most cases and Temporal ensures recovery from any transient failures.</p>
|
||
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/08bffdd67417b6b44554bacaf2bc8ef06e43cef1040e4ea5ea013329bd3b8552.png?w=768&format=webp&q=90" srcSet="https://cms.railway.com/media/08bffdd67417b6b44554bacaf2bc8ef06e43cef1040e4ea5ea013329bd3b8552.png?w=320&format=webp&q=90 320w, https://cms.railway.com/media/08bffdd67417b6b44554bacaf2bc8ef06e43cef1040e4ea5ea013329bd3b8552.png?w=480&format=webp&q=90 480w, https://cms.railway.com/media/08bffdd67417b6b44554bacaf2bc8ef06e43cef1040e4ea5ea013329bd3b8552.png?w=640&format=webp&q=90 640w, https://cms.railway.com/media/08bffdd67417b6b44554bacaf2bc8ef06e43cef1040e4ea5ea013329bd3b8552.png?w=768&format=webp&q=90 768w, https://cms.railway.com/media/08bffdd67417b6b44554bacaf2bc8ef06e43cef1040e4ea5ea013329bd3b8552.png?w=960&format=webp&q=90 960w, https://cms.railway.com/media/08bffdd67417b6b44554bacaf2bc8ef06e43cef1040e4ea5ea013329bd3b8552.png?w=1200&format=webp&q=90 1200w, https://cms.railway.com/media/08bffdd67417b6b44554bacaf2bc8ef06e43cef1040e4ea5ea013329bd3b8552.png?w=1440&format=webp&q=90 1440w, https://cms.railway.com/media/08bffdd67417b6b44554bacaf2bc8ef06e43cef1040e4ea5ea013329bd3b8552.png?w=1600&format=webp&q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="The timeline of a Host Import workflow" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">The timeline of a Host Import workflow</figcaption></figure>
|
||
<p class="mb-4 text-gray-800">The database record that is stored by a host workflow contains all the information you could want about a server. We generate:</p>
|
||
<ul class="list-disc pl-6 mb-4 space-y-4 text-gray-800">
|
||
<li>A list of Network Interface Cards, their Physical Location (Slot), and MAC addresses for each Port</li>
|
||
<li>A list of NVMe Drives, their Serial Numbers, Model Numbers, and Physical Slot IDs</li>
|
||
<li>System stats such as CPU core counts, RAM size, and hardware identifiers</li>
|
||
</ul>
|
||
<p class="mb-4 text-gray-800">We then match this hardware specification against a list of known configurations. These configurations encode details such as network interface names assigned to specific PCIe slots and NVMe drive bay identifiers. A hardware configuration is as simple as a set of conditionals in Golang that match the key distinguishing factors of a specific type of server, and a config object containing stable interface and drive names.</p>
|
||
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/d7e4a476ff49ae533591a4cc75e7d0110eabf0703129ce3d1fcb5beb11b25929.png?w=768&format=webp&q=90" srcSet="https://cms.railway.com/media/d7e4a476ff49ae533591a4cc75e7d0110eabf0703129ce3d1fcb5beb11b25929.png?w=320&format=webp&q=90 320w, https://cms.railway.com/media/d7e4a476ff49ae533591a4cc75e7d0110eabf0703129ce3d1fcb5beb11b25929.png?w=480&format=webp&q=90 480w, https://cms.railway.com/media/d7e4a476ff49ae533591a4cc75e7d0110eabf0703129ce3d1fcb5beb11b25929.png?w=640&format=webp&q=90 640w, https://cms.railway.com/media/d7e4a476ff49ae533591a4cc75e7d0110eabf0703129ce3d1fcb5beb11b25929.png?w=768&format=webp&q=90 768w, https://cms.railway.com/media/d7e4a476ff49ae533591a4cc75e7d0110eabf0703129ce3d1fcb5beb11b25929.png?w=960&format=webp&q=90 960w, https://cms.railway.com/media/d7e4a476ff49ae533591a4cc75e7d0110eabf0703129ce3d1fcb5beb11b25929.png?w=1200&format=webp&q=90 1200w, https://cms.railway.com/media/d7e4a476ff49ae533591a4cc75e7d0110eabf0703129ce3d1fcb5beb11b25929.png?w=1440&format=webp&q=90 1440w, https://cms.railway.com/media/d7e4a476ff49ae533591a4cc75e7d0110eabf0703129ce3d1fcb5beb11b25929.png?w=1600&format=webp&q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="An example hardware identification function, pb.SystemInfo is an encoding of raw data we’ve scraped from Redfish" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">An example hardware identification function, pb.SystemInfo is an encoding of raw data we’ve scraped from Redfish</figcaption></figure>
|
||
<p class="mb-4 text-gray-800">A custom plugin exposes this Hardware Config object to Ansible, allowing us to reference NVMe disks and network interface names with Jinja template expressions. For example, a NVMe drive in Bay 0 can be uniquely addressed as <code class="inline-code">/dev/disks/by-id/nvme-{{ drive_bays.DiskBay0.device_model }}_{{ drive_bays.DiskBay0.serial_number }}</code> .</p>
|
||
<p class="mb-4 text-gray-800">This approach lets us build config in any shape we want without leaving anything to chance. The import workflow also flags faults in the hardware; if a server is not reporting a NIC or a DIMM of RAM, the workflow will fail since the hardware won’t match a known configuration. We’ve thus far identified servers with faulty RAM and servers with NICs installed in the wrong slots through this mechanism.</p>
|
||
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/9980c9009fa61b5e84c6954a71184b3e01e7283aa971ba85889d12035ada00c1.gif" sizes="(max-width: 768px) 100vw, 736px" alt="CleanShot 2025-02-28 at 21.08.58.gif" class="w-full rounded-[8px]" loading="lazy" decoding="async"/></figure>
|
||
<p class="mb-4 text-gray-800">Configuration is one step, but getting at Ansible still needs an OS to get installed. So how do we one-click ourselves out of installing Linux in the first place? The answer involves a pinch of AI 🪄.</p>
|
||
<a class="relative no-underline" href="/p/data-center-build-part-two#who-needs-webhooks-when-we%E2%80%99ve-got-claude"><span id="who-needs-webhooks-when-we’ve-got-claude" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-16 mb-8">Who needs webhooks when we’ve got Claude</h2></a>
|
||
<p class="mb-4 text-gray-800">When we first started provisioning servers, we did it manually with 20 Web KVM tabs in Chrome and manually interacting with debian-installer. Over time we’ve evolved to use less and less human intervention.</p>
|
||
<p class="mb-4 text-gray-800">The Debian Installer can network boot from PXE, and <a class="markdown-inline-link" href="https://github.com/danderson/netboot/tree/main/pixiecore" target="_blank" rel="noreferrer noopener">Pixiecore from Dave Anderson</a> can wrap all the PXE complexity in a few HTTP calls. We use MetalCP as a backend to Pixicore and return a simple JSON payload describing the netboot kernel, initramfs, and kernel command-line. Debian can accept a pre-seed file over HTTP if networking is configured.</p>
|
||
<p class="mb-4 text-gray-800">We use our knowledge of the PXE booting servers MAC address, plus the system info we’ve scraped from Redfish, to create a kernel command-line and preseed file tailored to the booting machine.</p>
|
||
<p class="mb-4 text-gray-800">These are all exposed as HTTP APIs proxied by Pixiecore to the PXE booting machine.</p>
|
||
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/4222aee90a7a32a5ec0a16063fba540842d991437bedeac04cceff2634d997f8.png?w=768&format=webp&q=90" srcSet="https://cms.railway.com/media/4222aee90a7a32a5ec0a16063fba540842d991437bedeac04cceff2634d997f8.png?w=320&format=webp&q=90 320w, https://cms.railway.com/media/4222aee90a7a32a5ec0a16063fba540842d991437bedeac04cceff2634d997f8.png?w=480&format=webp&q=90 480w, https://cms.railway.com/media/4222aee90a7a32a5ec0a16063fba540842d991437bedeac04cceff2634d997f8.png?w=640&format=webp&q=90 640w, https://cms.railway.com/media/4222aee90a7a32a5ec0a16063fba540842d991437bedeac04cceff2634d997f8.png?w=768&format=webp&q=90 768w, https://cms.railway.com/media/4222aee90a7a32a5ec0a16063fba540842d991437bedeac04cceff2634d997f8.png?w=960&format=webp&q=90 960w, https://cms.railway.com/media/4222aee90a7a32a5ec0a16063fba540842d991437bedeac04cceff2634d997f8.png?w=1200&format=webp&q=90 1200w, https://cms.railway.com/media/4222aee90a7a32a5ec0a16063fba540842d991437bedeac04cceff2634d997f8.png?w=1440&format=webp&q=90 1440w, https://cms.railway.com/media/4222aee90a7a32a5ec0a16063fba540842d991437bedeac04cceff2634d997f8.png?w=1600&format=webp&q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="The function that generates our PXE boot response" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">The function that generates our PXE boot response</figcaption></figure>
|
||
<p class="mb-4 text-gray-800">Getting a host to a PXE bootable state requires us to reboot the server, but we don’t want this reboot action to happen on a server that may be running user code. To achieve this, we implement a logical state machine for each host in the provisioning process and orchestrate the OS install via another Temporal workflow.</p>
|
||
<p class="mb-4 text-gray-800">But how does the Workflow know which state the server is in during the install? Redfish APIs tell us that it’s powered on, but little else.</p>
|
||
<p class="mb-4 text-gray-800">Since it’s 2025, we just ask Claude.</p>
|
||
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/377b8ecb0e80974d8bd2abe6626f01b45d0b4423e1b23a37c2982363e7cad23e.png?w=768&format=webp&q=90" srcSet="https://cms.railway.com/media/377b8ecb0e80974d8bd2abe6626f01b45d0b4423e1b23a37c2982363e7cad23e.png?w=320&format=webp&q=90 320w, https://cms.railway.com/media/377b8ecb0e80974d8bd2abe6626f01b45d0b4423e1b23a37c2982363e7cad23e.png?w=480&format=webp&q=90 480w, https://cms.railway.com/media/377b8ecb0e80974d8bd2abe6626f01b45d0b4423e1b23a37c2982363e7cad23e.png?w=640&format=webp&q=90 640w, https://cms.railway.com/media/377b8ecb0e80974d8bd2abe6626f01b45d0b4423e1b23a37c2982363e7cad23e.png?w=768&format=webp&q=90 768w, https://cms.railway.com/media/377b8ecb0e80974d8bd2abe6626f01b45d0b4423e1b23a37c2982363e7cad23e.png?w=960&format=webp&q=90 960w, https://cms.railway.com/media/377b8ecb0e80974d8bd2abe6626f01b45d0b4423e1b23a37c2982363e7cad23e.png?w=1200&format=webp&q=90 1200w, https://cms.railway.com/media/377b8ecb0e80974d8bd2abe6626f01b45d0b4423e1b23a37c2982363e7cad23e.png?w=1440&format=webp&q=90 1440w, https://cms.railway.com/media/377b8ecb0e80974d8bd2abe6626f01b45d0b4423e1b23a37c2982363e7cad23e.png?w=1600&format=webp&q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="An example of Claude’s happily recognizing a server POST screen" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">An example of Claude’s happily recognizing a server POST screen</figcaption></figure>
|
||
<p class="mb-4 text-gray-800">Supermicro introduced a CaptureScreen OEM API in their Redfish 1.14 release; with this API we can obtain a near real-time image of the server screen. With a basic prompt to Claude, we can then get a JSON payload that describes the state of the server at this point. Combining this into a Temporal workflow - alongside the PXE boot automation above - we can achieve an OS install and provision with one gRPC API call.</p>
|
||
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/256bd5ce25e1ac181d517dc9bbd2cbec0440a0f4c4a75d7d85989caa47a4ec02.png?w=768&format=webp&q=90" srcSet="https://cms.railway.com/media/256bd5ce25e1ac181d517dc9bbd2cbec0440a0f4c4a75d7d85989caa47a4ec02.png?w=320&format=webp&q=90 320w, https://cms.railway.com/media/256bd5ce25e1ac181d517dc9bbd2cbec0440a0f4c4a75d7d85989caa47a4ec02.png?w=480&format=webp&q=90 480w, https://cms.railway.com/media/256bd5ce25e1ac181d517dc9bbd2cbec0440a0f4c4a75d7d85989caa47a4ec02.png?w=640&format=webp&q=90 640w, https://cms.railway.com/media/256bd5ce25e1ac181d517dc9bbd2cbec0440a0f4c4a75d7d85989caa47a4ec02.png?w=768&format=webp&q=90 768w, https://cms.railway.com/media/256bd5ce25e1ac181d517dc9bbd2cbec0440a0f4c4a75d7d85989caa47a4ec02.png?w=960&format=webp&q=90 960w, https://cms.railway.com/media/256bd5ce25e1ac181d517dc9bbd2cbec0440a0f4c4a75d7d85989caa47a4ec02.png?w=1200&format=webp&q=90 1200w, https://cms.railway.com/media/256bd5ce25e1ac181d517dc9bbd2cbec0440a0f4c4a75d7d85989caa47a4ec02.png?w=1440&format=webp&q=90 1440w, https://cms.railway.com/media/256bd5ce25e1ac181d517dc9bbd2cbec0440a0f4c4a75d7d85989caa47a4ec02.png?w=1600&format=webp&q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="The temporal workflow polls the screen and monitors the install to completion, updating an internal DB at each step" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">The temporal workflow polls the screen and monitors the install to completion, updating an internal DB at each step</figcaption></figure>
|
||
<p class="mb-4 text-gray-800">There are probably more effective methods of achieving the same, but it costs us less than a dollar to provision 50 servers using Claude to screen-scrape every minute during the install.</p>
|
||
<p class="mb-4 text-gray-800">Now that an OS is installed, some duct-tape and Ansible will allow us to get some basic software running on the machine. However, bringing up networking is something else that’s typically an annoyance.</p>
|
||
<a class="relative no-underline" href="/p/data-center-build-part-two#low-config-networking-with-bgp-unnumbered"><span id="low-config-networking-with-bgp-unnumbered" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-16 mb-8">Low Config Networking with BGP Unnumbered</h2></a>
|
||
<p class="mb-4 text-gray-800">All the solutions we’ve discussed thus far have relied on a Management (or Out of Band) network. This is a dedicated Gigabit Ethernet network that links a management NIC, BMCs and other support infrastructure inside the cage. This network isn’t built to scale as it instead relies on being limited to a few hundred hosts at most and uses off-the-shelf routers, DHCP and VLANs for isolation and operation with low fault tolerance. The network that carries user traffic, the dataplane network as we term it, has very different requirements.</p>
|
||
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/eab2c052ee3aa7408ec67b7fc3e897c761f00d7cdf9d2087794e3b49ad5b68e8.png?w=768&format=webp&q=90" srcSet="https://cms.railway.com/media/eab2c052ee3aa7408ec67b7fc3e897c761f00d7cdf9d2087794e3b49ad5b68e8.png?w=320&format=webp&q=90 320w, https://cms.railway.com/media/eab2c052ee3aa7408ec67b7fc3e897c761f00d7cdf9d2087794e3b49ad5b68e8.png?w=480&format=webp&q=90 480w, https://cms.railway.com/media/eab2c052ee3aa7408ec67b7fc3e897c761f00d7cdf9d2087794e3b49ad5b68e8.png?w=640&format=webp&q=90 640w, https://cms.railway.com/media/eab2c052ee3aa7408ec67b7fc3e897c761f00d7cdf9d2087794e3b49ad5b68e8.png?w=768&format=webp&q=90 768w, https://cms.railway.com/media/eab2c052ee3aa7408ec67b7fc3e897c761f00d7cdf9d2087794e3b49ad5b68e8.png?w=960&format=webp&q=90 960w, https://cms.railway.com/media/eab2c052ee3aa7408ec67b7fc3e897c761f00d7cdf9d2087794e3b49ad5b68e8.png?w=1200&format=webp&q=90 1200w, https://cms.railway.com/media/eab2c052ee3aa7408ec67b7fc3e897c761f00d7cdf9d2087794e3b49ad5b68e8.png?w=1440&format=webp&q=90 1440w, https://cms.railway.com/media/eab2c052ee3aa7408ec67b7fc3e897c761f00d7cdf9d2087794e3b49ad5b68e8.png?w=1600&format=webp&q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="The dataplane network is a CLOS topology with redundant switches and redundant links" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">The dataplane network is a CLOS topology with redundant switches and redundant links</figcaption></figure>
|
||
<p class="mb-4 text-gray-800">For starters, the dataplane has multiple redundant links and must tolerate link failure or maintenance of redundant network switches. This requires running a routing protocol between switches and servers, with the protocol needing to route around device or link failures. Typically this routing must be configured for every point-to-point link; but this scales poorly in large deployments as the config needs to be customized for each rack.</p>
|
||
<p class="mb-4 text-gray-800">Unlike typical BGP where the BGP peer relationship must be defined between two configured IPv4 addresses for each router-router or router-server link, BGP unnumbered allows the use of autogenerated IPv6 link-local addresses as next-hops and peer-addresses for IPv4 and IPv6 BGP routing.</p>
|
||
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/67d14a503f1c4f843265de61cd7c8718bcec1a6dfa1a670bade71665637c6fc1.png?w=768&format=webp&q=90" srcSet="https://cms.railway.com/media/67d14a503f1c4f843265de61cd7c8718bcec1a6dfa1a670bade71665637c6fc1.png?w=320&format=webp&q=90 320w, https://cms.railway.com/media/67d14a503f1c4f843265de61cd7c8718bcec1a6dfa1a670bade71665637c6fc1.png?w=480&format=webp&q=90 480w, https://cms.railway.com/media/67d14a503f1c4f843265de61cd7c8718bcec1a6dfa1a670bade71665637c6fc1.png?w=640&format=webp&q=90 640w, https://cms.railway.com/media/67d14a503f1c4f843265de61cd7c8718bcec1a6dfa1a670bade71665637c6fc1.png?w=768&format=webp&q=90 768w, https://cms.railway.com/media/67d14a503f1c4f843265de61cd7c8718bcec1a6dfa1a670bade71665637c6fc1.png?w=960&format=webp&q=90 960w, https://cms.railway.com/media/67d14a503f1c4f843265de61cd7c8718bcec1a6dfa1a670bade71665637c6fc1.png?w=1200&format=webp&q=90 1200w, https://cms.railway.com/media/67d14a503f1c4f843265de61cd7c8718bcec1a6dfa1a670bade71665637c6fc1.png?w=1440&format=webp&q=90 1440w, https://cms.railway.com/media/67d14a503f1c4f843265de61cd7c8718bcec1a6dfa1a670bade71665637c6fc1.png?w=1600&format=webp&q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="With FRR, BGP adjacencies can be declared on interfaces and FRR will pick the IPv6 Link-Local address of peer connected to that interface as its next-hop for routing" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">With FRR, BGP adjacencies can be declared on interfaces and FRR will pick the IPv6 Link-Local address of peer connected to that interface as its next-hop for routing</figcaption></figure>
|
||
<p class="mb-4 text-gray-800">This makes the routing setups on switches and servers uniform. Add all the interfaces connecting to a given type of device (eg: Top-of-Rack switch uplinks, server downlinks or Spine switch uplinks) as a BGP peer-group and configure BGP as normal; then the same config can be shipped to every equivalent device in the cluster.</p>
|
||
<p class="mb-4 text-gray-800">At Railway, we have 1 BGP config template per kind of device and roll these all via Ansible to all switches. We do not need to reconfigure any network gear as we scale a rack or cage.</p><div class="flex w-full p-4 my-8 rounded border border-transparent bg-blue-100"><div class="text-yellow-500">💡</div><div class="flex flex-col w-full"><div class="ml-4 text-foreground"><p class="mb-4 text-gray-800">Config updates will eventually be needed and the cleanest/easiest way to apply them is to update the on-disk configuration via Ansible and reboot the switch. Hot-reloads with FRR are complicated, the prevailing approach seems to be <a class="markdown-inline-link" href="https://github.com/FRRouting/frr/blob/master/tools/frr-reload.py" target="_blank" rel="noreferrer noopener">frr-reload.py</a> which diffs two textual configs and comes up with a list of CLI commands needed to reconcile them.</p>
|
||
<p class="mb-4 text-gray-800">Long-term we hope <a class="markdown-inline-link" href="https://docs.kernel.org/networking/switchdev.html" target="_blank" rel="noreferrer noopener">switchdev</a> and <a class="markdown-inline-link" href="https://www.danosproject.org/" target="_blank" rel="noreferrer noopener">DANOS</a> will get wider ASIC support so we can directly integrate with our control plane. Failing that, at larger scale, directly integrating with <a class="markdown-inline-link" href="https://www.opencompute.org/projects/sai" target="_blank" rel="noreferrer noopener">SAI</a> and going in a <a class="markdown-inline-link" href="https://github.com/facebook/fboss" target="_blank" rel="noreferrer noopener">FBoss</a>-esque direction seems inevitable.</p></div></div></div><figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/332f8320f29234dc83665dc868c74934d4d498576b28f982231316bc779e5210.png?w=768&format=webp&q=90" srcSet="https://cms.railway.com/media/332f8320f29234dc83665dc868c74934d4d498576b28f982231316bc779e5210.png?w=320&format=webp&q=90 320w, https://cms.railway.com/media/332f8320f29234dc83665dc868c74934d4d498576b28f982231316bc779e5210.png?w=480&format=webp&q=90 480w, https://cms.railway.com/media/332f8320f29234dc83665dc868c74934d4d498576b28f982231316bc779e5210.png?w=640&format=webp&q=90 640w, https://cms.railway.com/media/332f8320f29234dc83665dc868c74934d4d498576b28f982231316bc779e5210.png?w=768&format=webp&q=90 768w, https://cms.railway.com/media/332f8320f29234dc83665dc868c74934d4d498576b28f982231316bc779e5210.png?w=960&format=webp&q=90 960w, https://cms.railway.com/media/332f8320f29234dc83665dc868c74934d4d498576b28f982231316bc779e5210.png?w=1200&format=webp&q=90 1200w, https://cms.railway.com/media/332f8320f29234dc83665dc868c74934d4d498576b28f982231316bc779e5210.png?w=1440&format=webp&q=90 1440w, https://cms.railway.com/media/332f8320f29234dc83665dc868c74934d4d498576b28f982231316bc779e5210.png?w=1600&format=webp&q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="Every possible port that can connect to a Top-of-Rack switch is marked as such on a Spine switch, regardless if those Racks are populated or not. This makes expansion plug-and-play." class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">Every possible port that can connect to a Top-of-Rack switch is marked as such on a Spine switch, regardless if those Racks are populated or not. This makes expansion plug-and-play.</figcaption></figure>
|
||
<p class="mb-4 text-gray-800">With the switch fabric configured in this way, and by running FRR with this same configuration down to servers, we build a full L3 network with ECMP across redundant links.</p><div class="flex w-full p-4 my-8 rounded border border-transparent bg-blue-100"><div class="text-yellow-500">💡</div><div class="flex flex-col w-full"><div class="ml-4 text-foreground"><p class="mb-4 text-gray-800">In addition to BGP unnumbered, this L3 fabric also requires <code class="inline-code">bgp bestpath as-path multipath-relax</code> and specific AS numbering to avoid unintended consequences from BGP loop detection. FRR provides a set of “datacenter” defaults that adjust timing for fast eBGP convergence.</p></div></div></div><p class="mb-4 text-gray-800">When we want to add a new IP to a host on the network, we have a small agent insert a route into the Linux kernel routing table. A preconfigured FRR daemon picks this up and then propagates it through the rest of the network - as long as the routed prefix is within one of the subnets assigned to the site.</p>
|
||
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/a2bcbc576948277342fed5d794d4d5389a6d1921bd25f44e83a42c285c9236c3.png?w=768&format=webp&q=90" srcSet="https://cms.railway.com/media/a2bcbc576948277342fed5d794d4d5389a6d1921bd25f44e83a42c285c9236c3.png?w=320&format=webp&q=90 320w, https://cms.railway.com/media/a2bcbc576948277342fed5d794d4d5389a6d1921bd25f44e83a42c285c9236c3.png?w=480&format=webp&q=90 480w, https://cms.railway.com/media/a2bcbc576948277342fed5d794d4d5389a6d1921bd25f44e83a42c285c9236c3.png?w=640&format=webp&q=90 640w, https://cms.railway.com/media/a2bcbc576948277342fed5d794d4d5389a6d1921bd25f44e83a42c285c9236c3.png?w=768&format=webp&q=90 768w, https://cms.railway.com/media/a2bcbc576948277342fed5d794d4d5389a6d1921bd25f44e83a42c285c9236c3.png?w=960&format=webp&q=90 960w, https://cms.railway.com/media/a2bcbc576948277342fed5d794d4d5389a6d1921bd25f44e83a42c285c9236c3.png?w=1200&format=webp&q=90 1200w, https://cms.railway.com/media/a2bcbc576948277342fed5d794d4d5389a6d1921bd25f44e83a42c285c9236c3.png?w=1440&format=webp&q=90 1440w, https://cms.railway.com/media/a2bcbc576948277342fed5d794d4d5389a6d1921bd25f44e83a42c285c9236c3.png?w=1600&format=webp&q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="IPv4 routes with IPv6 nexthops. These IPv6 addresses are Link-Local addresses discovered via IPv6 Neighbor Discovery" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">IPv4 routes with IPv6 nexthops. These IPv6 addresses are Link-Local addresses discovered via IPv6 Neighbor Discovery</figcaption></figure>
|
||
<a class="relative no-underline" href="/p/data-center-build-part-two#building-software-to-run-hardware-to-run-software"><span id="building-software-to-run-hardware-to-run-software" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-16 mb-8">Building Software to Run Hardware to Run Software</h2></a>
|
||
<p class="mb-4 text-gray-800">Building <a class="markdown-inline-link" href="https://docs.railway.com/platform/railway-metal" target="_blank" rel="noreferrer noopener">Railway Metal</a>, we’re more conscious than ever that we need to invest in tooling to enable us to deliver the best Metal experience we can build. We’re finding off-the-shelf solutions to be lacking or outdated in various ways, and the tooling we’re building for Railway Metal is proving an entire software vertical in itself.</p>
|
||
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/e361ce4467b97385c5d7d88166d258b51a3d8eef6f88cd07dce594d7e3f048e7.png?w=768&format=webp&q=90" srcSet="https://cms.railway.com/media/e361ce4467b97385c5d7d88166d258b51a3d8eef6f88cd07dce594d7e3f048e7.png?w=320&format=webp&q=90 320w, https://cms.railway.com/media/e361ce4467b97385c5d7d88166d258b51a3d8eef6f88cd07dce594d7e3f048e7.png?w=480&format=webp&q=90 480w, https://cms.railway.com/media/e361ce4467b97385c5d7d88166d258b51a3d8eef6f88cd07dce594d7e3f048e7.png?w=640&format=webp&q=90 640w, https://cms.railway.com/media/e361ce4467b97385c5d7d88166d258b51a3d8eef6f88cd07dce594d7e3f048e7.png?w=768&format=webp&q=90 768w, https://cms.railway.com/media/e361ce4467b97385c5d7d88166d258b51a3d8eef6f88cd07dce594d7e3f048e7.png?w=960&format=webp&q=90 960w, https://cms.railway.com/media/e361ce4467b97385c5d7d88166d258b51a3d8eef6f88cd07dce594d7e3f048e7.png?w=1200&format=webp&q=90 1200w, https://cms.railway.com/media/e361ce4467b97385c5d7d88166d258b51a3d8eef6f88cd07dce594d7e3f048e7.png?w=1440&format=webp&q=90 1440w, https://cms.railway.com/media/e361ce4467b97385c5d7d88166d258b51a3d8eef6f88cd07dce594d7e3f048e7.png?w=1600&format=webp&q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="MetalCP is branching into Network Automation and is already our in-house RANCID/Oxidized replacement" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">MetalCP is branching into Network Automation and is already our in-house RANCID/Oxidized replacement</figcaption></figure>
|
||
<p class="mb-4 text-gray-800">We’ll continue to write more about our exploits, but in the interim - if you find any of this interesting or fun, we’re hiring!</p>
|
||
<p class="mb-4 text-gray-800">Pop on over to <a class="markdown-inline-link" href="https://railway.com/careers" target="_blank" rel="noreferrer noopener">railway.com/careers</a> and check out our open roles.</p></div></section></article><div class="max-w-6xl mx-auto"><div><header class="flex items-center justify-between mb-8"><h3 class="text-gray-500 text-h3">Continue Reading...</h3><a class="text-pink-500 hover:underline" href="/engineering">View All <!-- -->Engineering<!-- --> -></a></header><div class="grid grid-cols-1 md:grid-cols-2 gap-8"><a class="flex flex-col bg-secondaryBg p-6 rounded-lg hover:bg-gray-100 group" href="/p/dev-new"><div class="flex gap-2"><div class="post-category-pill font-medium max-w-max text-xs px-2 py-[3px] rounded-[4px] tracking-[0.06em] uppercase" data-category-tone="plum">Engineering</div></div><div class="flex-grow"><header class="font-medium font-serif text-lg mt-2 mb-1">Your Cloud should come with an app builder </header><p class="text-base text-gray-800 line-clamp-2">Announcing: dev.new, an agent that can build, preview, test, and deploy an app from the same place. We tested the whole loop by asking it to ship a real-time multiplayer FPS game that runs in the browser</p></div><div class="flex items-center gap-3 mt-6"><div class="flex items-center"><img src="https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg" alt="Avatar of Mahmoud Abdelwahab" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/></div><span class="text-sm text-gray-600">Mahmoud Abdelwahab</span><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><span class="text-sm text-gray-600">Aug 13, 2026</span></div></a><a class="flex flex-col bg-secondaryBg p-6 rounded-lg hover:bg-gray-100 group" href="/p/feature-flags"><div class="flex gap-2"><div class="post-category-pill font-medium max-w-max text-xs px-2 py-[3px] rounded-[4px] tracking-[0.06em] uppercase" data-category-tone="plum">Engineering</div></div><div class="flex-grow"><header class="font-medium font-serif text-lg mt-2 mb-1">Roll it out, roll it back, never redeploy</header><p class="text-base text-gray-800 line-clamp-2">Merge now, release when you're ready: turn a feature on for your team, then 10% of users, then everyone. Feature flags are now built into every Railway project, and your agents can run the rollout from the CLI, SDK, and MCP. No third-party service required.</p></div><div class="flex items-center gap-3 mt-6"><div class="flex items-center"><img src="https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg" alt="Avatar of Victor Ramirez" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/><img src="https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg" alt="Avatar of Mahmoud Abdelwahab" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:-8px" loading="lazy" decoding="async" width="24" height="24"/></div><span class="text-sm text-gray-600">Victor Ramirez & Mahmoud Abdelwahab</span><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><span class="text-sm text-gray-600">Jul 17, 2026</span></div></a></div></div><section class="post-bottom-cta my-16 border rounded-[16px] px-6 py-16 sm:px-12 sm:py-24 flex flex-col items-center justify-center text-center" aria-labelledby="post-bottom-cta-title"><h3 id="post-bottom-cta-title" class="text-h2 font-serif font-medium">Ready to get started?</h3><p class="mt-4 text-base sm:text-lg text-gray-600">Join millions of developers deploying applications on Railway</p><div class="mt-8 flex w-full flex-col items-stretch justify-center gap-3 sm:w-auto sm:flex-row"><a class="post-bottom-cta-primary inline-flex min-h-[48px] items-center justify-center rounded-[8px] px-6 py-3 text-base font-medium no-underline transition-colors duration-100" href="https://railway.com" target="_blank" rel="noreferrer noopener">Deploy a new project</a><a class="post-bottom-cta-secondary inline-flex min-h-[48px] items-center justify-center rounded-[8px] border px-6 py-3 text-base font-medium no-underline transition-colors duration-100" href="https://railway.com/enterprise" target="_blank" rel="noreferrer noopener">Book a demo</a></div></section></div></div></div><footer class="px-5 md:px-8 py-12 border-t border-gray-100"><div class="grid grid-cols-2 sm:grid-cols-12 gap-8 sm:gap-4 lg:gap-16 max-w-6xl mx-auto"><div class="hidden sm:flex col-span-3 lg:col-span-4 flex-col justify-between"><a class="underline" href="https://railway.com" target="_blank" rel="noreferrer noopener"><svg data-v-423bf9ae="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 59.99945071644884 60" class="logo w-8"><g data-v-423bf9ae="" id="6f639fe5-02e1-4640-a1fb-f4b22b64e5ef" transform="matrix(0.5999641134746578,0,0,0.5999641134746578,-437.5538206497148,-266.68705118956507)" stroke="none"><path d="M729.872 487.327a50.86 50.86 0 0 0-.464 5.033h75.879c-.265-.518-.621-.985-.98-1.442-12.972-16.769-19.95-15.315-29.932-15.741-3.328-.137-5.585-.192-18.832-.192-7.09 0-14.798.018-22.304.038.737-1.746 1.6-3.419 2.525-5.061-2.885 5.106-4.865 10.771-5.805 16.789l.891-4.397c.007-.031.018-.063.024-.094h38.883v5.067h-39.885zM805.885 497.432h-76.438c.08 1.352.206 2.686.388 4.002h70.571c3.146 0 4.907-1.786 5.479-4.002zM733.851 515.257a52.226 52.226 0 0 1-1.98-4.997c6.608 19.89 25.328 34.251 47.433 34.251 20.205 0 37.566-12.007 45.452-29.254h-90.905zM729.38 492.915c-.018.531-.08 1.055-.08 1.589 0 .538.063 1.059.08 1.59v-3.179zM824.77 515.229z"></path><path d="M779.303 444.505c-18.682 0-34.939 10.265-43.524 25.439 6.709-.014 19.775-.022 19.775-.022h.003v-.005c15.444 0 16.018.069 19.035.195l1.868.069c6.507.217 14.505.916 20.798 5.68 3.416 2.584 8.348 8.287 11.288 12.35 2.718 3.758 3.5 8.078 1.652 12.217-1.701 3.804-5.361 6.073-9.793 6.073H730.85l-.884-4.201c.426 2.707 1.037 5.344 1.879 7.886h94.914a49.863 49.863 0 0 0 2.546-15.682c.001-27.611-22.386-49.999-50.002-49.999z"></path></g></svg></a><div class="text-xs font-medium text-gray-600 w-full">Copyright © <!-- -->2026<!-- --> Railway Corp. <br/>All rights reserved.</div></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Product</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/changelog" target="_blank" rel="noreferrer noopener">Changelog</a></li><li><a class="hover:text-foreground" href="https://railway.com/pricing" target="_blank" rel="noreferrer noopener">Pricing</a></li><li><a class="hover:text-foreground" href="https://railway.com/deploy" target="_blank" rel="noreferrer noopener">Templates</a></li><li><a class="hover:text-foreground" href="https://station.railway.com/feedback" target="_blank" rel="noreferrer noopener">Feedback</a></li><li><a class="hover:text-foreground" href="https://railway.com/open-source-kickback" target="_blank" rel="noreferrer noopener">OSS Kickback</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Company</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/about" target="_blank" rel="noreferrer noopener">About</a></li><li><a class="hover:text-foreground" href="https://railway.com/careers" target="_blank" rel="noreferrer noopener">Careers</a></li><li><a class="hover:text-foreground" href="/">Blog</a></li><li><a class="hover:text-foreground" href="https://shop.railway.com" target="_blank" rel="noreferrer noopener">Shop</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Contact</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://discord.gg/railway" target="_blank" rel="noreferrer noopener">Discord</a></li><li><a class="hover:text-foreground" href="https://twitter.com/railway" target="_blank" rel="noreferrer noopener">Twitter</a></li><li><a class="hover:text-foreground" href="https://github.com/railwayapp" target="_blank" rel="noreferrer noopener">GitHub</a></li><li><a class="hover:text-foreground" href="mailto:contact@railway.com">Email</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Legal</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/legal/acceptable-use" target="_blank" rel="noreferrer noopener">Acceptable Use</a></li><li><a class="hover:text-foreground" href="https://railway.com/legal/privacy" target="_blank" rel="noreferrer noopener">Privacy Policy</a></li><li><a class="hover:text-foreground" href="https://railway.com/legal/terms" target="_blank" rel="noreferrer noopener">Terms of Service</a></li></ul></div></div></footer></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"page":{"authors":[{"avatar":{"alt":"Avatar of Charith Amarasinghe","height":null,"id":"1836","mimeType":"image/png","url":"https://cms.railway.com/media/452039afd9d5078059203ebfb4028e92717d03b94e10832c814d2ca23c5bce39.png","width":null},"avatarUrl":"https://cms.railway.com/media/452039afd9d5078059203ebfb4028e92717d03b94e10832c814d2ca23c5bce39.png","githubUrl":null,"id":"13","name":"Charith Amarasinghe","slug":"charith-amarasinghe","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":"We’ve all gotten used to clicking a button and getting a Linux machine running in the cloud. But when you’re building your own cloud, you’ve got to build the button first. \n\nLately we’ve been [writing](https://blog.railway.com/p/data-center-build-part-one) about building out our [Metal infrastructure](https://blog.railway.com/p/launch-week-02-welcome) one rack at a time. \n\nIn our [last blog](https://blog.railway.com/p/data-center-build-part-one), we spoke about the trials of building out the physical infrastructure. In this episode, we talk about how we operationalize the hardware once it’s installed.\n\n# Sorting your LEGO pieces\n\nYou’ve built your dream server with dual-redundant NICs and multiple redundant NVMe drives for resilience. You’ve ordered 100 units and got them all racked up and wired to your detailed diagrams. You go to the DC with your USB stick and reboot into your favorite Linux distro’s installer only to be greeted by the “Choose your Network Interface” screen with a dozen or so incomprehensible interface names. \n\nHerein lies our first hurdle — how do you map the physical arrangement of hardware to what your operating system sees? \n\n\n\n\u003caside\u003e\n💡\n\nWhen buying build-to-order servers, it’s essential to include instructions specifying exactly where each Network Card or NVMe Drive should get installed. Otherwise you might end up with multiple different configurations across different orders.\n\n\u003c/aside\u003e\n\nIt helps first to take a step back and discuss how Linux names devices.\n\nWhen a host boots Linux, the OS enumerates the attached hardware. Most commonly, devices are attached to the PCIe bus and Linux begins enumerating these according to the hierarchical structure of the bus. When Linux encounters a device during this traversal, the [udev](https://en.wikipedia.org/wiki/Udev) daemon will get an event and associate a number of identifiers with the device - it’ll then use these identifiers to formulate a name which it then assigns to the device nodes it creates in `/dev` and elsewhere.\n\nThe consequence of this approach is that device names can be very unstable, especially if the hardware layout changes between boots or if the enumeration order is non-deterministic. If you used Linux in the olden days, you’d know the pain of plugging in a new PCIe card and booting only to figure out that your networking broke. Despite the many critiques that could be leveled against SystemD, it does succeed in [addressing these problems for network interfaces since v197](https://github.com/systemd/systemd/blob/main/docs/PREDICTABLE_INTERFACE_NAMES.md). But storage device naming is still a crapshoot and better achieved by device serial number.\n\nOur approach to addressing this unpredictability is to lean on Redfish - a HTTP API for Board Management Controllers (BMCs) attached to server motherboards. Redfish APIs can enumerate the hardware on a board, detailing PCIe cards, NVMe drives, their serial numbers and/or MAC addresses and their physical locations.\n\n\n\nOur very first step once a rack is installed is to build a CSV of identifiers for the equipment in the rack - hostname, BMC MAC address, BMC password, and a few other details. We then push this data via gRPC to an internal control plane called MetalCP. \n\nMetalCP runs a Temporal worker which implements a Host Import workflow. For each server, we then kick off a workflow that runs through the following steps:\n\n1. Match the device to its representation in our internal DCIM tool (Railyard) \n2. Connect to the datacenter's management network via Tailscale\n3. Connect to the management router at the datacenter and identify the DHCP lease assigned to the BMC (via its MAC)\n4. Connect to the BMC of the server via this IP and scrape all available data\n5. Create an internal Protobuf representation of the hardware layout\n6. Create static DHCP leases for the BMC and for the management NIC on the host using discovered MAC addresses from the scrape\n7. Update a DB with all the details about the server\n\nAn import workflow takes less that a minute to complete in most cases and Temporal ensures recovery from any transient failures.\n\n\n\nThe database record that is stored by a host workflow contains all the information you could want about a server. We generate:\n\n- A list of Network Interface Cards, their Physical Location (Slot), and MAC addresses for each Port\n- A list of NVMe Drives, their Serial Numbers, Model Numbers, and Physical Slot IDs\n- System stats such as CPU core counts, RAM size, and hardware identifiers\n\nWe then match this hardware specification against a list of known configurations. These configurations encode details such as network interface names assigned to specific PCIe slots and NVMe drive bay identifiers. A hardware configuration is as simple as a set of conditionals in Golang that match the key distinguishing factors of a specific type of server, and a config object containing stable interface and drive names.\n\n\n\nA custom plugin exposes this Hardware Config object to Ansible, allowing us to reference NVMe disks and network interface names with Jinja template expressions. For example, a NVMe drive in Bay 0 can be uniquely addressed as `/dev/disks/by-id/nvme-{{ drive_bays.DiskBay0.device_model }}_{{ drive_bays.DiskBay0.serial_number }}` .\n\nThis approach lets us build config in any shape we want without leaving anything to chance. The import workflow also flags faults in the hardware; if a server is not reporting a NIC or a DIMM of RAM, the workflow will fail since the hardware won’t match a known configuration. We’ve thus far identified servers with faulty RAM and servers with NICs installed in the wrong slots through this mechanism.\n\n\n\nConfiguration is one step, but getting at Ansible still needs an OS to get installed. So how do we one-click ourselves out of installing Linux in the first place? The answer involves a pinch of AI 🪄.\n\n# Who needs webhooks when we’ve got Claude\n\nWhen we first started provisioning servers, we did it manually with 20 Web KVM tabs in Chrome and manually interacting with debian-installer. Over time we’ve evolved to use less and less human intervention.\n\nThe Debian Installer can network boot from PXE, and [Pixiecore from Dave Anderson](https://github.com/danderson/netboot/tree/main/pixiecore) can wrap all the PXE complexity in a few HTTP calls. We use MetalCP as a backend to Pixicore and return a simple JSON payload describing the netboot kernel, initramfs, and kernel command-line. Debian can accept a pre-seed file over HTTP if networking is configured. \n\nWe use our knowledge of the PXE booting servers MAC address, plus the system info we’ve scraped from Redfish, to create a kernel command-line and preseed file tailored to the booting machine. \n\nThese are all exposed as HTTP APIs proxied by Pixiecore to the PXE booting machine.\n\n\n\nGetting a host to a PXE bootable state requires us to reboot the server, but we don’t want this reboot action to happen on a server that may be running user code. To achieve this, we implement a logical state machine for each host in the provisioning process and orchestrate the OS install via another Temporal workflow. \n\nBut how does the Workflow know which state the server is in during the install? Redfish APIs tell us that it’s powered on, but little else. \n\nSince it’s 2025, we just ask Claude.\n\n\n\nSupermicro introduced a CaptureScreen OEM API in their Redfish 1.14 release; with this API we can obtain a near real-time image of the server screen. With a basic prompt to Claude, we can then get a JSON payload that describes the state of the server at this point. Combining this into a Temporal workflow - alongside the PXE boot automation above - we can achieve an OS install and provision with one gRPC API call.\n\n\n\nThere are probably more effective methods of achieving the same, but it costs us less than a dollar to provision 50 servers using Claude to screen-scrape every minute during the install.\n\nNow that an OS is installed, some duct-tape and Ansible will allow us to get some basic software running on the machine. However, bringing up networking is something else that’s typically an annoyance.\n\n# Low Config Networking with BGP Unnumbered\n\nAll the solutions we’ve discussed thus far have relied on a Management (or Out of Band) network. This is a dedicated Gigabit Ethernet network that links a management NIC, BMCs and other support infrastructure inside the cage. This network isn’t built to scale as it instead relies on being limited to a few hundred hosts at most and uses off-the-shelf routers, DHCP and VLANs for isolation and operation with low fault tolerance. The network that carries user traffic, the dataplane network as we term it, has very different requirements.\n\n\n\nFor starters, the dataplane has multiple redundant links and must tolerate link failure or maintenance of redundant network switches. This requires running a routing protocol between switches and servers, with the protocol needing to route around device or link failures. Typically this routing must be configured for every point-to-point link; but this scales poorly in large deployments as the config needs to be customized for each rack.\n\nUnlike typical BGP where the BGP peer relationship must be defined between two configured IPv4 addresses for each router-router or router-server link, BGP unnumbered allows the use of autogenerated IPv6 link-local addresses as next-hops and peer-addresses for IPv4 and IPv6 BGP routing.\n\n\n\nThis makes the routing setups on switches and servers uniform. Add all the interfaces connecting to a given type of device (eg: Top-of-Rack switch uplinks, server downlinks or Spine switch uplinks) as a BGP peer-group and configure BGP as normal; then the same config can be shipped to every equivalent device in the cluster. \n\nAt Railway, we have 1 BGP config template per kind of device and roll these all via Ansible to all switches. We do not need to reconfigure any network gear as we scale a rack or cage.\n\n\u003caside\u003e\n💡\n\nConfig updates will eventually be needed and the cleanest/easiest way to apply them is to update the on-disk configuration via Ansible and reboot the switch. Hot-reloads with FRR are complicated, the prevailing approach seems to be [frr-reload.py](https://github.com/FRRouting/frr/blob/master/tools/frr-reload.py) which diffs two textual configs and comes up with a list of CLI commands needed to reconcile them. \n\nLong-term we hope [switchdev](https://docs.kernel.org/networking/switchdev.html) and [DANOS](https://www.danosproject.org/) will get wider ASIC support so we can directly integrate with our control plane. Failing that, at larger scale, directly integrating with [SAI](https://www.opencompute.org/projects/sai) and going in a [FBoss](https://github.com/facebook/fboss)-esque direction seems inevitable.\n\n\u003c/aside\u003e\n\n\n\nWith the switch fabric configured in this way, and by running FRR with this same configuration down to servers, we build a full L3 network with ECMP across redundant links.\n\n\u003caside\u003e\n💡\n\nIn addition to BGP unnumbered, this L3 fabric also requires `bgp bestpath as-path multipath-relax` and specific AS numbering to avoid unintended consequences from BGP loop detection. FRR provides a set of “datacenter” defaults that adjust timing for fast eBGP convergence.\n\n\u003c/aside\u003e\n\nWhen we want to add a new IP to a host on the network, we have a small agent insert a route into the Linux kernel routing table. A preconfigured FRR daemon picks this up and then propagates it through the rest of the network - as long as the routed prefix is within one of the subnets assigned to the site.\n\n\n\n# Building Software to Run Hardware to Run Software\n\nBuilding [Railway Metal](https://docs.railway.com/platform/railway-metal), we’re more conscious than ever that we need to invest in tooling to enable us to deliver the best Metal experience we can build. We’re finding off-the-shelf solutions to be lacking or outdated in various ways, and the tooling we’re building for Railway Metal is proving an entire software vertical in itself.\n\n\n\nWe’ll continue to write more about our exploits, but in the interim - if you find any of this interesting or fun, we’re hiring! \n\nPop on over to [railway.com/careers](https://railway.com/careers) and check out our open roles.","createdAt":"2026-06-09T12:25:06.575Z","description":"We’ve all gotten used to clicking a button and getting a Linux machine running in the cloud. But when you’re building your own cloud, you’ve got to build the button first.","externalAuthor":false,"featured":false,"featuredImage":{"alt":"Zero-Touch Bare Metal at Scale featured image","height":null,"id":"796","mimeType":"image/png","url":"https://cms.railway.com/media/590257b1336d9568268b1339d21ffbccbc424b118d6e32bba29ec21bcf83945e.png","width":null},"id":"202","publishedAt":"2025-03-21T00:00:00.000Z","seoDescription":"We’re used to clicking a button and getting a Linux machine in the cloud. When you’re building your own cloud, you’ve got to build the button first.","seoTitle":"Zero-Touch Bare Metal at Scale","slug":"data-center-build-part-two","socialImage":{"alt":"Zero-Touch Bare Metal at Scale social image","height":null,"id":"793","mimeType":"image/png","url":"https://cms.railway.com/media/11072737e2dc057b4e9bea2042e030aa7f20352ee34ae755a197c5c00fcad880.png","width":null},"title":"Zero-Touch Bare Metal at Scale","updatedAt":"2026-06-23T17:35:28.656Z"},"relatedPosts":[{"authors":[{"avatar":{"alt":"Mahmoud Abdelwahab headshot","height":460,"id":"1846","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","width":460},"avatarUrl":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","githubUrl":null,"id":"4","name":"Mahmoud Abdelwahab","slug":"mahmoud-abdelwahab","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":null,"createdAt":"2026-08-06T08:07:06.228Z","description":"Announcing: dev.new, an agent that can build, preview, test, and deploy an app from the same place. We tested the whole loop by asking it to ship a real-time multiplayer FPS game that runs in the browser","externalAuthor":false,"featured":true,"featuredImage":{"alt":"dev.new featured image","height":1440,"id":"1983","mimeType":"image/png","url":"https://cms.railway.com/media/dev.new-featured-image-4.png","width":3264},"id":"289","publishedAt":"2026-08-13T08:07:00.000Z","seoDescription":null,"seoTitle":null,"slug":"dev-new","socialImage":{"alt":"dev.new featured image 1","height":720,"id":"1976","mimeType":"image/png","url":"https://cms.railway.com/media/dev.new-featured-image-3.png","width":1632},"title":"Your Cloud should come with an app builder ","updatedAt":"2026-08-14T20:19:27.322Z"},{"authors":[{"avatar":{"alt":"Avatar of Victor Ramirez","height":null,"id":"1855","mimeType":"image/jpeg","url":"https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg","width":null},"avatarUrl":"https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg","githubUrl":null,"id":"34","name":"Victor Ramirez","slug":"victor-ramirez","title":null},{"avatar":{"alt":"Mahmoud Abdelwahab headshot","height":460,"id":"1846","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","width":460},"avatarUrl":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","githubUrl":null,"id":"4","name":"Mahmoud Abdelwahab","slug":"mahmoud-abdelwahab","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":null,"createdAt":"2026-07-15T22:26:48.036Z","description":"Merge now, release when you're ready: turn a feature on for your team, then 10% of users, then everyone. Feature flags are now built into every Railway project, and your agents can run the rollout from the CLI, SDK, and MCP. No third-party service required.","externalAuthor":false,"featured":false,"featuredImage":{"alt":"feature flags featured image","height":1440,"id":"1968","mimeType":"image/png","url":"https://cms.railway.com/media/feature-flags-featured-image.png","width":3264},"id":"278","publishedAt":"2026-07-17T07:00:00.000Z","seoDescription":null,"seoTitle":null,"slug":"feature-flags","socialImage":null,"title":"Roll it out, roll it back, never redeploy","updatedAt":"2026-08-13T16:53:55.126Z"}]},"__N_SSG":true},"page":"/p/[slug]","query":{"slug":"data-center-build-part-two"},"buildId":"nFX7R3M0MdLMAf-Ufh9qa","isFallback":false,"isExperimentalCompile":false,"gsp":true,"scriptLoader":[]}</script></body></html> |