Files
nexus/sreweekly/articles/464/01-so-you-want-to-build-your-own-data-center.html
2026-09-12 17:23:01 +08:00

287 lines
81 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8" data-next-head=""/><meta name="viewport" content="width=device-width" data-next-head=""/><meta name="twitter:card" content="summary_large_image" data-next-head=""/><meta name="twitter:site" content="@Railway" data-next-head=""/><meta name="twitter:creator" content="@Railway" data-next-head=""/><meta property="og:image:type" content="image/png" data-next-head=""/><meta property="og:image:width" content="1200" data-next-head=""/><meta property="og:image:height" content="630" data-next-head=""/><meta property="og:site_name" content="Railway Blog" data-next-head=""/><title data-next-head="">So You Want to Build Your Own Data Center</title><meta name="robots" content="index,follow" data-next-head=""/><meta name="description" content="When it comes to infrastructure engineering, building a data center is probably closer to building a house than to deploying a Terraform stack." data-next-head=""/><meta property="og:title" content="So You Want to Build Your Own Data Center" data-next-head=""/><meta property="og:description" content="When it comes to infrastructure engineering, building a data center is probably closer to building a house than to deploying a Terraform stack." data-next-head=""/><meta property="og:url" content="https://blog.railway.com/p/data-center-build-part-one" data-next-head=""/><meta property="og:type" content="article" data-next-head=""/><meta property="article:published_time" content="2025-01-17T00:00:00.000Z" data-next-head=""/><meta property="article:modified_time" content="2025-01-17T00:00:00.000Z" data-next-head=""/><meta property="article:author" content="Charith Amarasinghe" data-next-head=""/><meta property="article:section" content="Engineering" data-next-head=""/><meta property="og:image" content="https://cms.railway.com/media/d3356891ab8c76de1c7872dfb46c3ace7de5eb6b478d19e2f317d71b15710567.png?w=1200&amp;q=90" data-next-head=""/><link rel="canonical" href="https://blog.railway.com/p/data-center-build-part-one" data-next-head=""/><meta name="twitter:title" content="So You Want to Build Your Own Data Center" data-next-head=""/><meta name="twitter:description" content="When it comes to infrastructure engineering, building a data center is probably closer to building a house than to deploying a Terraform stack." data-next-head=""/><meta name="twitter:image" content="https://cms.railway.com/media/d3356891ab8c76de1c7872dfb46c3ace7de5eb6b478d19e2f317d71b15710567.png?w=1200&amp;q=90" data-next-head=""/><link rel="alternate" type="text/markdown" href="/p/data-center-build-part-one.md" data-next-head=""/><link rel="alternate" type="application/rss+xml" title="Railway Blog" href="https://blog.railway.com/rss.xml"/><link rel="preload" href="/_next/static/media/e4af272ccee01ff0-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/91c3caac73720166-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/3359c9acc30ef440-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/aff755c8b4bb3089-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/3988c35203e0830a-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/427e4a37d3642943-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/css/619a01ff31a99955.css" as="style"/><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"BlogPosting","headline":"So You Want to Build Your Own Data Center","description":"When it comes to infrastructure engineering, building a data center is probably closer to building a house than to deploying a Terraform stack.","image":["https://cms.railway.com/media/d3356891ab8c76de1c7872dfb46c3ace7de5eb6b478d19e2f317d71b15710567.png?w=1200&q=90"],"datePublished":"2025-01-17T00:00:00.000Z","dateModified":"2025-01-17T00:00:00.000Z","author":{"@type":"Person","name":"Charith Amarasinghe"},"publisher":{"@type":"Organization","name":"Railway","logo":{"@type":"ImageObject","url":"https://railway.com/brand/logo-dark.png"}},"mainEntityOfPage":{"@type":"WebPage","@id":"https://blog.railway.com/p/data-center-build-part-one"}}</script><script type="application/ld+json" data-next-head="">{"@context":"https://schema.org","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https://blog.railway.com"},{"@type":"ListItem","position":2,"name":"Engineering","item":"https://blog.railway.com/engineering"},{"@type":"ListItem","position":3,"name":"So You Want to Build Your Own Data Center","item":"https://blog.railway.com/p/data-center-build-part-one"}]}</script><style>:root {
--foreground: hsl(0, 0%, 100%);
--background: #181622;
--secondaryBg: hsl(250, 21%, 11%);
--blue-50: hsl(220, 55%, 10%);
--blue-100: hsl(220, 55%, 13%);
--blue-200: hsl(220, 62%, 25%);
--blue-300: hsl(220, 68%, 35%);
--blue-400: hsl(220, 72%, 45%);
--blue-500: hsl(220, 80%, 55%);
--blue-600: hsl(220, 80%, 65%);
--blue-700: hsl(220, 80%, 75%);
--blue-800: hsl(220, 80%, 85%);
--blue-900: hsl(220, 80%, 95%);
--blue-950: hsl(220, 55%, 97%);
--gray-50: hsl(248, 21%, 13%);
--gray-100: hsl(246, 18%, 15%);
--gray-200: hsl(246, 11%, 22%);
--gray-300: hsl(246, 8%, 35%);
--gray-400: hsl(246, 7%, 45%);
--gray-500: hsl(246, 6%, 55%);
--gray-600: hsl(246, 6%, 65%);
--gray-700: hsl(246, 6%, 78%);
--gray-800: hsl(246, 6%, 87%);
--gray-900: hsl(246, 6%, 95%);
--gray-950: hsl(246, 6%, 95%);
--red-50: hsl(1, 35%, 10%);
--red-100: hsl(1, 45%, 12%);
--red-200: hsl(1, 55%, 20%);
--red-300: hsl(1, 62%, 28%);
--red-400: hsl(1, 62%, 35%);
--red-500: hsl(1, 62%, 44%);
--red-600: hsl(1, 62%, 60%);
--red-700: hsl(1, 62%, 76%);
--red-800: hsl(1, 64%, 85%);
--red-900: hsl(1, 68%, 95%);
--red-950: hsl(1, 55%, 98%);
--pink-50: hsl(270, 40%, 16%);
--pink-100: hsl(270, 40%, 16%);
--pink-200: hsl(270, 45%, 24%);
--pink-300: hsl(270, 50%, 32%);
--pink-400: hsl(270, 55%, 43%);
--pink-500: hsl(270, 60%, 52%);
--pink-600: hsl(270, 70%, 65%);
--pink-700: hsl(270, 70%, 75%);
--pink-800: hsl(270, 70%, 85%);
--pink-900: hsl(270, 70%, 95%);
--pink-950: hsl(270, 70%, 95%);
--green-50: hsl(152, 15%, 10%);
--green-100: hsl(152, 26%, 11%);
--green-200: hsl(152, 32%, 16%);
--green-300: hsl(152, 38%, 24%);
--green-400: hsl(152, 38%, 34%);
--green-500: hsl(152, 38%, 42%);
--green-600: hsl(152, 38%, 60%);
--green-700: hsl(152, 38%, 70%);
--green-800: hsl(152, 38%, 80%);
--green-900: hsl(152, 38%, 91%);
--green-950: hsl(152, 40%, 97%);
--yellow-50: hsl(44, 95%, 12%);
--yellow-100: hsl(44, 95%, 12%);
--yellow-200: hsl(44, 96%, 24%);
--yellow-300: hsl(44, 95%, 36%);
--yellow-400: hsl(44, 95%, 48%);
--yellow-500: hsl(44, 95%, 60%);
--yellow-600: hsl(44, 95%, 69%);
--yellow-700: hsl(44, 95%, 78%);
--yellow-800: hsl(44, 95%, 86%);
--yellow-900: hsl(44, 95%, 95%);
--yellow-950: hsl(44, 95%, 95%);
}
.light {
--foreground: hsl(250, 24%, 9%);
--background: #F1F0EF;
--secondaryBg: hsl(0, 0%, 98%);
--blue-50: hsl(220, 55%, 97%);
--blue-100: hsl(220, 80%, 95%);
--blue-200: hsl(220, 80%, 85%);
--blue-300: hsl(220, 80%, 75%);
--blue-400: hsl(220, 80%, 65%);
--blue-500: hsl(220, 80%, 55%);
--blue-600: hsl(220, 72%, 45%);
--blue-700: hsl(220, 68%, 35%);
--blue-800: hsl(220, 62%, 25%);
--blue-900: hsl(220, 55%, 13%);
--blue-950: hsl(220, 55%, 10%);
--gray-50: hsl(246, 6%, 95%);
--gray-100: hsl(246, 6%, 95%);
--gray-200: hsl(246, 6%, 87%);
--gray-300: hsl(246, 6%, 78%);
--gray-400: hsl(246, 6%, 65%);
--gray-500: hsl(246, 6%, 55%);
--gray-600: hsl(246, 7%, 45%);
--gray-700: hsl(246, 8%, 35%);
--gray-800: hsl(246, 11%, 22%);
--gray-900: hsl(246, 18%, 15%);
--gray-950: hsl(248, 21%, 13%);
--red-50: hsl(1, 55%, 98%);
--red-100: hsl(1, 68%, 95%);
--red-200: hsl(1, 64%, 85%);
--red-300: hsl(1, 62%, 76%);
--red-400: hsl(1, 62%, 60%);
--red-500: hsl(1, 62%, 44%);
--red-600: hsl(1, 62%, 35%);
--red-700: hsl(1, 62%, 28%);
--red-800: hsl(1, 55%, 20%);
--red-900: hsl(1, 45%, 12%);
--red-950: hsl(1, 35%, 10%);
--pink-50: hsl(270, 70%, 95%);
--pink-100: hsl(270, 70%, 95%);
--pink-200: hsl(270, 70%, 85%);
--pink-300: hsl(270, 70%, 75%);
--pink-400: hsl(270, 70%, 65%);
--pink-500: hsl(270, 60%, 52%);
--pink-600: hsl(270, 55%, 43%);
--pink-700: hsl(270, 50%, 32%);
--pink-800: hsl(270, 45%, 24%);
--pink-900: hsl(270, 40%, 16%);
--pink-950: hsl(270, 40%, 16%);
--green-50: hsl(152, 40%, 97%);
--green-100: hsl(152, 38%, 91%);
--green-200: hsl(152, 38%, 80%);
--green-300: hsl(152, 38%, 70%);
--green-400: hsl(152, 38%, 60%);
--green-500: hsl(152, 38%, 42%);
--green-600: hsl(152, 38%, 34%);
--green-700: hsl(152, 38%, 24%);
--green-800: hsl(152, 32%, 16%);
--green-900: hsl(152, 26%, 11%);
--green-950: hsl(152, 15%, 10%);
--yellow-50: hsl(44, 95%, 95%);
--yellow-100: hsl(44, 95%, 95%);
--yellow-200: hsl(44, 95%, 86%);
--yellow-300: hsl(44, 95%, 78%);
--yellow-400: hsl(44, 95%, 69%);
--yellow-500: hsl(44, 95%, 60%);
--yellow-600: hsl(44, 95%, 48%);
--yellow-700: hsl(44, 95%, 36%);
--yellow-800: hsl(44, 96%, 24%);
--yellow-900: hsl(44, 95%, 12%);
--yellow-950: hsl(44, 95%, 12%);
}
.dark {
--foreground: hsl(0, 0%, 100%);
--background: #181622;
--secondaryBg: hsl(250, 21%, 11%);
--blue-50: hsl(220, 55%, 10%);
--blue-100: hsl(220, 55%, 13%);
--blue-200: hsl(220, 62%, 25%);
--blue-300: hsl(220, 68%, 35%);
--blue-400: hsl(220, 72%, 45%);
--blue-500: hsl(220, 80%, 55%);
--blue-600: hsl(220, 80%, 65%);
--blue-700: hsl(220, 80%, 75%);
--blue-800: hsl(220, 80%, 85%);
--blue-900: hsl(220, 80%, 95%);
--blue-950: hsl(220, 55%, 97%);
--gray-50: hsl(248, 21%, 13%);
--gray-100: hsl(246, 18%, 15%);
--gray-200: hsl(246, 11%, 22%);
--gray-300: hsl(246, 8%, 35%);
--gray-400: hsl(246, 7%, 45%);
--gray-500: hsl(246, 6%, 55%);
--gray-600: hsl(246, 6%, 65%);
--gray-700: hsl(246, 6%, 78%);
--gray-800: hsl(246, 6%, 87%);
--gray-900: hsl(246, 6%, 95%);
--gray-950: hsl(246, 6%, 95%);
--red-50: hsl(1, 35%, 10%);
--red-100: hsl(1, 45%, 12%);
--red-200: hsl(1, 55%, 20%);
--red-300: hsl(1, 62%, 28%);
--red-400: hsl(1, 62%, 35%);
--red-500: hsl(1, 62%, 44%);
--red-600: hsl(1, 62%, 60%);
--red-700: hsl(1, 62%, 76%);
--red-800: hsl(1, 64%, 85%);
--red-900: hsl(1, 68%, 95%);
--red-950: hsl(1, 55%, 98%);
--pink-50: hsl(270, 40%, 16%);
--pink-100: hsl(270, 40%, 16%);
--pink-200: hsl(270, 45%, 24%);
--pink-300: hsl(270, 50%, 32%);
--pink-400: hsl(270, 55%, 43%);
--pink-500: hsl(270, 60%, 52%);
--pink-600: hsl(270, 70%, 65%);
--pink-700: hsl(270, 70%, 75%);
--pink-800: hsl(270, 70%, 85%);
--pink-900: hsl(270, 70%, 95%);
--pink-950: hsl(270, 70%, 95%);
--green-50: hsl(152, 15%, 10%);
--green-100: hsl(152, 26%, 11%);
--green-200: hsl(152, 32%, 16%);
--green-300: hsl(152, 38%, 24%);
--green-400: hsl(152, 38%, 34%);
--green-500: hsl(152, 38%, 42%);
--green-600: hsl(152, 38%, 60%);
--green-700: hsl(152, 38%, 70%);
--green-800: hsl(152, 38%, 80%);
--green-900: hsl(152, 38%, 91%);
--green-950: hsl(152, 40%, 97%);
--yellow-50: hsl(44, 95%, 12%);
--yellow-100: hsl(44, 95%, 12%);
--yellow-200: hsl(44, 96%, 24%);
--yellow-300: hsl(44, 95%, 36%);
--yellow-400: hsl(44, 95%, 48%);
--yellow-500: hsl(44, 95%, 60%);
--yellow-600: hsl(44, 95%, 69%);
--yellow-700: hsl(44, 95%, 78%);
--yellow-800: hsl(44, 95%, 86%);
--yellow-900: hsl(44, 95%, 95%);
--yellow-950: hsl(44, 95%, 95%);
}</style><link rel="stylesheet" href="/_next/static/css/619a01ff31a99955.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" noModule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-0b474a604a067f92.js" defer=""></script><script src="/_next/static/chunks/framework-90fc9b0f7c4706c1.js" defer=""></script><script src="/_next/static/chunks/main-e89abd6570bbb9dc.js" defer=""></script><script src="/_next/static/chunks/pages/_app-3049b372678b389f.js" defer=""></script><script src="/_next/static/chunks/5399-d49efedd870f41de.js" defer=""></script><script src="/_next/static/chunks/3801-cdd2681d27083ad8.js" defer=""></script><script src="/_next/static/chunks/9594-1c44b12020a3af0a.js" defer=""></script><script src="/_next/static/chunks/4696-fd1712bc36a5bc50.js" defer=""></script><script src="/_next/static/chunks/pages/p/%5Bslug%5D-ddadb1ca54eaab0f.js" defer=""></script><script src="/_next/static/nFX7R3M0MdLMAf-Ufh9qa/_buildManifest.js" defer=""></script><script src="/_next/static/nFX7R3M0MdLMAf-Ufh9qa/_ssgManifest.js" defer=""></script></head><body><div id="__next"><script>!function(){try{var d=document.documentElement,c=d.classList;c.remove('light','dark');var e=localStorage.getItem('theme');if('system'===e||(!e&&true)){var t='(prefers-color-scheme: dark)',m=window.matchMedia(t);if(m.media!==t||m.matches){d.style.colorScheme = 'dark';c.add('dark')}else{d.style.colorScheme = 'light';c.add('light')}}else if(e){c.add(e|| '')}if(e==='light'||e==='dark')d.style.colorScheme=e}catch(e){}}()</script><div class="__variable_f367f3 __variable_0b668e __variable_ecea63 font-sans"><div class="px-5 md:px-8"><nav class="max-w-6xl mx-auto py-6 flex justify-between items-center border-b border-gray-100"><a class="flex items-center space-x-4" href="/"><svg data-v-423bf9ae="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 59.99945071644884 60" class="logo w-8"><g data-v-423bf9ae="" id="6f639fe5-02e1-4640-a1fb-f4b22b64e5ef" transform="matrix(0.5999641134746578,0,0,0.5999641134746578,-437.5538206497148,-266.68705118956507)" stroke="none"><path d="M729.872 487.327a50.86 50.86 0 0 0-.464 5.033h75.879c-.265-.518-.621-.985-.98-1.442-12.972-16.769-19.95-15.315-29.932-15.741-3.328-.137-5.585-.192-18.832-.192-7.09 0-14.798.018-22.304.038.737-1.746 1.6-3.419 2.525-5.061-2.885 5.106-4.865 10.771-5.805 16.789l.891-4.397c.007-.031.018-.063.024-.094h38.883v5.067h-39.885zM805.885 497.432h-76.438c.08 1.352.206 2.686.388 4.002h70.571c3.146 0 4.907-1.786 5.479-4.002zM733.851 515.257a52.226 52.226 0 0 1-1.98-4.997c6.608 19.89 25.328 34.251 47.433 34.251 20.205 0 37.566-12.007 45.452-29.254h-90.905zM729.38 492.915c-.018.531-.08 1.055-.08 1.589 0 .538.063 1.059.08 1.59v-3.179zM824.77 515.229z"></path><path d="M779.303 444.505c-18.682 0-34.939 10.265-43.524 25.439 6.709-.014 19.775-.022 19.775-.022h.003v-.005c15.444 0 16.018.069 19.035.195l1.868.069c6.507.217 14.505.916 20.798 5.68 3.416 2.584 8.348 8.287 11.288 12.35 2.718 3.758 3.5 8.078 1.652 12.217-1.701 3.804-5.361 6.073-9.793 6.073H730.85l-.884-4.201c.426 2.707 1.037 5.344 1.879 7.886h94.914a49.863 49.863 0 0 0 2.546-15.682c.001-27.611-22.386-49.999-50.002-49.999z"></path></g></svg><span class="text-xl font-semibold">Blog</span></a><div class="text-gray-600 flex items-center space-x-6"><a class="text-sm hover:text-pink-600" href="https://railway.com" target="_blank" rel="noreferrer noopener">Go to Homepage</a><a class="hover:text-pink-600" href="/rss.xml" title="RSS Feed"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4 11a9 9 0 0 1 9 9"></path><path d="M4 4a16 16 0 0 1 16 16"></path><circle cx="5" cy="19" r="1"></circle></svg></a></div></nav></div><div class="min-h-screen overflow-x-hidden relative"><div class="mt-10 mb-5 px-5 md:px-8 mx-auto"><article class="max-w-6xl mx-auto mt-24 mb-12 border-b border-gray-100 pb-32"><div class="max-w-[704px] mx-auto flex items-center text-sm text-gray-600 space-x-3"><div class="flex items-center space-x-3"><div class="flex items-center"><img src="https://cms.railway.com/media/452039afd9d5078059203ebfb4028e92717d03b94e10832c814d2ca23c5bce39.png" alt="Avatar of Charith Amarasinghe" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/></div><span>Charith Amarasinghe</span></div><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><time dateTime="2025-01-17T00:00:00.000Z">Jan 17, 2025</time></div><header class="mt-5 mb-16 max-w-[704px] mx-auto"><h1 class="text-post-title font-medium font-serif">So You Want to Build Your Own Data Center</h1></header><section class="max-w-[704px] mx-auto text-base sm:text-lg"><nav style="position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border-width:0" aria-label="Table of Contents"><h2>Table of Contents</h2><ol><li style="margin-left:1.5rem"><a href="#so-you-want-to-build-a-cloud">So you want to build a cloud</a></li><li style="margin-left:1.5rem"><a href="#with-great-power-comes-great-responsibility">With great power comes great responsibility</a></li><li style="margin-left:1.5rem"><a href="#let-there-be-light">Let there be light</a></li><li style="margin-left:1.5rem"><a href="#aisles,-racks-and-overhead-infrastructure">Aisles, racks and overhead infrastructure</a></li><li style="margin-left:1.5rem"><a href="#the-rack-and-stack">The rack and stack</a></li><li style="margin-left:1.5rem"><a href="#pedal-on-metal">Pedal on Metal</a></li></ol></nav><div><p class="mb-4 text-gray-800">Since the beginning, Railway’s compute has been built on top of Google Cloud Platform. The platform supported Railway&#x27;s initial journey, but it has caused a multitude of problems that have posed an existential risk to our business. More importantly, building on a hyperscaler prevents us from delivering the best possible platform to our customers.</p>
<p class="mb-4 text-gray-800">It directly affected the pricing we could offer (egress fees anyone?), limited the level of service we could deliver, and introduced engineering constraints that restricted the features we could build.</p>
<p class="mb-4 text-gray-800">And not only is it <a class="markdown-inline-link" href="https://github.com/GoogleCloudPlatform/guest-agent/issues/401" target="_blank" rel="noreferrer noopener">rare</a> that we understand <a class="markdown-inline-link" href="https://blog.railway.com/p/2023-12-01-incident-report" target="_blank" rel="noreferrer noopener">why</a> <a class="markdown-inline-link" href="https://x.com/JustJake/status/1667478906591666176" target="_blank" rel="noreferrer noopener">things</a> <a class="markdown-inline-link" href="https://blog.railway.com/p/incident-december-16-2024" target="_blank" rel="noreferrer noopener">break</a> upstream, but also despite multi-million dollar annual spend, we get about as much support from them as you would spending $100.</p>
<p class="mb-4 text-gray-800">So in response, we kicked off a Railway Metal project last year. Nine months later we were live with the first site in California, having designed, spec-ed, and installed everything from the fiber optic cables in the cage to the various contracts with ISPs. We’re lighting up three more data center regions as we speak.</p>
<p class="mb-4 text-gray-800">To deliver an “infra-less” cloud experience to our customers, we’ve needed to get good fast at building out our own physical infrastructure. That’s the topic of our blogpost today.</p>
<a class="relative no-underline" href="/p/data-center-build-part-one#so-you-want-to-build-a-cloud"><span id="so-you-want-to-build-a-cloud" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">So you want to build a cloud</h2></a>
<p class="mb-4 text-gray-800">From kicking off the Railway Metal project in January 2024, it took us five long months to get the first servers plugged in. It took us an additional three months before we felt comfortable letting our users onto the hardware (and an additional few months before we started writing about it here).</p>
<p class="mb-4 text-gray-800">The first step was finding some space.</p>
<p class="mb-4 text-gray-800">When you go “on-prem” in cloud-speak, you need somewhere to put your shiny servers and reliable power to keep them running. Also you want enough cooling so they don’t melt down.</p>
<p class="mb-4 text-gray-800">In general you have three main choices: Greenfield buildout (buying or leasing a datacenter), Cage Colocation (getting a private space inside a provider&#x27;s datacenter enclosed by mesh walls), or Rack colocation (leasing individual racks or partitions of racks in a colocation datacenter).</p>
<p class="mb-4 text-gray-800">We chose the second option: a cage to give us four walls, a secure door, and a blank slate for everything else.</p>
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/ce8d56ecaf1283f837c13bc69296d93fb8e4b1689626c3d589f2b4146957bb3d.png?w=768&amp;format=webp&amp;q=90" srcSet="https://cms.railway.com/media/ce8d56ecaf1283f837c13bc69296d93fb8e4b1689626c3d589f2b4146957bb3d.png?w=320&amp;format=webp&amp;q=90 320w, https://cms.railway.com/media/ce8d56ecaf1283f837c13bc69296d93fb8e4b1689626c3d589f2b4146957bb3d.png?w=480&amp;format=webp&amp;q=90 480w, https://cms.railway.com/media/ce8d56ecaf1283f837c13bc69296d93fb8e4b1689626c3d589f2b4146957bb3d.png?w=640&amp;format=webp&amp;q=90 640w, https://cms.railway.com/media/ce8d56ecaf1283f837c13bc69296d93fb8e4b1689626c3d589f2b4146957bb3d.png?w=768&amp;format=webp&amp;q=90 768w, https://cms.railway.com/media/ce8d56ecaf1283f837c13bc69296d93fb8e4b1689626c3d589f2b4146957bb3d.png?w=960&amp;format=webp&amp;q=90 960w, https://cms.railway.com/media/ce8d56ecaf1283f837c13bc69296d93fb8e4b1689626c3d589f2b4146957bb3d.png?w=1200&amp;format=webp&amp;q=90 1200w, https://cms.railway.com/media/ce8d56ecaf1283f837c13bc69296d93fb8e4b1689626c3d589f2b4146957bb3d.png?w=1440&amp;format=webp&amp;q=90 1440w, https://cms.railway.com/media/ce8d56ecaf1283f837c13bc69296d93fb8e4b1689626c3d589f2b4146957bb3d.png?w=1600&amp;format=webp&amp;q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="A cage before any racks have been fitted" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">A cage before any racks have been fitted</figcaption></figure>
<p class="mb-4 text-gray-800">The space itself doesn’t cost much, but power (and by proxy, cooling) costs the most. Depending on the geography, the $/kW rate can vary hugely — on the US west coast for example we may pay less than half as much as we pay in Singapore. Power is paid for as a fixed monthly commit, regardless of whether its consumed or not, to guarantee it will be available on-demand.</p>
<p class="mb-4 text-gray-800">But how much power do you need?</p>
<a class="relative no-underline" href="/p/data-center-build-part-one#with-great-power-comes-great-responsibility"><span id="with-great-power-comes-great-responsibility" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">With great power comes great responsibility</h2></a>
<p class="mb-4 text-gray-800">Ideally if you’ve embarked on your data center migration mission, you should have an idea of the rough amount of compute you want to deploy. We started with a target number of vCPUs, GBs of RAM, and TBs of NVMe to match our capacity on GCP.</p>
<p class="mb-4 text-gray-800">Using these figures, we converged on a server and CPU choice. There are many knobs to turn when doing this computation — probably worth a blogpost in itself — but the single biggest factor for us was power density e.g. how do we get the compute density we want inside of a specific power draw.</p>
<p class="mb-4 text-gray-800">The calculations aren’t as simple as summing watts though, especially with 3-phase feeds — Cloudflare has <a class="markdown-inline-link" href="https://blog.cloudflare.com/an-introduction-to-three-phase-power-and-pdus/" target="_blank" rel="noreferrer noopener">a great blogpost</a> covering this topic.</p>
<p class="mb-4 text-gray-800">Power is the most critical resource for data centers, and a power outage can have extremely long recovery times. So redundancy is critical, and it’s important to have two fully independent power feeds per rack. Both feeds will share load under normal operation, but the design must be resilient to a feed going down.</p>
<p class="mb-4 text-gray-800">To deliver this power to your servers, you’ll also want a Power Distribution Unit, which you&#x27;ll select based on the number of sockets and management features it provides. The basic ones are glorified extension cords, while the ones we deploy allow control and metering of individual sockets.</p>
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/726fb15777885c0f483780040814464609209f58bebef47d2c380aa8e0fca508.png?w=768&amp;format=webp&amp;q=90" srcSet="https://cms.railway.com/media/726fb15777885c0f483780040814464609209f58bebef47d2c380aa8e0fca508.png?w=320&amp;format=webp&amp;q=90 320w, https://cms.railway.com/media/726fb15777885c0f483780040814464609209f58bebef47d2c380aa8e0fca508.png?w=480&amp;format=webp&amp;q=90 480w, https://cms.railway.com/media/726fb15777885c0f483780040814464609209f58bebef47d2c380aa8e0fca508.png?w=640&amp;format=webp&amp;q=90 640w, https://cms.railway.com/media/726fb15777885c0f483780040814464609209f58bebef47d2c380aa8e0fca508.png?w=768&amp;format=webp&amp;q=90 768w, https://cms.railway.com/media/726fb15777885c0f483780040814464609209f58bebef47d2c380aa8e0fca508.png?w=960&amp;format=webp&amp;q=90 960w, https://cms.railway.com/media/726fb15777885c0f483780040814464609209f58bebef47d2c380aa8e0fca508.png?w=1200&amp;format=webp&amp;q=90 1200w, https://cms.railway.com/media/726fb15777885c0f483780040814464609209f58bebef47d2c380aa8e0fca508.png?w=1440&amp;format=webp&amp;q=90 1440w, https://cms.railway.com/media/726fb15777885c0f483780040814464609209f58bebef47d2c380aa8e0fca508.png?w=1600&amp;format=webp&amp;q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="Each PDU is accessible over the network and individual sockets can be remotely metered and controlled" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">Each PDU is accessible over the network and individual sockets can be remotely metered and controlled</figcaption></figure>
<p class="mb-4 text-gray-800">With that, power is now available in the cage.</p>
<a class="relative no-underline" href="/p/data-center-build-part-one#let-there-be-light"><span id="let-there-be-light" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Let there be light</h2></a>
<p class="mb-4 text-gray-800">No cloud machine is an island and that&#x27;s where networks come into play.</p>
<p class="mb-4 text-gray-800">To achieve the lowest possible latency on Railway, we need to set you up with solid connections to the rest of the world.</p>
<p class="mb-4 text-gray-800">We look for DC facilities that are on-network with Tier 1 Internet Services Providers (ISPs), that are part of an Internet Exchange (IX), and that have available fiber to other data centers in close proximity.</p>
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/0ceb2f6d1000dc97aeedd06e34033a6427a46ac0c5ad2b3fbd635915ce42210d.png?w=768&amp;format=webp&amp;q=90" srcSet="https://cms.railway.com/media/0ceb2f6d1000dc97aeedd06e34033a6427a46ac0c5ad2b3fbd635915ce42210d.png?w=320&amp;format=webp&amp;q=90 320w, https://cms.railway.com/media/0ceb2f6d1000dc97aeedd06e34033a6427a46ac0c5ad2b3fbd635915ce42210d.png?w=480&amp;format=webp&amp;q=90 480w, https://cms.railway.com/media/0ceb2f6d1000dc97aeedd06e34033a6427a46ac0c5ad2b3fbd635915ce42210d.png?w=640&amp;format=webp&amp;q=90 640w, https://cms.railway.com/media/0ceb2f6d1000dc97aeedd06e34033a6427a46ac0c5ad2b3fbd635915ce42210d.png?w=768&amp;format=webp&amp;q=90 768w, https://cms.railway.com/media/0ceb2f6d1000dc97aeedd06e34033a6427a46ac0c5ad2b3fbd635915ce42210d.png?w=960&amp;format=webp&amp;q=90 960w, https://cms.railway.com/media/0ceb2f6d1000dc97aeedd06e34033a6427a46ac0c5ad2b3fbd635915ce42210d.png?w=1200&amp;format=webp&amp;q=90 1200w, https://cms.railway.com/media/0ceb2f6d1000dc97aeedd06e34033a6427a46ac0c5ad2b3fbd635915ce42210d.png?w=1440&amp;format=webp&amp;q=90 1440w, https://cms.railway.com/media/0ceb2f6d1000dc97aeedd06e34033a6427a46ac0c5ad2b3fbd635915ce42210d.png?w=1600&amp;format=webp&amp;q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="Each cage gets diverse and redundant network links" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">Each cage gets diverse and redundant network links</figcaption></figure>
<p class="mb-4 text-gray-800">Your applications deployed to Railway will want to connect to a diverse mix of endpoints over the network — be it a home internet user in Sydney, Australia or a API hosted on an AWS server in the US. To get you the best possible latency and the lowest bandwidth cost, we contract with a mix of internet providers optimized for each use case.</p>
<p class="mb-4 text-gray-800">We select ISPs for the maturity of their networks in each geography we target. Partnering with the wrong ISP in a region can lead to extra network hops (and thus latency) to reach specific target markets — or in the worst case — convoluted network routes. So for each region, we pick at least two separate networks based on their regional footprints.</p>
<p class="mb-4 text-gray-800">Once connected, we receive full internet routing tables from each ISP and consolidate them on our network switches to resolve the best path for each IP prefix. If you have an end user in Australia trying to reach an app deployed to Singapore, we’ll likely hand those packets off directly to <a class="markdown-inline-link" href="https://bgp.tools/as/4637#connectivity" target="_blank" rel="noreferrer noopener">Telstra</a> who have one of the densest access networks in Australia. If that same app needs to send packets to a end-user or server in Japan, then we’d likely be handing them over to PCCW who peer directly with NTT in Japan and have a <a class="markdown-inline-link" href="https://www.pccwglobal.com/wp-content/uploads/2019/11/world_fold_20191115.pdf" target="_blank" rel="noreferrer noopener">dense footprint in APAC</a>.</p><div class="flex w-full p-4 my-8 rounded border border-transparent bg-blue-100"><div class="text-yellow-500">👉</div><div class="flex flex-col w-full"><div class="ml-4 text-foreground"><p class="mb-4 text-gray-800">Peering information is public, head-over to <a class="markdown-inline-link" href="https://bgp.tools/" target="_blank" rel="noreferrer noopener">bgp.tools</a> to see how your favorite networks interconnect.</p></div></div></div><p class="mb-4 text-gray-800">For redundancy we’re building out multiple zones in each region, and interconnectivity between these sites is also critical for our expansion. There are several tools such as dark fiber or wavelength services that we look for to plan this expansion. The result is that your apps won’t notice if your database is in the same room or if it’s 4 blocks over in a neighboring building — this is a feature not a bug — as it builds resilience against the failure of an individual data center.</p>
<p class="mb-4 text-gray-800">...</p>
<p class="mb-4 text-gray-800">Ok, now that you’ve found a space you like, signed a deal with a data center, and signed deals with several ISPs, you&#x27;re all-systems-go to install some servers, right?</p>
<p class="mb-4 text-gray-800">Well, not exactly. First you need a bunch of other things to give your server a nice snug home to warm up in.</p>
<a class="relative no-underline" href="/p/data-center-build-part-one#aisles,-racks-and-overhead-infrastructure"><span id="aisles,-racks-and-overhead-infrastructure" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Aisles, racks and overhead infrastructure</h2></a>
<p class="mb-4 text-gray-800">In a data center, racks are arranged in rows, and the space between racks, the aisle, is used for airflow.</p>
<p class="mb-4 text-gray-800">The Cold Aisle is where cold air is blown in from the DC facility, and servers in your rack suck this air up and exhaust it towards the rear onto the Hot Aisle. The DC facility will remove this hot air from the Hot Aisle. For optimum efficiency, you don’t want air between these aisles to mix.</p>
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/de2d850903b61c0498252628043c4644a3a6c279cf40a4d338dcbb4e567cc032.png?w=768&amp;format=webp&amp;q=90" srcSet="https://cms.railway.com/media/de2d850903b61c0498252628043c4644a3a6c279cf40a4d338dcbb4e567cc032.png?w=320&amp;format=webp&amp;q=90 320w, https://cms.railway.com/media/de2d850903b61c0498252628043c4644a3a6c279cf40a4d338dcbb4e567cc032.png?w=480&amp;format=webp&amp;q=90 480w, https://cms.railway.com/media/de2d850903b61c0498252628043c4644a3a6c279cf40a4d338dcbb4e567cc032.png?w=640&amp;format=webp&amp;q=90 640w, https://cms.railway.com/media/de2d850903b61c0498252628043c4644a3a6c279cf40a4d338dcbb4e567cc032.png?w=768&amp;format=webp&amp;q=90 768w, https://cms.railway.com/media/de2d850903b61c0498252628043c4644a3a6c279cf40a4d338dcbb4e567cc032.png?w=960&amp;format=webp&amp;q=90 960w, https://cms.railway.com/media/de2d850903b61c0498252628043c4644a3a6c279cf40a4d338dcbb4e567cc032.png?w=1200&amp;format=webp&amp;q=90 1200w, https://cms.railway.com/media/de2d850903b61c0498252628043c4644a3a6c279cf40a4d338dcbb4e567cc032.png?w=1440&amp;format=webp&amp;q=90 1440w, https://cms.railway.com/media/de2d850903b61c0498252628043c4644a3a6c279cf40a4d338dcbb4e567cc032.png?w=1600&amp;format=webp&amp;q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="A cage schematic - all equipment must be mounted such that fans blow air from the cold aisle to the hot" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">A cage schematic - all equipment must be mounted such that fans blow air from the cold aisle to the hot</figcaption></figure>
<p class="mb-4 text-gray-800">The racks themselves have some variability, even if you opt to use conventional 19&quot; wide equipment. You can select the height, width, and depth to suit your equipment and cabling needs.</p>
<p class="mb-4 text-gray-800">Most server equipment can slide on rails to allow for easy maintenance, so it’s important to ensure that cage dimensions allow for this. Cabling and cable management also requires some space, so there’s a tradeoff to be made with how crowded you want each rack to be vs. how many racks you can fit into a cage.</p>
<p class="mb-4 text-gray-800">In our experience, power and cooling is often the limiting factor rather than the actual space available. In newer sites we opt for wider 800mm racks to allow for better airflow by getting cables out of the way of the exhausts.</p>
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/4405868f310c22342375a9a0b89e15141dc2fe431e79363c1927fc5c9f8ad361.jpeg?w=768&amp;format=webp&amp;q=90" srcSet="https://cms.railway.com/media/4405868f310c22342375a9a0b89e15141dc2fe431e79363c1927fc5c9f8ad361.jpeg?w=320&amp;format=webp&amp;q=90 320w, https://cms.railway.com/media/4405868f310c22342375a9a0b89e15141dc2fe431e79363c1927fc5c9f8ad361.jpeg?w=480&amp;format=webp&amp;q=90 480w, https://cms.railway.com/media/4405868f310c22342375a9a0b89e15141dc2fe431e79363c1927fc5c9f8ad361.jpeg?w=640&amp;format=webp&amp;q=90 640w, https://cms.railway.com/media/4405868f310c22342375a9a0b89e15141dc2fe431e79363c1927fc5c9f8ad361.jpeg?w=768&amp;format=webp&amp;q=90 768w, https://cms.railway.com/media/4405868f310c22342375a9a0b89e15141dc2fe431e79363c1927fc5c9f8ad361.jpeg?w=960&amp;format=webp&amp;q=90 960w, https://cms.railway.com/media/4405868f310c22342375a9a0b89e15141dc2fe431e79363c1927fc5c9f8ad361.jpeg?w=1200&amp;format=webp&amp;q=90 1200w, https://cms.railway.com/media/4405868f310c22342375a9a0b89e15141dc2fe431e79363c1927fc5c9f8ad361.jpeg?w=1440&amp;format=webp&amp;q=90 1440w, https://cms.railway.com/media/4405868f310c22342375a9a0b89e15141dc2fe431e79363c1927fc5c9f8ad361.jpeg?w=1600&amp;format=webp&amp;q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="A rack-mount server on rails can be pulled out and serviced in place, each weighs nearly 20kg…" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">A rack-mount server on rails can be pulled out and serviced in place, each weighs nearly 20kg…</figcaption></figure>
<p class="mb-4 text-gray-800">In addition to racks, you&#x27;ll need several bits of infrastructure to get power and data to your racks. This will likely involve installing some overhead infrastructure and trays that let you route fiber cables from the edge of your cage to each of your racks, and to route cables between racks. This is something the DC operator will throw in when quoting the cage.</p>
<p class="mb-4 text-gray-800">Depending on your design, you’ll want to optimize for short cable paths by ensuring your overhead infrastructure, rack local cabling, and device orientations align. Because our racks have dense switch-to-server fiber cabling in each rack, we buy switches that have their ports oriented to the back of the rack (these are called reverse airflow switches because they exhaust air on the side with the network ports).</p>
<p class="mb-4 text-gray-800">This allows us to align the cable trays such that all cabling happens on one side of the rack and there’s no zig-zagging of cables between the front and back of the rack.</p>
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/f5ccc298f7c705fe4a62a29ec9cae38c20ad382215943ddf9e78afbbec575c8c.jpeg?w=768&amp;format=webp&amp;q=90" srcSet="https://cms.railway.com/media/f5ccc298f7c705fe4a62a29ec9cae38c20ad382215943ddf9e78afbbec575c8c.jpeg?w=320&amp;format=webp&amp;q=90 320w, https://cms.railway.com/media/f5ccc298f7c705fe4a62a29ec9cae38c20ad382215943ddf9e78afbbec575c8c.jpeg?w=480&amp;format=webp&amp;q=90 480w, https://cms.railway.com/media/f5ccc298f7c705fe4a62a29ec9cae38c20ad382215943ddf9e78afbbec575c8c.jpeg?w=640&amp;format=webp&amp;q=90 640w, https://cms.railway.com/media/f5ccc298f7c705fe4a62a29ec9cae38c20ad382215943ddf9e78afbbec575c8c.jpeg?w=768&amp;format=webp&amp;q=90 768w, https://cms.railway.com/media/f5ccc298f7c705fe4a62a29ec9cae38c20ad382215943ddf9e78afbbec575c8c.jpeg?w=960&amp;format=webp&amp;q=90 960w, https://cms.railway.com/media/f5ccc298f7c705fe4a62a29ec9cae38c20ad382215943ddf9e78afbbec575c8c.jpeg?w=1200&amp;format=webp&amp;q=90 1200w, https://cms.railway.com/media/f5ccc298f7c705fe4a62a29ec9cae38c20ad382215943ddf9e78afbbec575c8c.jpeg?w=1440&amp;format=webp&amp;q=90 1440w, https://cms.railway.com/media/f5ccc298f7c705fe4a62a29ec9cae38c20ad382215943ddf9e78afbbec575c8c.jpeg?w=1600&amp;format=webp&amp;q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="A ladder rack and fibre cable tray allow routing cables to racks from above" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">A ladder rack and fibre cable tray allow routing cables to racks from above</figcaption></figure>
<p class="mb-4 text-gray-800">So you’ve got the space, signed up ISPs, ordered the hardware, got the racks, and a pretty good picture on how to lay it all out. But it’s still a pretty expensive lego set sitting in the loading bay of a data center. To assemble it you now need to leverage the most versatile programming tool ever devised in the history of mankind … Microsoft Excel.</p>
<a class="relative no-underline" href="/p/data-center-build-part-one#the-rack-and-stack"><span id="the-rack-and-stack" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">The rack and stack</h2></a>
<p class="mb-4 text-gray-800">Let’s first step back and publish a disclaimer: neat and organized cabling requires a lot of practice; we tried it ourselves first - with… <em>mixed…</em> results.</p>
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/e2548904f65a867960c50b2001b723b208b4abe8e20f95f18c6e86382c7aa99f.png?w=768&amp;format=webp&amp;q=90" srcSet="https://cms.railway.com/media/e2548904f65a867960c50b2001b723b208b4abe8e20f95f18c6e86382c7aa99f.png?w=320&amp;format=webp&amp;q=90 320w, https://cms.railway.com/media/e2548904f65a867960c50b2001b723b208b4abe8e20f95f18c6e86382c7aa99f.png?w=480&amp;format=webp&amp;q=90 480w, https://cms.railway.com/media/e2548904f65a867960c50b2001b723b208b4abe8e20f95f18c6e86382c7aa99f.png?w=640&amp;format=webp&amp;q=90 640w, https://cms.railway.com/media/e2548904f65a867960c50b2001b723b208b4abe8e20f95f18c6e86382c7aa99f.png?w=768&amp;format=webp&amp;q=90 768w, https://cms.railway.com/media/e2548904f65a867960c50b2001b723b208b4abe8e20f95f18c6e86382c7aa99f.png?w=960&amp;format=webp&amp;q=90 960w, https://cms.railway.com/media/e2548904f65a867960c50b2001b723b208b4abe8e20f95f18c6e86382c7aa99f.png?w=1200&amp;format=webp&amp;q=90 1200w, https://cms.railway.com/media/e2548904f65a867960c50b2001b723b208b4abe8e20f95f18c6e86382c7aa99f.png?w=1440&amp;format=webp&amp;q=90 1440w, https://cms.railway.com/media/e2548904f65a867960c50b2001b723b208b4abe8e20f95f18c6e86382c7aa99f.png?w=1600&amp;format=webp&amp;q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="Our DIY attempts at cabling to get the first server online vs. what our installation partner put together" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">Our DIY attempts at cabling to get the first server online vs. what our installation partner put together</figcaption></figure>
<p class="mb-4 text-gray-800">To install it properly, we bring in professionals, but the professionals need to know what to install. A comprehensive documentation pack is essential. A cabling matrix and rack elevation are common documents that communicate to contractors how to rack and wire-up servers.</p>
<p class="mb-4 text-gray-800">A cabling matrix describes the termination of each cable, specifying the device position and port for each side of the connection, along with the specification of the cable itself (type of fiber, length, etc). The rack elevation is a visual representation of the rack itself, showing the position and orientation of each device.</p>
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/367c1ecfc1b4fe721b06d71ee2b82cfead5e65133e3815985d72f4c968fbcb16.png?w=768&amp;format=webp&amp;q=90" srcSet="https://cms.railway.com/media/367c1ecfc1b4fe721b06d71ee2b82cfead5e65133e3815985d72f4c968fbcb16.png?w=320&amp;format=webp&amp;q=90 320w, https://cms.railway.com/media/367c1ecfc1b4fe721b06d71ee2b82cfead5e65133e3815985d72f4c968fbcb16.png?w=480&amp;format=webp&amp;q=90 480w, https://cms.railway.com/media/367c1ecfc1b4fe721b06d71ee2b82cfead5e65133e3815985d72f4c968fbcb16.png?w=640&amp;format=webp&amp;q=90 640w, https://cms.railway.com/media/367c1ecfc1b4fe721b06d71ee2b82cfead5e65133e3815985d72f4c968fbcb16.png?w=768&amp;format=webp&amp;q=90 768w, https://cms.railway.com/media/367c1ecfc1b4fe721b06d71ee2b82cfead5e65133e3815985d72f4c968fbcb16.png?w=960&amp;format=webp&amp;q=90 960w, https://cms.railway.com/media/367c1ecfc1b4fe721b06d71ee2b82cfead5e65133e3815985d72f4c968fbcb16.png?w=1200&amp;format=webp&amp;q=90 1200w, https://cms.railway.com/media/367c1ecfc1b4fe721b06d71ee2b82cfead5e65133e3815985d72f4c968fbcb16.png?w=1440&amp;format=webp&amp;q=90 1440w, https://cms.railway.com/media/367c1ecfc1b4fe721b06d71ee2b82cfead5e65133e3815985d72f4c968fbcb16.png?w=1600&amp;format=webp&amp;q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="The cabling matrix for our first install - 300+ cables all manually entered and cross-checked" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">The cabling matrix for our first install - 300+ cables all manually entered and cross-checked</figcaption></figure>
<p class="mb-4 text-gray-800">The documentation exercise can be intense, each of our install phases involved 60+ devices, 300+ discrete cables, and dozens of little details. This was all handcrafted into written specifications and spreadsheets we used as a basis for the installation and commissioning. From the materials being on site to getting everything installed takes us about 6-14 days.</p>
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/a15e16dd1983bce5d2da56e69a5c96efdc3060178088deb62d227a04ff533223.png?w=768&amp;format=webp&amp;q=90" srcSet="https://cms.railway.com/media/a15e16dd1983bce5d2da56e69a5c96efdc3060178088deb62d227a04ff533223.png?w=320&amp;format=webp&amp;q=90 320w, https://cms.railway.com/media/a15e16dd1983bce5d2da56e69a5c96efdc3060178088deb62d227a04ff533223.png?w=480&amp;format=webp&amp;q=90 480w, https://cms.railway.com/media/a15e16dd1983bce5d2da56e69a5c96efdc3060178088deb62d227a04ff533223.png?w=640&amp;format=webp&amp;q=90 640w, https://cms.railway.com/media/a15e16dd1983bce5d2da56e69a5c96efdc3060178088deb62d227a04ff533223.png?w=768&amp;format=webp&amp;q=90 768w, https://cms.railway.com/media/a15e16dd1983bce5d2da56e69a5c96efdc3060178088deb62d227a04ff533223.png?w=960&amp;format=webp&amp;q=90 960w, https://cms.railway.com/media/a15e16dd1983bce5d2da56e69a5c96efdc3060178088deb62d227a04ff533223.png?w=1200&amp;format=webp&amp;q=90 1200w, https://cms.railway.com/media/a15e16dd1983bce5d2da56e69a5c96efdc3060178088deb62d227a04ff533223.png?w=1440&amp;format=webp&amp;q=90 1440w, https://cms.railway.com/media/a15e16dd1983bce5d2da56e69a5c96efdc3060178088deb62d227a04ff533223.png?w=1600&amp;format=webp&amp;q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="We have now built internal tooling to automate generating build specifications" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">We have now built internal tooling to automate generating build specifications</figcaption></figure>
<p class="mb-4 text-gray-800">This all seems very far removed from software, DevOps, or what you’d typically think of as “infrastructure,” and that is very true — building a datacenter cage is probably closer to building a house than to deploying a Terraform stack.</p>
<p class="mb-4 text-gray-800">To compound this, every datacenter facility, contractor and vendor will do things slightly differently, even within the same organization. The operational aspect requires you to stay on your toes and be extremely detail-oriented.</p>
<p class="mb-4 text-gray-800">Some what-the-duck moments we’ve had thus far:</p>
<ul class="list-disc pl-6 mb-4 space-y-4 text-gray-800">
<li>Contractor: “We need longer power cables” - the PDUs at that site were upside down because the power came in from the floor, so our socket numbering was reversed in the plan</li>
<li>Phonecall from Amsterdam: “There’s no demarcation point at the site?” - a specific facility installs external fibre links direct to a box in one of our racks rather than via a dedicated demarcation point overhead</li>
<li>Railway Discord quote: “Why are the phases wired so weirdly on this PDU?” - the facility was wired differently to our other sites and the power sockets were wired phase-to-neutral vs. phase-to-phase (WYE vs Delta circuits for you EE’s)</li>
<li>Contractor: “Your data cables are too short” - the contractor didn’t realize the network gear was reverse-airflow and tried to mount things the wrong way around</li>
<li>Us raising a support ticket: “There’s no link coming up on this cable” - the fiber was wired in the wrong polarity; we learnt what “rolling fibre cables” was that day… it’s when they rip out the plugs from the <a class="markdown-inline-link" href="https://www.fs.com/uk/blog/lc-fiber-optics-a-comprehensive-guide-2684.html" target="_blank" rel="noreferrer noopener">LC connector</a> and swap them around</li>
<li>Railway Discord quote: “I brought a rubber mallet from HomeDepot today” - a batch of nearly 24 PDUs from one vendor were delivered with faulty sockets that didn’t properly engage with the power plugs, even with <del>appropriate</del> extreme mechanical force being applied</li>
</ul>
<p class="mb-4 text-gray-800">But from this point - the hardware is in place the task begins to feel more familiar; we’re now needing to do some BGP, install some OS’es, setup monitoring and bring everything up.</p>
<figure class="flex flex-col my-8 space-y-2"><img src="https://cms.railway.com/media/2ed17ac47150832aae6332e806daa1a6d380cb1b7b902f7715957e89968da5c7.jpeg?w=768&amp;format=webp&amp;q=90" srcSet="https://cms.railway.com/media/2ed17ac47150832aae6332e806daa1a6d380cb1b7b902f7715957e89968da5c7.jpeg?w=320&amp;format=webp&amp;q=90 320w, https://cms.railway.com/media/2ed17ac47150832aae6332e806daa1a6d380cb1b7b902f7715957e89968da5c7.jpeg?w=480&amp;format=webp&amp;q=90 480w, https://cms.railway.com/media/2ed17ac47150832aae6332e806daa1a6d380cb1b7b902f7715957e89968da5c7.jpeg?w=640&amp;format=webp&amp;q=90 640w, https://cms.railway.com/media/2ed17ac47150832aae6332e806daa1a6d380cb1b7b902f7715957e89968da5c7.jpeg?w=768&amp;format=webp&amp;q=90 768w, https://cms.railway.com/media/2ed17ac47150832aae6332e806daa1a6d380cb1b7b902f7715957e89968da5c7.jpeg?w=960&amp;format=webp&amp;q=90 960w, https://cms.railway.com/media/2ed17ac47150832aae6332e806daa1a6d380cb1b7b902f7715957e89968da5c7.jpeg?w=1200&amp;format=webp&amp;q=90 1200w, https://cms.railway.com/media/2ed17ac47150832aae6332e806daa1a6d380cb1b7b902f7715957e89968da5c7.jpeg?w=1440&amp;format=webp&amp;q=90 1440w, https://cms.railway.com/media/2ed17ac47150832aae6332e806daa1a6d380cb1b7b902f7715957e89968da5c7.jpeg?w=1600&amp;format=webp&amp;q=90 1600w" sizes="(max-width: 768px) 100vw, 736px" alt="The completed cage ready for configuration" class="w-full rounded-[8px]" loading="lazy" decoding="async"/><figcaption class="text-gray-600 mt-3 text-sm">The completed cage ready for configuration</figcaption></figure>
<a class="relative no-underline" href="/p/data-center-build-part-one#pedal-on-metal"><span id="pedal-on-metal" aria-hidden="true" class="absolute inline-block w-px top-[-2rem]"></span><h2 class="text-h2 mt-10 mb-6">Pedal on Metal</h2></a>
<p class="mb-4 text-gray-800">The installed cage is but a blank canvas, the network devices need configuring, router config needs writing, RIR (regional internet registry) records need updating, and we must interact with the likes of <a class="markdown-inline-link" href="https://www.dmtf.org/standards/redfish" target="_blank" rel="noreferrer noopener">Redfish APIs</a> (HTTP APIs to dedicated controllers on server motherboards and PDUs) and PXE (a protocol to boot servers over the network) to get everything up and running.</p>
<p class="mb-4 text-gray-800">We&#x27;ve also not discussed how networking works. Our design uses <a class="markdown-inline-link" href="https://frrouting.org/" target="_blank" rel="noreferrer noopener">FRR</a> and whitebox network switches running <a class="markdown-inline-link" href="https://sonicfoundation.dev/" target="_blank" rel="noreferrer noopener">SONiC</a> to build a L3-only software-driven network that deeply integrates with our control plane.</p>
<p class="mb-4 text-gray-800">We’ve regaled you with tales from the frontline … but any more and you’d be here all day.</p>
<p class="mb-4 text-gray-800">In a future post, we’ll discuss how we go from a bunch of servers in a room to a functional Railway zone. In the space of the last few months we’ve built two new software tools, <em>Railyard</em> and <em>MetalCP</em>, to enable a button click experience from designing a new cage, tracing and visualizing the cabling, to installing OSes on servers and getting them on the internet.</p>
<p class="mb-4 text-gray-800">Until then, if any of this excites you, check out our <a class="markdown-inline-link" href="https://railway.com/careers/infra-platform" target="_blank" rel="noreferrer noopener">open Infrastructure Engineering roles</a> and reach out if they catch your interest.</p></div></section></article><div class="max-w-6xl mx-auto"><div><header class="flex items-center justify-between mb-8"><h3 class="text-gray-500 text-h3">Continue Reading...</h3><a class="text-pink-500 hover:underline" href="/engineering">View All <!-- -->Engineering<!-- --> -&gt;</a></header><div class="grid grid-cols-1 md:grid-cols-2 gap-8"><a class="flex flex-col bg-secondaryBg p-6 rounded-lg hover:bg-gray-100 group" href="/p/dev-new"><div class="flex gap-2"><div class="post-category-pill font-medium max-w-max text-xs px-2 py-[3px] rounded-[4px] tracking-[0.06em] uppercase" data-category-tone="plum">Engineering</div></div><div class="flex-grow"><header class="font-medium font-serif text-lg mt-2 mb-1">Your Cloud should come with an app builder </header><p class="text-base text-gray-800 line-clamp-2">Announcing: dev.new, an agent that can build, preview, test, and deploy an app from the same place. We tested the whole loop by asking it to ship a real-time multiplayer FPS game that runs in the browser</p></div><div class="flex items-center gap-3 mt-6"><div class="flex items-center"><img src="https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg" alt="Avatar of Mahmoud Abdelwahab" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/></div><span class="text-sm text-gray-600">Mahmoud Abdelwahab</span><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><span class="text-sm text-gray-600">Aug 13, 2026</span></div></a><a class="flex flex-col bg-secondaryBg p-6 rounded-lg hover:bg-gray-100 group" href="/p/feature-flags"><div class="flex gap-2"><div class="post-category-pill font-medium max-w-max text-xs px-2 py-[3px] rounded-[4px] tracking-[0.06em] uppercase" data-category-tone="plum">Engineering</div></div><div class="flex-grow"><header class="font-medium font-serif text-lg mt-2 mb-1">Roll it out, roll it back, never redeploy</header><p class="text-base text-gray-800 line-clamp-2">Merge now, release when you&#x27;re ready: turn a feature on for your team, then 10% of users, then everyone. Feature flags are now built into every Railway project, and your agents can run the rollout from the CLI, SDK, and MCP. No third-party service required.</p></div><div class="flex items-center gap-3 mt-6"><div class="flex items-center"><img src="https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg" alt="Avatar of Victor Ramirez" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:0" loading="lazy" decoding="async" width="24" height="24"/><img src="https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg" alt="Avatar of Mahmoud Abdelwahab" class="w-6 h-6 rounded-full overflow-hidden border-2 border-background" style="margin-left:-8px" loading="lazy" decoding="async" width="24" height="24"/></div><span class="text-sm text-gray-600">Victor Ramirez &amp; Mahmoud Abdelwahab</span><svg width="6" height="24" viewBox="0 0 6 24" fill="none" xmlns="http://www.w3.org/2000/svg"><rect x="4.16754" y="0.00866699" width="2" height="24" rx="1" transform="rotate(10 4.16754 0.00866699)" fill="currentColor" fill-opacity="0.1"></rect></svg><span class="text-sm text-gray-600">Jul 17, 2026</span></div></a></div></div><section class="post-bottom-cta my-16 border rounded-[16px] px-6 py-16 sm:px-12 sm:py-24 flex flex-col items-center justify-center text-center" aria-labelledby="post-bottom-cta-title"><h3 id="post-bottom-cta-title" class="text-h2 font-serif font-medium">Ready to get started?</h3><p class="mt-4 text-base sm:text-lg text-gray-600">Join millions of developers deploying applications on Railway</p><div class="mt-8 flex w-full flex-col items-stretch justify-center gap-3 sm:w-auto sm:flex-row"><a class="post-bottom-cta-primary inline-flex min-h-[48px] items-center justify-center rounded-[8px] px-6 py-3 text-base font-medium no-underline transition-colors duration-100" href="https://railway.com" target="_blank" rel="noreferrer noopener">Deploy a new project</a><a class="post-bottom-cta-secondary inline-flex min-h-[48px] items-center justify-center rounded-[8px] border px-6 py-3 text-base font-medium no-underline transition-colors duration-100" href="https://railway.com/enterprise" target="_blank" rel="noreferrer noopener">Book a demo</a></div></section></div></div></div><footer class="px-5 md:px-8 py-12 border-t border-gray-100"><div class="grid grid-cols-2 sm:grid-cols-12 gap-8 sm:gap-4 lg:gap-16 max-w-6xl mx-auto"><div class="hidden sm:flex col-span-3 lg:col-span-4 flex-col justify-between"><a class="underline" href="https://railway.com" target="_blank" rel="noreferrer noopener"><svg data-v-423bf9ae="" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 59.99945071644884 60" class="logo w-8"><g data-v-423bf9ae="" id="6f639fe5-02e1-4640-a1fb-f4b22b64e5ef" transform="matrix(0.5999641134746578,0,0,0.5999641134746578,-437.5538206497148,-266.68705118956507)" stroke="none"><path d="M729.872 487.327a50.86 50.86 0 0 0-.464 5.033h75.879c-.265-.518-.621-.985-.98-1.442-12.972-16.769-19.95-15.315-29.932-15.741-3.328-.137-5.585-.192-18.832-.192-7.09 0-14.798.018-22.304.038.737-1.746 1.6-3.419 2.525-5.061-2.885 5.106-4.865 10.771-5.805 16.789l.891-4.397c.007-.031.018-.063.024-.094h38.883v5.067h-39.885zM805.885 497.432h-76.438c.08 1.352.206 2.686.388 4.002h70.571c3.146 0 4.907-1.786 5.479-4.002zM733.851 515.257a52.226 52.226 0 0 1-1.98-4.997c6.608 19.89 25.328 34.251 47.433 34.251 20.205 0 37.566-12.007 45.452-29.254h-90.905zM729.38 492.915c-.018.531-.08 1.055-.08 1.589 0 .538.063 1.059.08 1.59v-3.179zM824.77 515.229z"></path><path d="M779.303 444.505c-18.682 0-34.939 10.265-43.524 25.439 6.709-.014 19.775-.022 19.775-.022h.003v-.005c15.444 0 16.018.069 19.035.195l1.868.069c6.507.217 14.505.916 20.798 5.68 3.416 2.584 8.348 8.287 11.288 12.35 2.718 3.758 3.5 8.078 1.652 12.217-1.701 3.804-5.361 6.073-9.793 6.073H730.85l-.884-4.201c.426 2.707 1.037 5.344 1.879 7.886h94.914a49.863 49.863 0 0 0 2.546-15.682c.001-27.611-22.386-49.999-50.002-49.999z"></path></g></svg></a><div class="text-xs font-medium text-gray-600 w-full">Copyright © <!-- -->2026<!-- --> Railway Corp. <br/>All rights reserved.</div></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Product</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/changelog" target="_blank" rel="noreferrer noopener">Changelog</a></li><li><a class="hover:text-foreground" href="https://railway.com/pricing" target="_blank" rel="noreferrer noopener">Pricing</a></li><li><a class="hover:text-foreground" href="https://railway.com/deploy" target="_blank" rel="noreferrer noopener">Templates</a></li><li><a class="hover:text-foreground" href="https://station.railway.com/feedback" target="_blank" rel="noreferrer noopener">Feedback</a></li><li><a class="hover:text-foreground" href="https://railway.com/open-source-kickback" target="_blank" rel="noreferrer noopener">OSS Kickback</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Company</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/about" target="_blank" rel="noreferrer noopener">About</a></li><li><a class="hover:text-foreground" href="https://railway.com/careers" target="_blank" rel="noreferrer noopener">Careers</a></li><li><a class="hover:text-foreground" href="/">Blog</a></li><li><a class="hover:text-foreground" href="https://shop.railway.com" target="_blank" rel="noreferrer noopener">Shop</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Contact</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://discord.gg/railway" target="_blank" rel="noreferrer noopener">Discord</a></li><li><a class="hover:text-foreground" href="https://twitter.com/railway" target="_blank" rel="noreferrer noopener">Twitter</a></li><li><a class="hover:text-foreground" href="https://github.com/railwayapp" target="_blank" rel="noreferrer noopener">GitHub</a></li><li><a class="hover:text-foreground" href="mailto:contact@railway.com">Email</a></li></ul></div><div class="col-span-1 sm:col-span-2"><p class="text-xs font-medium uppercase text-gray-600 mb-4">Legal</p><ul class="text-gray-600 space-y-4"><li><a class="hover:text-foreground" href="https://railway.com/legal/acceptable-use" target="_blank" rel="noreferrer noopener">Acceptable Use</a></li><li><a class="hover:text-foreground" href="https://railway.com/legal/privacy" target="_blank" rel="noreferrer noopener">Privacy Policy</a></li><li><a class="hover:text-foreground" href="https://railway.com/legal/terms" target="_blank" rel="noreferrer noopener">Terms of Service</a></li></ul></div></div></footer></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"page":{"authors":[{"avatar":{"alt":"Avatar of Charith Amarasinghe","height":null,"id":"1836","mimeType":"image/png","url":"https://cms.railway.com/media/452039afd9d5078059203ebfb4028e92717d03b94e10832c814d2ca23c5bce39.png","width":null},"avatarUrl":"https://cms.railway.com/media/452039afd9d5078059203ebfb4028e92717d03b94e10832c814d2ca23c5bce39.png","githubUrl":null,"id":"13","name":"Charith Amarasinghe","slug":"charith-amarasinghe","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":"Since the beginning, Railway’s compute has been built on top of Google Cloud Platform. The platform supported Railway's initial journey, but it has caused a multitude of problems that have posed an existential risk to our business. More importantly, building on a hyperscaler prevents us from delivering the best possible platform to our customers. \n\nIt directly affected the pricing we could offer (egress fees anyone?), limited the level of service we could deliver, and introduced engineering constraints that restricted the features we could build. \n\nAnd not only is it [rare](https://github.com/GoogleCloudPlatform/guest-agent/issues/401) that we understand [why](https://blog.railway.com/p/2023-12-01-incident-report) [things](https://x.com/JustJake/status/1667478906591666176) [break](https://blog.railway.com/p/incident-december-16-2024) upstream, but also despite multi-million dollar annual spend, we get about as much support from them as you would spending $100. \n\nSo in response, we kicked off a Railway Metal project last year. Nine months later we were live with the first site in California, having designed, spec-ed, and installed everything from the fiber optic cables in the cage to the various contracts with ISPs. We’re lighting up three more data center regions as we speak.\n\nTo deliver an “infra-less” cloud experience to our customers, we’ve needed to get good fast at building out our own physical infrastructure. That’s the topic of our blogpost today.\n\n## So you want to build a cloud\n\nFrom kicking off the Railway Metal project in January 2024, it took us five long months to get the first servers plugged in. It took us an additional three months before we felt comfortable letting our users onto the hardware (and an additional few months before we started writing about it here). \n\nThe first step was finding some space.\n\nWhen you go “on-prem” in cloud-speak, you need somewhere to put your shiny servers and reliable power to keep them running. Also you want enough cooling so they don’t melt down. \n\nIn general you have three main choices: Greenfield buildout (buying or leasing a datacenter), Cage Colocation (getting a private space inside a provider's datacenter enclosed by mesh walls), or Rack colocation (leasing individual racks or partitions of racks in a colocation datacenter). \n\nWe chose the second option: a cage to give us four walls, a secure door, and a blank slate for everything else.\n\n![A cage before any racks have been fitted](https://cms.railway.com/media/ce8d56ecaf1283f837c13bc69296d93fb8e4b1689626c3d589f2b4146957bb3d.png)\n\nThe space itself doesn’t cost much, but power (and by proxy, cooling) costs the most. Depending on the geography, the $/kW rate can vary hugely — on the US west coast for example we may pay less than half as much as we pay in Singapore. Power is paid for as a fixed monthly commit, regardless of whether its consumed or not, to guarantee it will be available on-demand.\n\nBut how much power do you need?\n\n## With great power comes great responsibility\n\nIdeally if you’ve embarked on your data center migration mission, you should have an idea of the rough amount of compute you want to deploy. We started with a target number of vCPUs, GBs of RAM, and TBs of NVMe to match our capacity on GCP. \n\nUsing these figures, we converged on a server and CPU choice. There are many knobs to turn when doing this computation — probably worth a blogpost in itself — but the single biggest factor for us was power density e.g. how do we get the compute density we want inside of a specific power draw. \n\nThe calculations aren’t as simple as summing watts though, especially with 3-phase feeds — Cloudflare has [a great blogpost](https://blog.cloudflare.com/an-introduction-to-three-phase-power-and-pdus/) covering this topic.\n\nPower is the most critical resource for data centers, and a power outage can have extremely long recovery times. So redundancy is critical, and it’s important to have two fully independent power feeds per rack. Both feeds will share load under normal operation, but the design must be resilient to a feed going down.\n\nTo deliver this power to your servers, you’ll also want a Power Distribution Unit, which you'll select based on the number of sockets and management features it provides. The basic ones are glorified extension cords, while the ones we deploy allow control and metering of individual sockets.\n\n![Each PDU is accessible over the network and individual sockets can be remotely metered and controlled](https://cms.railway.com/media/726fb15777885c0f483780040814464609209f58bebef47d2c380aa8e0fca508.png)\n\nWith that, power is now available in the cage.\n\n## Let there be light\n\nNo cloud machine is an island and that's where networks come into play. \n\nTo achieve the lowest possible latency on Railway, we need to set you up with solid connections to the rest of the world. \n\nWe look for DC facilities that are on-network with Tier 1 Internet Services Providers (ISPs), that are part of an Internet Exchange (IX), and that have available fiber to other data centers in close proximity. \n\n![Each cage gets diverse and redundant network links](https://cms.railway.com/media/0ceb2f6d1000dc97aeedd06e34033a6427a46ac0c5ad2b3fbd635915ce42210d.png)\n\nYour applications deployed to Railway will want to connect to a diverse mix of endpoints over the network — be it a home internet user in Sydney, Australia or a API hosted on an AWS server in the US. To get you the best possible latency and the lowest bandwidth cost, we contract with a mix of internet providers optimized for each use case.\n\nWe select ISPs for the maturity of their networks in each geography we target. Partnering with the wrong ISP in a region can lead to extra network hops (and thus latency) to reach specific target markets — or in the worst case — convoluted network routes. So for each region, we pick at least two separate networks based on their regional footprints. \n\nOnce connected, we receive full internet routing tables from each ISP and consolidate them on our network switches to resolve the best path for each IP prefix. If you have an end user in Australia trying to reach an app deployed to Singapore, we’ll likely hand those packets off directly to [Telstra](https://bgp.tools/as/4637#connectivity) who have one of the densest access networks in Australia. If that same app needs to send packets to a end-user or server in Japan, then we’d likely be handing them over to PCCW who peer directly with NTT in Japan and have a [dense footprint in APAC](https://www.pccwglobal.com/wp-content/uploads/2019/11/world_fold_20191115.pdf).\n\n\u003caside\u003e\n👉\n\nPeering information is public, head-over to [bgp.tools](https://bgp.tools/) to see how your favorite networks interconnect.\n\n\u003c/aside\u003e\n\nFor redundancy we’re building out multiple zones in each region, and interconnectivity between these sites is also critical for our expansion. There are several tools such as dark fiber or wavelength services that we look for to plan this expansion. The result is that your apps won’t notice if your database is in the same room or if it’s 4 blocks over in a neighboring building — this is a feature not a bug — as it builds resilience against the failure of an individual data center.\n\n...\n\nOk, now that you’ve found a space you like, signed a deal with a data center, and signed deals with several ISPs, you're all-systems-go to install some servers, right? \n\nWell, not exactly. First you need a bunch of other things to give your server a nice snug home to warm up in.\n\n## Aisles, racks and overhead infrastructure\n\nIn a data center, racks are arranged in rows, and the space between racks, the aisle, is used for airflow. \n\nThe Cold Aisle is where cold air is blown in from the DC facility, and servers in your rack suck this air up and exhaust it towards the rear onto the Hot Aisle. The DC facility will remove this hot air from the Hot Aisle. For optimum efficiency, you don’t want air between these aisles to mix.\n\n![A cage schematic - all equipment must be mounted such that fans blow air from the cold aisle to the hot](https://cms.railway.com/media/de2d850903b61c0498252628043c4644a3a6c279cf40a4d338dcbb4e567cc032.png)\n\nThe racks themselves have some variability, even if you opt to use conventional 19\" wide equipment. You can select the height, width, and depth to suit your equipment and cabling needs. \n\nMost server equipment can slide on rails to allow for easy maintenance, so it’s important to ensure that cage dimensions allow for this. Cabling and cable management also requires some space, so there’s a tradeoff to be made with how crowded you want each rack to be vs. how many racks you can fit into a cage. \n\nIn our experience, power and cooling is often the limiting factor rather than the actual space available. In newer sites we opt for wider 800mm racks to allow for better airflow by getting cables out of the way of the exhausts.\n\n![A rack-mount server on rails can be pulled out and serviced in place, each weighs nearly 20kg…](https://cms.railway.com/media/4405868f310c22342375a9a0b89e15141dc2fe431e79363c1927fc5c9f8ad361.jpeg)\n\nIn addition to racks, you'll need several bits of infrastructure to get power and data to your racks. This will likely involve installing some overhead infrastructure and trays that let you route fiber cables from the edge of your cage to each of your racks, and to route cables between racks. This is something the DC operator will throw in when quoting the cage.\n\nDepending on your design, you’ll want to optimize for short cable paths by ensuring your overhead infrastructure, rack local cabling, and device orientations align. Because our racks have dense switch-to-server fiber cabling in each rack, we buy switches that have their ports oriented to the back of the rack (these are called reverse airflow switches because they exhaust air on the side with the network ports). \n\nThis allows us to align the cable trays such that all cabling happens on one side of the rack and there’s no zig-zagging of cables between the front and back of the rack.\n\n![A ladder rack and fibre cable tray allow routing cables to racks from above](https://cms.railway.com/media/f5ccc298f7c705fe4a62a29ec9cae38c20ad382215943ddf9e78afbbec575c8c.jpeg)\n\nSo you’ve got the space, signed up ISPs, ordered the hardware, got the racks, and a pretty good picture on how to lay it all out. But it’s still a pretty expensive lego set sitting in the loading bay of a data center. To assemble it you now need to leverage the most versatile programming tool ever devised in the history of mankind … Microsoft Excel.\n\n## The rack and stack\n\nLet’s first step back and publish a disclaimer: neat and organized cabling requires a lot of practice; we tried it ourselves first - with… *mixed…* results. \n\n![Our DIY attempts at cabling to get the first server online vs. what our installation partner put together](https://cms.railway.com/media/e2548904f65a867960c50b2001b723b208b4abe8e20f95f18c6e86382c7aa99f.png)\n\nTo install it properly, we bring in professionals, but the professionals need to know what to install. A comprehensive documentation pack is essential. A cabling matrix and rack elevation are common documents that communicate to contractors how to rack and wire-up servers.\n\nA cabling matrix describes the termination of each cable, specifying the device position and port for each side of the connection, along with the specification of the cable itself (type of fiber, length, etc). The rack elevation is a visual representation of the rack itself, showing the position and orientation of each device.\n\n![The cabling matrix for our first install - 300+ cables all manually entered and cross-checked](https://cms.railway.com/media/367c1ecfc1b4fe721b06d71ee2b82cfead5e65133e3815985d72f4c968fbcb16.png)\n\nThe documentation exercise can be intense, each of our install phases involved 60+ devices, 300+ discrete cables, and dozens of little details. This was all handcrafted into written specifications and spreadsheets we used as a basis for the installation and commissioning. From the materials being on site to getting everything installed takes us about 6-14 days.\n\n![We have now built internal tooling to automate generating build specifications](https://cms.railway.com/media/a15e16dd1983bce5d2da56e69a5c96efdc3060178088deb62d227a04ff533223.png)\n\nThis all seems very far removed from software, DevOps, or what you’d typically think of as “infrastructure,” and that is very true — building a datacenter cage is probably closer to building a house than to deploying a Terraform stack. \n\nTo compound this, every datacenter facility, contractor and vendor will do things slightly differently, even within the same organization. The operational aspect requires you to stay on your toes and be extremely detail-oriented. \n\nSome what-the-duck moments we’ve had thus far:\n\n- Contractor: “We need longer power cables” - the PDUs at that site were upside down because the power came in from the floor, so our socket numbering was reversed in the plan\n- Phonecall from Amsterdam: “There’s no demarcation point at the site?” - a specific facility installs external fibre links direct to a box in one of our racks rather than via a dedicated demarcation point overhead\n- Railway Discord quote: “Why are the phases wired so weirdly on this PDU?” - the facility was wired differently to our other sites and the power sockets were wired phase-to-neutral vs. phase-to-phase (WYE vs Delta circuits for you EE’s)\n- Contractor: “Your data cables are too short” - the contractor didn’t realize the network gear was reverse-airflow and tried to mount things the wrong way around\n- Us raising a support ticket: “There’s no link coming up on this cable” - the fiber was wired in the wrong polarity; we learnt what “rolling fibre cables” was that day… it’s when they rip out the plugs from the [LC connector](https://www.fs.com/uk/blog/lc-fiber-optics-a-comprehensive-guide-2684.html) and swap them around\n- Railway Discord quote: “I brought a rubber mallet from HomeDepot today” - a batch of nearly 24 PDUs from one vendor were delivered with faulty sockets that didn’t properly engage with the power plugs, even with ~~appropriate~~ extreme mechanical force being applied\n\nBut from this point - the hardware is in place the task begins to feel more familiar; we’re now needing to do some BGP, install some OS’es, setup monitoring and bring everything up.\n\n![The completed cage ready for configuration](https://cms.railway.com/media/2ed17ac47150832aae6332e806daa1a6d380cb1b7b902f7715957e89968da5c7.jpeg)\n\n## Pedal on Metal\n\nThe installed cage is but a blank canvas, the network devices need configuring, router config needs writing, RIR (regional internet registry) records need updating, and we must interact with the likes of [Redfish APIs](https://www.dmtf.org/standards/redfish) (HTTP APIs to dedicated controllers on server motherboards and PDUs) and PXE (a protocol to boot servers over the network) to get everything up and running. \n\nWe've also not discussed how networking works. Our design uses [FRR](https://frrouting.org/) and whitebox network switches running [SONiC](https://sonicfoundation.dev/) to build a L3-only software-driven network that deeply integrates with our control plane. \n\nWe’ve regaled you with tales from the frontline … but any more and you’d be here all day. \n\nIn a future post, we’ll discuss how we go from a bunch of servers in a room to a functional Railway zone. In the space of the last few months we’ve built two new software tools, *Railyard* and *MetalCP*, to enable a button click experience from designing a new cage, tracing and visualizing the cabling, to installing OSes on servers and getting them on the internet.\n\nUntil then, if any of this excites you, check out our [open Infrastructure Engineering roles](https://railway.com/careers/infra-platform) and reach out if they catch your interest.","createdAt":"2026-06-09T12:22:08.252Z","description":"When it comes to infrastructure engineering, building a data center is probably closer to building a house than to deploying a Terraform stack.","externalAuthor":false,"featured":false,"featuredImage":{"alt":"So You Want to Build Your Own Data Center featured image","height":null,"id":"630","mimeType":"image/png","url":"https://cms.railway.com/media/33dfbc6ee42a00e46203cd1fa7045249ad7b48e294f95c86201dea687ec7f9d5.png","width":null},"id":"146","publishedAt":"2025-01-17T00:00:00.000Z","seoDescription":"When it comes to infrastructure engineering, building a data center is probably closer to building a house than to deploying a Terraform stack.","seoTitle":"So You Want to Build Your Own Data Center","slug":"data-center-build-part-one","socialImage":{"alt":"So You Want to Build Your Own Data Center social image","height":null,"id":"633","mimeType":"image/png","url":"https://cms.railway.com/media/d3356891ab8c76de1c7872dfb46c3ace7de5eb6b478d19e2f317d71b15710567.png","width":null},"title":"So You Want to Build Your Own Data Center","updatedAt":"2025-01-17T00:00:00.000Z"},"relatedPosts":[{"authors":[{"avatar":{"alt":"Mahmoud Abdelwahab headshot","height":460,"id":"1846","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","width":460},"avatarUrl":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","githubUrl":null,"id":"4","name":"Mahmoud Abdelwahab","slug":"mahmoud-abdelwahab","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":null,"createdAt":"2026-08-06T08:07:06.228Z","description":"Announcing: dev.new, an agent that can build, preview, test, and deploy an app from the same place. We tested the whole loop by asking it to ship a real-time multiplayer FPS game that runs in the browser","externalAuthor":false,"featured":true,"featuredImage":{"alt":"dev.new featured image","height":1440,"id":"1983","mimeType":"image/png","url":"https://cms.railway.com/media/dev.new-featured-image-4.png","width":3264},"id":"289","publishedAt":"2026-08-13T08:07:00.000Z","seoDescription":null,"seoTitle":null,"slug":"dev-new","socialImage":{"alt":"dev.new featured image 1","height":720,"id":"1976","mimeType":"image/png","url":"https://cms.railway.com/media/dev.new-featured-image-3.png","width":1632},"title":"Your Cloud should come with an app builder ","updatedAt":"2026-08-14T20:19:27.322Z"},{"authors":[{"avatar":{"alt":"Avatar of Victor Ramirez","height":null,"id":"1855","mimeType":"image/jpeg","url":"https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg","width":null},"avatarUrl":"https://cms.railway.com/media/226dbd7dc83c91b74e5849e780b6946ce1ce1165d8dd849498d4cb31d2e09a10.jpg","githubUrl":null,"id":"34","name":"Victor Ramirez","slug":"victor-ramirez","title":null},{"avatar":{"alt":"Mahmoud Abdelwahab headshot","height":460,"id":"1846","mimeType":"image/jpeg","url":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","width":460},"avatarUrl":"https://cms.railway.com/media/person-mahmoud-abdelwahab-284bce9137a0.jpeg","githubUrl":null,"id":"4","name":"Mahmoud Abdelwahab","slug":"mahmoud-abdelwahab","title":null}],"category":{"description":null,"id":"4","order":3,"seoDescription":null,"seoTitle":null,"showInNavigation":true,"slug":"engineering","title":"Engineering","visible":true},"content":null,"createdAt":"2026-07-15T22:26:48.036Z","description":"Merge now, release when you're ready: turn a feature on for your team, then 10% of users, then everyone. Feature flags are now built into every Railway project, and your agents can run the rollout from the CLI, SDK, and MCP. No third-party service required.","externalAuthor":false,"featured":false,"featuredImage":{"alt":"feature flags featured image","height":1440,"id":"1968","mimeType":"image/png","url":"https://cms.railway.com/media/feature-flags-featured-image.png","width":3264},"id":"278","publishedAt":"2026-07-17T07:00:00.000Z","seoDescription":null,"seoTitle":null,"slug":"feature-flags","socialImage":null,"title":"Roll it out, roll it back, never redeploy","updatedAt":"2026-08-13T16:53:55.126Z"}]},"__N_SSG":true},"page":"/p/[slug]","query":{"slug":"data-center-build-part-one"},"buildId":"nFX7R3M0MdLMAf-Ufh9qa","isFallback":false,"isExperimentalCompile":false,"gsp":true,"scriptLoader":[]}</script></body></html>