sreweekly: 528 期数据 + 全文抓取(articles/pages + markdown 正文扩充)

This commit is contained in:
2026-09-10 20:52:28 +08:00
parent f715edd2ef
commit 6f956f9c08
100 changed files with 38563 additions and 5 deletions

View File

@@ -0,0 +1,137 @@
<!DOCTYPE html><html lang="en" data-astro-cid-37fxchfa> <head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><meta name="color-scheme" content="light dark"><title>Storage at scale: what I actually watched | Sridhar Rajarao</title><meta name="description" content="For eight years I ran SRE for a storage system measured in exabytes. The dashboard I checked every morning shrank to seven numbers. Here they are."><meta name="author" content="Sridhar Rajarao"><link rel="canonical" href="https://sridharrajarao.com/blog/storage-at-scale/"><!-- Open Graph --><meta property="og:type" content="article"><meta property="og:title" content="Storage at scale: what I actually watched | Sridhar Rajarao"><meta property="og:description" content="For eight years I ran SRE for a storage system measured in exabytes. The dashboard I checked every morning shrank to seven numbers. Here they are."><meta property="og:url" content="https://sridharrajarao.com/blog/storage-at-scale/"><meta property="og:image" content="https://sridharrajarao.com/og/storage-at-scale.png"><meta property="og:image:width" content="1200"><meta property="og:image:height" content="630"><meta property="og:image:alt" content="Sridhar Rajarao — For eight years I ran SRE for a storage system measured in exabytes. The dashboard I checked every morning shrank to seven numbers. Here they are."><meta property="og:site_name" content="Sridhar Rajarao"><meta property="og:locale" content="en_US"><meta property="article:published_time" content="2026-05-28T00:00:00.000Z"><meta property="article:author" content="Sridhar Rajarao"><meta property="article:tag" content="sre"><meta property="article:tag" content="storage"><meta property="article:tag" content="reliability"><meta property="article:tag" content="metrics"><!-- Twitter / X --><meta name="twitter:card" content="summary_large_image"><meta name="twitter:title" content="Storage at scale: what I actually watched | Sridhar Rajarao"><meta name="twitter:description" content="For eight years I ran SRE for a storage system measured in exabytes. The dashboard I checked every morning shrank to seven numbers. Here they are."><meta name="twitter:image" content="https://sridharrajarao.com/og/storage-at-scale.png"><meta name="twitter:image:alt" content="Sridhar Rajarao — For eight years I ran SRE for a storage system measured in exabytes. The dashboard I checked every morning shrank to seven numbers. Here they are."><!-- Icons --><link rel="icon" type="image/svg+xml" href="/favicon.svg"><!-- Feed and sitemap discovery --><link rel="alternate" type="application/rss+xml" title="Sridhar Rajarao" href="/rss.xml"><link rel="sitemap" type="application/xml" title="Sitemap" href="/sitemap-index.xml"><!-- JSON-LD structured data --><script type="application/ld+json">{"@context":"https://schema.org","@type":"BlogPosting","headline":"Storage at scale: what I actually watched","description":"For eight years I ran SRE for a storage system measured in exabytes. The dashboard I checked every morning shrank to seven numbers. Here they are.","image":"https://sridharrajarao.com/og/storage-at-scale.png","datePublished":"2026-05-28T00:00:00.000Z","dateModified":"2026-05-28T00:00:00.000Z","author":{"@type":"Person","name":"Sridhar Rajarao","url":"https://sridharrajarao.com/about/","sameAs":["https://www.linkedin.com/in/sridharrajarao/"]},"publisher":{"@type":"Person","name":"Sridhar Rajarao","url":"https://sridharrajarao.com/"},"mainEntityOfPage":{"@type":"WebPage","@id":"https://sridharrajarao.com/blog/storage-at-scale/"},"keywords":"sre, storage, reliability, metrics","url":"https://sridharrajarao.com/blog/storage-at-scale/"}</script><!-- Security: opt out of legacy referrer leakage --><meta name="referrer" content="strict-origin-when-cross-origin"><!-- Theme bootstrap: read stored preference and apply before paint
to prevent flash of unstyled (wrong-theme) content. --><script>
(function () {
try {
var stored = localStorage.getItem('theme');
if (stored === 'light' || stored === 'dark') {
document.documentElement.setAttribute('data-theme', stored);
}
} catch (_) {}
})();
</script><link rel="stylesheet" href="/_astro/about.gLv2NvM3.css">
<style>.share-row[data-astro-cid-ung7bpju]{display:flex;align-items:center;gap:1.25rem;margin:3rem 0 1rem;padding-top:1.5rem;border-top:1px solid var(--border);flex-wrap:wrap;font-size:.85rem}.share-label[data-astro-cid-ung7bpju]{font-family:var(--font-mono);font-size:.7rem;color:var(--fg-subtle);letter-spacing:.16em;text-transform:uppercase}.share-row[data-astro-cid-ung7bpju] a[data-astro-cid-ung7bpju],.share-row[data-astro-cid-ung7bpju] .share-copy[data-astro-cid-ung7bpju]{color:var(--fg-muted);background:none;border:0;padding:0;font:inherit;font-size:.85rem;cursor:pointer;text-decoration:none;transition:color .15s var(--ease)}.share-row[data-astro-cid-ung7bpju] a[data-astro-cid-ung7bpju]:hover,.share-row[data-astro-cid-ung7bpju] .share-copy[data-astro-cid-ung7bpju]:hover,.share-row[data-astro-cid-ung7bpju] .share-copy[data-astro-cid-ung7bpju].copied{color:var(--accent)}
.newsletter[data-astro-cid-motrwrji]{display:grid;grid-template-columns:minmax(0,.8fr) minmax(20rem,1fr);gap:2.5rem;align-items:center;padding:clamp(1.5rem,4vw,2.5rem);background:var(--bg-elevated);border:1px solid var(--border-strong)}.newsletter[data-astro-cid-motrwrji] h2[data-astro-cid-motrwrji]{font-size:clamp(1.4rem,2.5vw,2rem);margin:.35rem 0 .6rem}.newsletter-intro[data-astro-cid-motrwrji]>p[data-astro-cid-motrwrji]:last-child{color:var(--fg-muted);max-width:24rem}.newsletter-form[data-astro-cid-motrwrji]{min-width:0}@media(max-width:700px){.newsletter[data-astro-cid-motrwrji]{grid-template-columns:1fr;gap:1.5rem}}
</style>
<link rel="stylesheet" href="/_astro/about.Dd-w852w.css">
<style>.post-page[data-astro-cid-7jjqptxk]{max-width:56rem}.post-header[data-astro-cid-7jjqptxk]{padding-block:clamp(3rem,6vw,5rem) 2rem}.post-header[data-astro-cid-7jjqptxk] .eyebrow[data-astro-cid-7jjqptxk]{margin-bottom:1rem;color:var(--fg-subtle);letter-spacing:.14em}.tags[data-astro-cid-7jjqptxk]{color:var(--accent)}.post-title[data-astro-cid-7jjqptxk]{font-family:var(--font-display);font-size:clamp(2.5rem,6vw,4.5rem);font-weight:900;line-height:.98;letter-spacing:-.035em;margin-bottom:1.25rem;text-transform:none;color:var(--fg)}.kicker[data-astro-cid-7jjqptxk]{color:var(--fg)}.updated[data-astro-cid-7jjqptxk]{margin-top:.75rem;font-size:.85rem}.post-body[data-astro-cid-7jjqptxk]{margin-inline:0;max-width:40rem}.post-newsletter[data-astro-cid-7jjqptxk]{margin-top:3.5rem;max-width:40rem}.back[data-astro-cid-7jjqptxk]{margin-top:3rem;padding-top:2rem;border-top:1px solid var(--border)}.back[data-astro-cid-7jjqptxk] a[data-astro-cid-7jjqptxk]{color:var(--fg-muted);font-size:.85rem;letter-spacing:.04em;transition:color .15s var(--ease)}.back[data-astro-cid-7jjqptxk] a[data-astro-cid-7jjqptxk]:hover{color:var(--accent)}.related[data-astro-cid-7jjqptxk]{margin-top:3.5rem;padding-top:2rem;border-top:1px solid var(--border);max-width:40rem}.related[data-astro-cid-7jjqptxk] h2[data-astro-cid-7jjqptxk]{font-size:clamp(1.5rem,3vw,2rem);margin:.4rem 0 1rem}.related-list[data-astro-cid-7jjqptxk]{display:grid;border-top:1px solid var(--border)}.related-list[data-astro-cid-7jjqptxk] a[data-astro-cid-7jjqptxk]{display:grid;gap:.25rem;padding:1rem 0;border-bottom:1px solid var(--border);color:var(--fg)}.related-list[data-astro-cid-7jjqptxk] a[data-astro-cid-7jjqptxk]:hover span[data-astro-cid-7jjqptxk]{color:var(--accent)}.related-list[data-astro-cid-7jjqptxk] small[data-astro-cid-7jjqptxk]{color:var(--fg-subtle);font-size:.75rem}
</style></head> <body data-astro-cid-37fxchfa> <a class="skip-link" href="#main" data-astro-cid-37fxchfa>Skip to content</a> <header class="site-header" data-astro-cid-3ef6ksr2> <div class="container header-inner" data-astro-cid-3ef6ksr2> <a href="/" class="brand" aria-label="Home" data-astro-cid-3ef6ksr2> <span class="wordmark" data-astro-cid-3ef6ksr2> <span class="wordmark-mark" data-astro-cid-3ef6ksr2>SR</span> <span class="wordmark-dot" aria-hidden="true" data-astro-cid-3ef6ksr2></span> </span> </a> <button id="nav-toggle" class="nav-toggle" aria-label="Open menu" aria-expanded="false" data-astro-cid-3ef6ksr2> <span class="bar" data-astro-cid-3ef6ksr2></span><span class="bar" data-astro-cid-3ef6ksr2></span><span class="bar" data-astro-cid-3ef6ksr2></span> </button> <nav aria-label="Primary" id="primary-nav" data-astro-cid-3ef6ksr2> <ul data-astro-cid-3ef6ksr2> <li data-astro-cid-3ef6ksr2> <a href="/blog/" class="active" data-astro-cid-3ef6ksr2>Writing</a> </li><li data-astro-cid-3ef6ksr2> <a href="/work/" data-astro-cid-3ef6ksr2>Work</a> </li><li data-astro-cid-3ef6ksr2> <a href="/about/" data-astro-cid-3ef6ksr2>About</a> </li><li data-astro-cid-3ef6ksr2> <a href="/contact/" data-astro-cid-3ef6ksr2>Contact</a> </li> <li class="theme-cell" data-astro-cid-3ef6ksr2><button id="theme-toggle" type="button" aria-label="Toggle color theme" title="Toggle theme" data-astro-cid-x3pjskd3> <svg class="icon-sun" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" data-astro-cid-x3pjskd3> <circle cx="12" cy="12" r="4" data-astro-cid-x3pjskd3></circle> <path d="M12 2v2" data-astro-cid-x3pjskd3></path><path d="M12 20v2" data-astro-cid-x3pjskd3></path> <path d="m4.93 4.93 1.41 1.41" data-astro-cid-x3pjskd3></path><path d="m17.66 17.66 1.41 1.41" data-astro-cid-x3pjskd3></path> <path d="M2 12h2" data-astro-cid-x3pjskd3></path><path d="M20 12h2" data-astro-cid-x3pjskd3></path> <path d="m6.34 17.66-1.41 1.41" data-astro-cid-x3pjskd3></path><path d="m19.07 4.93-1.41 1.41" data-astro-cid-x3pjskd3></path> </svg> <svg class="icon-moon" width="18" height="18" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true" data-astro-cid-x3pjskd3> <path d="M21 12.79A9 9 0 1 1 11.21 3 7 7 0 0 0 21 12.79z" data-astro-cid-x3pjskd3></path> </svg> </button> <script>
(function () {
const btn = document.getElementById('theme-toggle');
if (!btn) return;
btn.addEventListener('click', function () {
const root = document.documentElement;
const current = root.getAttribute('data-theme');
const systemDark = window.matchMedia('(prefers-color-scheme: dark)').matches;
const effective = current || (systemDark ? 'dark' : 'light');
const next = effective === 'dark' ? 'light' : 'dark';
root.setAttribute('data-theme', next);
try { localStorage.setItem('theme', next); } catch (_) {}
});
})();
</script></li> </ul> </nav> </div> </header> <script>
(function () {
const btn = document.getElementById('nav-toggle');
const nav = document.getElementById('primary-nav');
if (!btn || !nav) return;
btn.addEventListener('click', function () {
const open = nav.classList.toggle('open');
btn.setAttribute('aria-expanded', open ? 'true' : 'false');
btn.setAttribute('aria-label', open ? 'Close menu' : 'Open menu');
});
})();
</script> <main id="main" data-astro-cid-37fxchfa> <article class="container post-page" data-astro-cid-7jjqptxk> <header class="post-header section" data-astro-cid-7jjqptxk> <p class="eyebrow" data-astro-cid-7jjqptxk> <time datetime="2026-05-28T00:00:00.000Z" data-astro-cid-7jjqptxk>May 28, 2026</time> <span class="tags" data-astro-cid-7jjqptxk>
&nbsp;&middot;&nbsp;
<span class="tag" data-astro-cid-7jjqptxk>sre</span><span class="tag" data-astro-cid-7jjqptxk>, storage</span><span class="tag" data-astro-cid-7jjqptxk>, reliability</span><span class="tag" data-astro-cid-7jjqptxk>, metrics</span> </span> </p> <hr class="accent-rule" data-astro-cid-7jjqptxk> <h1 class="post-title" data-astro-cid-7jjqptxk>Storage at scale: what I actually watched</h1> <p class="kicker" data-astro-cid-7jjqptxk>For eight years I ran SRE for a storage system measured in exabytes. The dashboard I checked every morning shrank to seven numbers. Here they are.</p> </header> <div class="prose post-body" data-astro-cid-7jjqptxk> <p>For eight years I ran the SRE team behind a storage system measured in exabytes. Over time, the dashboard I checked every morning shrank to a handful of numbers. These are the seven that told me whether the service was healthy.</p>
<blockquote>
<p>Availability tells you if the system is up. Durability tells you if your data is still there. The two are not the same.</p>
</blockquote>
<p>Here’s the short version.</p>
<table><thead><tr><th>KPI</th><th>What it measures</th><th>How we tracked it</th></tr></thead><tbody><tr><td>Availability</td><td>Percent of requests succeeding</td><td>99.99% per region, per service</td></tr><tr><td>Durability</td><td>Probability your data survives</td><td>11 nines (10^-11 annual loss)</td></tr><tr><td>TTFB</td><td>Time to first byte returned</td><td>p50, p95, p99 latency per object size</td></tr><tr><td>Canaries</td><td>Synthetic test traffic</td><td>Continuous PUT/GET from every region</td></tr><tr><td>Hotspots</td><td>Skew across storage nodes</td><td>Top-N node load vs cluster median</td></tr><tr><td>IOPS</td><td>Operations per second</td><td>Read/write IOPS per shard, per disk</td></tr><tr><td>DB Shards</td><td>Metadata partition health</td><td>Shard CPU, lag, hot-key skew</td></tr></tbody></table>
<h2 id="availability-and-durability-are-the-two-non-negotiables">Availability and durability are the two non-negotiables</h2>
<p>Availability is uptime. Durability is whether the data survives. You can be 100% available and lose data, you can be 100% durable and offline. Customers care about both. We hit 11 nines of durability by writing every object to multiple availability domains with erasure coding, and proved it monthly with a recovery drill.</p>
<h2 id="ttfb-is-what-users-actually-feel">TTFB is what users actually feel</h2>
<p>Aggregate availability hides slow tails. A 99.99% available service with a p99 TTFB of 2 seconds feels broken. Always track latency by object size bucket. A 10 MB read should not share an SLO with a 100 byte HEAD.</p>
<h2 id="canaries-are-your-truth">Canaries are your truth</h2>
<p>Customers don’t tell you when they’re sad. They leave. Canaries are synthetic PUT/GET/LIST traffic running continuously from every region. If a canary fails for 30 seconds, you find out before your customer’s pager goes off.</p>
<h2 id="hotspots-and-iops-surface-the-silent-failures">Hotspots and IOPS surface the silent failures</h2>
<p>A storage cluster can be 99.99% available while one node is on fire. Track per-node IOPS and bytes-served, and alert on the top-N nodes diverging from cluster median. Hotspots are the leading indicator of a customer key range overwhelming a shard.</p>
<h2 id="db-shards-are-the-part-nobody-talks-about">DB shards are the part nobody talks about</h2>
<p>Object storage looks stateless, but the metadata layer is a sharded database. One hot shard, one rebalance gone wrong, and your control plane stalls. Watch shard CPU, replication lag, and hot-key skew the same way you watch the data plane.</p>
<blockquote>
<p>The data plane scales. The control plane bites.</p>
</blockquote>
<p>Those seven numbers, watched together, told me almost everything I needed to know about whether the service was healthy.</p> </div> <div class="share-row" data-astro-cid-ung7bpju> <span class="share-label" data-astro-cid-ung7bpju>Share</span> <a href="https://www.linkedin.com/sharing/share-offsite/?url=https%3A%2F%2Fsridharrajarao.com%2Fblog%2Fstorage-at-scale%2F" target="_blank" rel="noopener noreferrer" data-astro-cid-ung7bpju>LinkedIn</a> <a href="https://twitter.com/intent/tweet?url=https%3A%2F%2Fsridharrajarao.com%2Fblog%2Fstorage-at-scale%2F&text=Storage%20at%20scale%3A%20what%20I%20actually%20watched" target="_blank" rel="noopener noreferrer" data-astro-cid-ung7bpju>X</a> <a href="https://news.ycombinator.com/submitlink?u=https%3A%2F%2Fsridharrajarao.com%2Fblog%2Fstorage-at-scale%2F&t=Storage%20at%20scale%3A%20what%20I%20actually%20watched" target="_blank" rel="noopener noreferrer" data-astro-cid-ung7bpju>Hacker News</a> <button type="button" class="share-copy" data-url="https://sridharrajarao.com/blog/storage-at-scale/" data-astro-cid-ung7bpju>Copy link</button> </div> <script>
(function () {
document.querySelectorAll('.share-copy').forEach(function (btn) {
btn.addEventListener('click', async function (e) {
var el = e.currentTarget;
var url = el.getAttribute('data-url');
if (!url) return;
try {
await navigator.clipboard.writeText(url);
} catch (_) {
// Fallback for older browsers
var ta = document.createElement('textarea');
ta.value = url;
document.body.appendChild(ta);
ta.select();
try { document.execCommand('copy'); } catch (_) {}
document.body.removeChild(ta);
}
var original = el.textContent;
el.textContent = 'Copied!';
el.classList.add('copied');
setTimeout(function () {
el.textContent = original;
el.classList.remove('copied');
}, 1500);
});
});
})();
</script> <aside class="post-newsletter" aria-label="Newsletter signup" data-astro-cid-7jjqptxk> <section class="newsletter compact" id="newsletter" aria-labelledby="newsletter-title" data-astro-cid-motrwrji> <div class="newsletter-intro" data-astro-cid-motrwrji> <p class="eyebrow" data-astro-cid-motrwrji>Newsletter</p> <h2 id="newsletter-title" data-astro-cid-motrwrji>One useful note a week.</h2> <p data-astro-cid-motrwrji>Practical writing on systems, operations, and AI. No noise.</p> </div> <div class="newsletter-form" data-astro-cid-motrwrji> <script async data-uid="64b9153b21" src="https://sridhar-rajarao.kit.com/64b9153b21/index.js"></script> </div> </section> </aside> <aside class="related" aria-label="Related writing" data-astro-cid-7jjqptxk> <p class="eyebrow" data-astro-cid-7jjqptxk>Keep reading</p> <h2 data-astro-cid-7jjqptxk>Related writing</h2> <div class="related-list" data-astro-cid-7jjqptxk> <a href="/blog/why-github-feels-less-reliable/" data-astro-cid-7jjqptxk> <span data-astro-cid-7jjqptxk>Why GitHub feels less reliable lately</span> <small data-astro-cid-7jjqptxk>sre · reliability</small> </a><a href="/blog/tls-rotation-observability/" data-astro-cid-7jjqptxk> <span data-astro-cid-7jjqptxk>Your TLS rotation is not reliable until production proves it</span> <small data-astro-cid-7jjqptxk>sre · reliability</small> </a><a href="/blog/openai-15-tbps-session-1/" data-astro-cid-7jjqptxk> <span data-astro-cid-7jjqptxk>How we shipped 15 Tbps for OpenAI in 90 days (Session 1 of 3)</span> <small data-astro-cid-7jjqptxk>sre · storage</small> </a> </div> </aside> <p class="back" data-astro-cid-7jjqptxk><a href="/blog/" data-astro-cid-7jjqptxk>&larr; All writing</a></p> </article> </main> <footer class="site-footer" data-astro-cid-sz7xmlte> <div class="container footer-inner" data-astro-cid-sz7xmlte> <div class="foot-brand" data-astro-cid-sz7xmlte> <p class="foot-mark" data-astro-cid-sz7xmlte>Sridhar Rajarao<span class="foot-dot" data-astro-cid-sz7xmlte>.</span></p> <p class="foot-tag" data-astro-cid-sz7xmlte>Platform engineering and reliability at hyperscale.</p> </div> <div class="foot-links" data-astro-cid-sz7xmlte> <a href="https://www.linkedin.com/in/sridharrajarao/" target="_blank" rel="me noopener noreferrer" data-astro-cid-sz7xmlte>LinkedIn</a> <a href="/blog/#newsletter" data-astro-cid-sz7xmlte>Newsletter</a> <a href="/start-here/" data-astro-cid-sz7xmlte>Start here</a> <a href="/journal/" data-astro-cid-sz7xmlte>Journal</a> <a href="/rss.xml" data-astro-cid-sz7xmlte>RSS</a> <a href="/contact/" data-astro-cid-sz7xmlte>Contact</a> </div> </div> <div class="container foot-copy" data-astro-cid-sz7xmlte> <p class="muted" data-astro-cid-sz7xmlte>&copy; 2026 Sridhar Rajarao &middot; Pleasanton, CA</p> </div> </footer> <script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" data-cf-beacon='{"version":"2024.11.0","token":"6784c32cf7bb4fd1ab1f17a00678a0f0","r":1,"spa":2}' crossorigin="anonymous"></script>
</body> </html>