Files
nexus/sreweekly/articles/268/03-seeing-like-an-sre-site-reliability-engineering-as-high-modernism.html
2026-09-12 17:23:01 +08:00

228 lines
35 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en" dir="ltr" prefix="og: http://ogp.me/ns#">
<head>
<meta charset="utf-8" />
<link rel="shortcut icon" href="https://www.usenix.org/themes/motherboard/favicon.ico" type="image/vnd.microsoft.icon" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta content="general" name="rating" />
<meta content="Drupal 7 (http://drupal.org)" name="generator" />
<link rel="canonical" href="https://www.usenix.org/publications/loginonline/seeing-sre-site-reliability-engineering-high-modernism" />
<link rel="shortlink" href="https://www.usenix.org/node/273658" />
<meta content="website" property="og:type" />
<meta content="USENIX" property="og:site_name" />
<meta content="Seeing Like an SRE: Site Reliability Engineering as High Modernism" property="og:title" />
<meta content="https://www.usenix.org/publications/loginonline/seeing-sre-site-reliability-engineering-high-modernism" property="og:url" />
<meta content="2023-02-08T16:09:06-08:00" property="og:updated_time" />
<meta content="https://www.usenix.org/sites/default/files/usenix_2025_og_1200x630.png" property="og:image" />
<meta content="https://www.usenix.org/sites/default/files/usenix_2025_og_1200x630.png" property="og:image:url" />
<meta content="https://www.usenix.org/sites/default/files/usenix_2025_og_1200x630.png" property="og:image:secure_url" />
<meta content="image/png" property="og:image:type" />
<meta content="2021-04-20T19:00:00-07:00" property="article:published_time" />
<meta content="2023-02-08T16:09:06-08:00" property="article:modified_time" />
<title>Seeing Like an SRE: Site Reliability Engineering as High Modernism | USENIX</title>
<link rel="stylesheet" href="https://www.usenix.org/sites/default/files/css/css_nCq0Tqg9eqZPa5fXfe0k9wAX8jnxbXlZ75LfGb5rq7A.css" media="all" />
<link rel="stylesheet" href="https://www.usenix.org/sites/default/files/css/css_2i5hNf7GrbmdIRSKLAmKbqqhLsXPdxtef5MzSaLSGkY.css" media="all" />
<style media="all">#backtotop{bottom:20px;right:20px;}
</style>
<link rel="stylesheet" href="https://www.usenix.org/sites/default/files/css/css_K9stUlVccdAb6O9D60Lwdlif4rvmqiwmXCQESGsrHis.css" media="all" />
<link rel="stylesheet" href="https://www.usenix.org/sites/default/files/css/css_qFgpkJD9LCbzFq1MQrtO4IkZu4u0bPvQ41XA-NJVJPM.css" media="all and (min-width: 60em)" />
<link rel="stylesheet" href="https://www.usenix.org/sites/default/files/css/css_B99K36ryG5rmvImk01aGYNU6pRjl9ax1F1V5MSwJ1l0.css" media="all" />
<link rel="stylesheet" href="https://www.usenix.org/sites/default/files/css/css_cU3KfR4jRp9zSJ4c3qKFtdvl9cZqjJmX6xjYh4peo_E.css" media="all and (min-width: 60em)" />
<link rel="stylesheet" href="https://www.usenix.org/sites/default/files/css/css_gKxwQGISVq7A4HqYbamWQZ1bWNK34i4N81lwb2mAo6Q.css" media="all" />
<script>window.Backdrop = {settings: {"basePath":"\/","pathPrefix":"","drupalCompatibility":true,"ajaxPageState":{"theme":"motherboard","theme_token":"aosUoufPl-_-iwetipzUeCbYFrlwDo-cBys7TIkyLiI","css":{"core\/misc\/normalize.css":1,"core\/modules\/system\/css\/system.css":1,"core\/modules\/system\/css\/system.theme.css":1,"core\/modules\/system\/css\/messages.theme.css":1,"modules\/contrib\/back_to_top\/css\/back_to_top.css":1,"modules\/custom\/adamm\/css\/base.css":1,"modules\/custom\/adamm\/css\/dropdown.css":1,"core\/modules\/comment\/css\/comment.css":1,"core\/modules\/date\/css\/date.css":1,"core\/modules\/field\/css\/field.css":1,"modules\/contrib\/og\/css\/og.css":1,"modules\/contrib\/paragraphs\/css\/paragraphs.css":1,"core\/modules\/search\/search.theme.css":1,"modules\/contrib\/special_menu_items\/css\/special_menu_items.css":1,"modules\/custom\/usenix_conference\/css\/timezone-picker.css":1,"modules\/custom\/usenix_logo_preview\/css\/usenix_logo_preview.css":1,"core\/modules\/user\/css\/user.css":1,"modules\/contrib\/workflow\/workflow_admin_ui\/workflow_admin_ui.css":1,"core\/modules\/views\/css\/views.css":1,"modules\/contrib\/rules\/rules.css":1,"0":1,"modules\/contrib\/geshifilter\/geshifilter.css":1,"modules\/custom\/biblio\/biblio.css":1,"layouts\/motherboard\/motherboard.css":1,"modules\/contrib\/field_collection\/field_collection.theme.css":1,"core\/modules\/system\/css\/menu-dropdown.theme.breakpoint.css":1,"core\/modules\/system\/css\/menu-dropdown.theme.breakpoint-queries.css":1,"core\/modules\/system\/css\/menu-toggle.theme.breakpoint.css":1,"core\/modules\/system\/css\/menu-toggle.theme.breakpoint-queries.css":1,"themes\/motherboard\/core-css-overrides\/system.theme.css":1,"themes\/motherboard\/css\/style.css":1},"js":{"public:\/\/google_tag\/google_tag.script.js":1,"modules\/contrib\/jquery_update\/replace\/jquery\/1.12\/jquery.min.js":1,"core\/misc\/jquery.once.js":1,"core\/misc\/backdrop.js":1,"modules\/contrib\/behavior_weights\/js\/behavior_weights.js":1,"modules\/contrib\/jquery_update\/js\/jquery_browser.js":1,"modules\/contrib\/beautytips\/js\/jquery.bt.min.js":1,"modules\/contrib\/beautytips\/js\/beautytips.js":1,"modules\/contrib\/back_to_top\/js\/back_to_top.js":1,"modules\/custom\/adamm\/js\/accessible-menu\/dist\/disclosure-menu.iife.js":1,"modules\/custom\/adamm\/js\/contentexpander\/contentExpander.js":1,"modules\/custom\/adamm\/js\/adamm.js":1,"modules\/contrib\/views_slideshow\/js\/views_slideshow.js":1,"modules\/contrib\/views_slideshow\/contrib\/views_slideshow_cycle\/js\/formoptions.js":1,"modules\/contrib\/views_slideshow\/contrib\/views_slideshow_cycle\/js\/views_slideshow_cycle.js":1,"modules\/contrib\/field_group\/js\/field_group.js":1,"modules\/contrib\/field_group\/js\/field_groups.js":1,"core\/modules\/system\/js\/menus.js":1,"modules\/contrib\/matomo\/matomo.js":1,"0":1,"themes\/motherboard\/js\/scripts.js":1}},"back_to_top":{"distance":"100","text":"Back to top","title":1,"type":"image","disable_mobile":0},"beautytipStyles":{"default":[],"plain":[],"netflix":{"positions":["right","left"],"fill":"#FFF","padding":5,"shadow":true,"shadowBlur":12,"strokeStyle":"#B9090B","spikeLength":50,"spikeGirth":20,"cornerRadius":10,"centerPointY":0.1,"overlap":-8,"cssStyles":{"fontSize":"12px","fontFamily":"arial,helvetica,sans-serif"}},"facebook":{"fill":"#F7F7F7","padding":8,"strokeStyle":"#B7B7B7","cornerRadius":0,"cssStyles":{"fontFamily":"\u0022lucida grande\u0022,tahoma,verdana,arial,sans-serif","fontSize":"11px"}},"transparent":{"fill":"rgba(0, 0, 0, .8)","padding":20,"strokeStyle":"#CC0","strokeWidth":3,"spikeLength":40,"spikeGirth":40,"cornerRadius":40,"cssStyles":{"color":"#FFF","fontWeight":"bold"}},"big-green":{"fill":"#00FF4E","padding":20,"strokeWidth":0,"spikeLength":40,"spikeGirth":40,"cornerRadius":15,"cssStyles":{"fontFamily":"\u0022lucida grande\u0022,tahoma,verdana,arial,sans-serif","fontSize":"14px"}},"google-maps":{"positions":["top","bottom"],"fill":"#FFF","padding":15,"strokeStyle":"#ABABAB","strokeWidth":1,"spikeLength":65,"spikeGirth":40,"cornerRadius":25,"centerPointX":0.9,"cssStyles":{"fontSize":"12px","fontFamily":"arial,helvetica,sans-serif"}},"hulu":{"fill":"#F4F4F4","strokeStyle":"#666666","spikeLength":20,"spikeGirth":10,"width":350,"overlap":0,"centerPointY":1,"cornerRadius":0,"cssStyles":{"fontFamily":"\u0022Lucida Grande\u0022,Helvetica,Arial,Verdana,sans-serif","fontSize":"12px","padding":"10px 14px"},"shadow":true,"shadowColor":"rgba(0,0,0,.5)","shadowBlur":8,"shadowOffsetX":4,"shadowOffsetY":4}},"beautytips":{".beautytips":{"cssSelect":".beautytips","style":null}},"field_group":{"fieldset":"full","div":"full"},"matomo":{"trackMailto":0}}};</script>
<script defer="defer" src="https://www.usenix.org/sites/default/files/google_tag/google_tag.script.js?tl8841"></script>
<script src="https://www.usenix.org/sites/default/files/js/js_bZlYCeUOgUn_Zy3vTWwKIeA3h8k-K96O5_iBg4r0XoU.js"></script>
<script src="https://www.usenix.org/sites/default/files/js/js_nMqxMI6GSYBj21S6Dfarn5Pj4k7mEy4GyxtpkWgmSvM.js"></script>
<script src="https://www.usenix.org/sites/default/files/js/js_Hlb3Ws6cGMVcY0OsEf_ANFK6ZxjxYMG37Nel5e0OEqc.js"></script>
<script src="https://www.usenix.org/sites/default/files/js/js_GvooabyPOa7oSKo06TPm2BWTZj6uwA6fzxtfNPLI3ps.js"></script>
<script>var _paq = _paq || [];(function(){var u=(("https:" == document.location.protocol) ? "https://usenix.matomo.cloud/" : "http://usenix.matomo.cloud/");_paq.push(["setSiteId", "2"]);_paq.push(["setTrackerUrl", u+"matomo.php"]);_paq.push(["setDocumentTitle", "Seeing%20Like%20an%20SRE%3A%20Site%20Reliability%20Engineering%20as%20High%20Modernism"]);_paq.push(["setDownloadExtensions", "pdf|epub|mobi|zip|7z|tar|tgz|gz|gzip"]);_paq.push(["setDoNotTrack", 1]);_paq.push(["trackPageView"]);_paq.push(["setIgnoreClasses", ["no-tracking","colorbox"]]);_paq.push(["enableLinkTracking"]);var d=document,g=d.createElement("script"),s=d.getElementsByTagName("script")[0];g.type="text/javascript";g.defer=true;g.async=true;g.src=u+"matomo.js";s.parentNode.insertBefore(g,s);})();</script>
<script src="https://www.usenix.org/sites/default/files/js/js_lGXVUO6ublGZPqBOtfA6PeEk9UZEK29Dq376dDoap4I.js"></script>
</head>
<body class="page node-type-login-online user-is-non-member">
<div class="layout--motherboard layout path--publications-loginonline-seeing-sre-site-reliability-engineering-high-modernism node--seeing-like-an-sre-site-reliability-engineering-as-high-modernism node-type--login-online layout--no-sidebars">
<div class="layout-top-hat-wrapper">
<div class="layout-top-hat container">
<div class="block block-block-motherboard-top-banner">
<div class="block-content">
<!--
<p>SREcon25 Europe/Middle East/Africa registration is open!<br />
<a href="/conference/srecon25emea">Register Now</a></p>
--> </div>
</div>
<div class="block block-menu-menu-motherboard-utility-menu" role="navigation">
<div class="block-content">
<ul class="menu-top-only menu" data-menu-style="top_only" data-clickdown="0" data-collapse="default"><li class="first leaf menu-mlid-48513"><a href="https://www.usenix.org/donate" class="motherboard-donate">Donate</a></li>
<li class="last leaf menu-mlid-48514"><a href="/user/login">Log In</a></li>
</ul> </div>
</div>
</div>
</div>
<div class="layout-header__top-indicator"></div>
<header class="layout-header-wrapper">
<div class="layout-header container">
<div class="block block-system-header">
<div class="block-content">
<a href="/" title="Home" rel="home" class="logo">
<img src="https://www.usenix.org/themes/motherboard/logo.png" alt="Home" />
</a>
</div>
</div>
<div class="block block-menu-menu-motherboard-main-menu" role="navigation">
<div class="block-content">
<input id="menu-toggle-state" class="menu-toggle-state element-invisible" type="checkbox" aria-controls="menu-toggle-state" /><label class="menu-toggle-button" for="menu-toggle-state"><span class="menu-toggle-button-icon"></span><span class="menu-toggle-button-text">Menu</span><span class="menu-toggle-assistive-text element-invisible">Toggle menu visibility</span></label><ul class="menu-tree menu" data-menu-style="tree" data-clickdown="0" data-collapse="default" data-menu-toggle-id="menu-toggle-state"><li class="first expanded has-children menu-mlid-48716"><a href="/about">About</a><ul><li class="first last expanded has-children menu-mlid-48755"><span class="nolink" tabindex="0">About</span><ul><li class="first leaf menu-mlid-48728"><a href="/about">About Us</a></li>
<li class="leaf menu-mlid-48722"><a href="/board">Our Board of Directors</a></li>
<li class="leaf menu-mlid-48726"><a href="/board-meeting-minutes">Board Meeting Minutes</a></li>
<li class="leaf menu-mlid-48725"><a href="/board/elections26">Board Elections</a></li>
<li class="leaf menu-mlid-48759"><a href="/blog">Updates &amp; Announcements</a></li>
<li class="leaf menu-mlid-48723"><a href="/staff">Our Staff</a></li>
<li class="leaf menu-mlid-48724"><a href="/about/governance-financials">Governance &amp; Financials</a></li>
<li class="last leaf menu-mlid-48729"><a href="/about/awards/flame">Lifetime Achievement Award</a></li>
</ul></li>
</ul></li>
<li class="expanded has-children menu-mlid-48717"><a href="/conferences">Events</a><ul><li class="first last expanded has-children menu-mlid-48756"><span class="nolink" tabindex="0">Events</span><ul><li class="first leaf menu-mlid-48731"><a href="/conferences/upcoming">Upcoming</a></li>
<li class="leaf menu-mlid-48740"><a href="/conferences/past">Past</a></li>
<li class="leaf menu-mlid-48736"><a href="/conferences/faq">Conference FAQ</a></li>
<li class="leaf menu-mlid-48737"><a href="/conferences/policies-resources">Conference Policies</a></li>
<li class="leaf menu-mlid-48738"><a href="/conferences/coc">Code of Conduct</a></li>
<li class="leaf menu-mlid-48732"><a href="/conferences/calls-for-papers">Calls for Papers</a></li>
<li class="leaf menu-mlid-48739"><a href="/conferences/author-resources">Author Resources</a></li>
<li class="leaf menu-mlid-48733"><a href="/conferences/grants">Grant Opportunities</a></li>
<li class="leaf menu-mlid-48734"><a href="/conferences/best-papers">Best Papers</a></li>
<li class="last leaf menu-mlid-48735"><a href="/conferences/test-of-time-awards">Test of Time Awards</a></li>
</ul></li>
</ul></li>
<li class="expanded has-children menu-mlid-48719"><a href="/membership">Join &amp; Support</a><ul><li class="first last expanded has-children menu-mlid-48757"><span class="nolink" tabindex="0">Join & Support</span><ul><li class="first leaf menu-mlid-48753"><a href="/membership">Become a Member</a></li>
<li class="leaf menu-mlid-48754"><a href="/donate">Ways to Give</a></li>
<li class="leaf menu-mlid-48750"><a href="/supporters">Our Supporters</a></li>
<li class="leaf menu-mlid-48751"><a href="/students">Student Opportunities</a></li>
<li class="last leaf menu-mlid-48752"><a href="/conferences/sponsorship">Sponsorship Opportunities</a></li>
</ul></li>
</ul></li>
<li class="expanded has-children menu-mlid-48718"><a href="/publications">Archive</a><ul><li class="first last expanded has-children menu-mlid-48758"><span class="nolink" tabindex="0">Archive</span><ul><li class="first leaf menu-mlid-48743"><a href="/publications/proceedings">Proceedings</a></li>
<li class="leaf menu-mlid-48742"><a href="/conferences/multimedia">Multimedia</a></li>
<li class="leaf menu-mlid-48744"><a href="/publications/login">;login: Archive</a></li>
<li class="leaf menu-mlid-48746 dropdown-menu-item--new-column-at-desktop"><a href="/short-topics">Short Topics in System Administration Series</a></li>
<li class="leaf menu-mlid-48745"><a href="/jesa">Journal of Education in System Administration (JESA)</a></li>
<li class="leaf menu-mlid-48748"><a href="/journal/jets">Journal of Election Technology and Systems (JETS)</a></li>
<li class="last leaf menu-mlid-48747"><a href="/publications/compsystems/computing-systems">Computing Systems Journal</a></li>
</ul></li>
</ul></li>
<li class="last leaf menu-mlid-48720"><a href="/search/site" class="motherboard-search">Search</a></li>
</ul> </div>
</div>
</div>
</header>
<div class="layout-subheader-wrapper shadow-bottom">
<div class="layout-subheader container">
<div class="block block-block-162">
<div class="block-content">
<div class="login-v2-article-header">
<div class="login-v2-discussion">
<div class="login-v2-discussion-text">
<a href="#comments"><span class="login-icon-chat"></span>Join the conversation</a><br>
<a href="/publications/loginonline">Back to ;login: Online</a>
</div>
</div>
</div>
<div class="login-v2-logo-title"></div> </div>
</div>
</div>
</div>
<div class="layout-highlighted-wrapper">
<div class="layout-highlighted centered container">
<div class="block block-system-title">
<div class="block-content">
<h1 class="page-title">Seeing Like an SRE: Site Reliability Engineering as High Modernism</h1>
</div>
</div>
</div>
</div>
<div class="layout-page-wrapper row">
<main role="main" class="layout-content-wrapper">
<a id="main-content" tabindex="-1"></a>
<div class="layout-content container">
<div class="l-page-title">
</div>
<article id="node-273658" class="node node-login-online view-mode-full clearfix">
<div class="content clearfix">
<div class="group-article-body-wrapper field-group-div"><div class="field field-name-field-lv2-publication-date field-type-datetime field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><span class="date-display-single">April 28, 2021</span></div></div></div><div class="field field-name-field-lv2-article-type field-type-taxonomy-term-reference field-label-hidden"><div class="field-items field-items"><div class="field-item odd">Column</div></div></div><div class="field field-label-inline clearfix field-type-text-long field-pseudo-field field-pseudo-field--author-list"><div class="field-label">Authors:&nbsp;</div><div class="field-items"><a href="#Laura Nolan" title="Laura Nolan">Laura Nolan</a></div></div><div class="field field-name-field-lv2-shepherds field-type-user-reference field-label-inline clearfix"><div class="field-label">Article shepherded by:&nbsp;</div><div class="field-items field-items"><div class="field-item odd"><span class="usenix-user-reference-names">Rik Farrow</span></div></div></div>
<div class="paragraphs-items paragraphs-items-field-lv2-body paragraphs-items-field-lv2-body-full paragraphs-items-full">
<div class="field field-name-field-lv2-body field-type-paragraphs field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><div class="paragraphs-item paragraphs-item-single-column-text paragraphs-item-single-column-text paragraphs-item-full paragraphs-item-4983">
<div class="content">
<div class="field field-name-field-single-column-text field-type-text-long field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><p>I recently spent some time trying to write a set of general guidelines for what to monitor in a software system. I came up with this list:</p> <ul><li><span>Latency distribution and successful/unsuccessful request counts (plus error types) for all RPCs served.</span></li><li>Latency distribution and success rate for all other services depended on, as well as circuit breakers tripping.</li><li>Monitor the last success time for anything that’s supposed to happen periodically.</li><li>Percentage utilisation for resources (quotas, rate limits, physical and logical system resources), as well as saturation signals for the same, and errors or timeouts.</li><li>How many instances are up and healthy/unhealthy, restarts, running versions of binaries.</li><li>System invariants: other properties of your specific system. For instance, the count of leaders for a leader-elected system (expected to be one - you want to know if it isn’t). Other examples could include the number of replicas of parts of a replicated dataset, cache hit rates, and so on.</li></ul> <p>This is an OK list. But like a lot of what we do when we try to create generalised knowledge about software operations, it’s a little vague and unsatisfying. If I’ve got a system of significant complexity and scope and a new engineer, I probably can’t hand them that list and expect them to implement comprehensive monitoring. That engineer doesn’t know the specific system well, it’ll take them a long time to do the work and quite likely some things will be missed.</p> <p>A lot of things in software operations generally, and SRE specifically, are like this. You need a lot of knowledge of a specific system to do things well. <a href="https://sre.google/sre-book/evolving-sre-engagement-model/">Production Readiness Reviews</a> for new services, or services which are being handed off to an SRE team, are a good example. In some organisations, these are done as a collaboration between the development team who built the service and the SRE team who are onboarding it. In others, a centralised, consulting SRE team works with the developers to assess compliance with a set of production standards.</p> <p>These processes are called the same thing, but they look quite different. Typically, the former kind of PRR will take a quarter or more, because invariably, new large services have a significant amount of tasks to do to get them production-ready. The SRE team onboarding the service will spend significant time finding gaps, understanding what happens when dependencies of the service fail, improving monitoring and runbooks and automation.</p> <p>The second kind of PRR typically does not uncover much to be done, and devolves into a tick-box exercise where the developers attempt to demonstrate compliance with the organisation’s production standards. The consultant SREs don’t know the ins-and-outs of the system in question, and while generic expertise in production systems may spot some issues, significant things will almost certainly be missed. Discussing this problem, a friend of mine with over 25 years of experience said: “I was that person. I reviewed the system and it seemed solid. It was only when I tried to use it and found the bugs I found it was a giant Potemkin village.”</p> <p>Over the winter holidays, I read James C. Scott’s <em>Seeing Like a State: How Certain Schemes to Improve the Human Condition have Failed </em><a href="#Reference1">[1]</a>. Scott is a political scientist and anthropologist, and his work mainly deals with nation states and how they understand and control their populations. As well as attempting to understand what they govern, states try to simplify things, to make them easier to understand - the term Scott uses is ‘legible’. Scott puts forward a range of examples. Standardizing on an official state language means that it’s easier to run a bureaucracy. Introducing surnames makes it easier to track whether people are paying taxes. Building cities on a grid pattern makes it easier to get around, and easier to put down uprisings. Monocultures in forestry make it easier to know how many trees you can harvest.</p> <p>What we do in software operations has a lot in common with this project Scott describes. We spend a lot of time trying to make our systems legible. We do this by adding logging, monitoring, tracing, status pages and other tooling.</p> <p>We also attempt to standardise and simplify our systems. There is huge power in this. For example, not every system at Google is uniform, but there’s a great deal of uniformity. Almost all software that Google runs is built in-house, generally using one of a small number of common frameworks. RPC mechanisms, readiness and liveness checks, metrics endpoints and status pages (to surface information and controls for human operators) are built in. Essentially everything runs on <a href="https://research.google/pubs/pub43438/">Borg</a> — all jobs have been assimilated. Standardisation means it’s easier for systems to interoperate, and it’s easier to build infrastructure because you don’t need to support many different ways of doing the same thing. The degree of standardisation means that it’s possible to run a generic class for new Google engineers that gives them the basic skills to investigate almost any job running at Google that they have privileges to access: where it is running, rich statistics about RPCs in and out, logs, traces, and so on.</p> <p>The same project has come to the rest of our industry, it’s just not as far advanced. Kubernetes is standardised orchestration. Services meshes aim to standardise service to service communication and make it more legible and manageable. Immutable infrastructure in general is a way of controlling what we deploy and run, ensuring it’s consistent. Serverless is another form of standardisation and simplification. <a href="https://cloud.google.com/blog/products/devops-sre/sre-fundamentals-slis-slas-and-slos">Service Level Objectives</a> (SLOs) exist to provide uniform signals about the health of services. All of these are our attempts to constrain and standardise what we build and run, to make it uniform and legible, the software equivalent of Scott’s grid layout for cities.</p> <p>The moment we are seeing now in software operations seems to be a type of high modernism. High modernism is a phenomenon that came about during the Cold War, characterized by a faith in scientific management and development, and a rejection of crafts and traditions. It was a vision of a well-ordered utopia, in the form of high rise living, in planned cities with simple geometries.</p> <p>Like high modernism, software high modernism is experiencing mixed success. Kubernetes is as divisive as ever Le Corbusier’s high rise buildings were. Switching a large organisation’s suite of microservices to a service mesh is almost as messy and disruptive as imposing a grid system on a medieval city (and done for much the same reasons of increasing legibility and hygiene). SLOs are useful, but defining meaningful non-cookie-cutter SLOs is a lot of work.</p> <p>However, the dark side of the current wave of software high modernism is that applying these principles may mean picking up your existing systems and moving them onto a very different infrastructure — which you then need to run. For very small organisations, the added complexity of running Kubernetes or a service mesh may not make sense. For larger organisations, getting to the goal of uniformity means moving a larger set of systems to a new infrastructure — a lengthy migration. This is a significant investment of time, engineering effort, and, most likely, error budgets.</p> <p>Increasing legibility doesn’t always mean migrating your entire infrastructure in a high modernist-style five year plan, though. <a href="https://www.usenix.org/conference/srecon20americas/presentation/limoncelli">Tom Limoncelli’s ‘Low Context Devops’</a> (also in <em><a href="https://www.usenix.org/publications/loginonline/low-context-devops" title="Low Context DevOps">;login:</a></em>) is a great example of some less disruptive things we can do to increase our systems’ legibility, such as standardising documentation and linking it from error messages and alerts, setting good defaults, providing base libraries that embody recommended practices.</p> <p>Increasing legibility and standardization of our systems is a good thing, as long as it comes at a reasonable cost, but it only gets us so far. Scott distinguishes between two sorts of knowledge: <em>techne</em> and <em>metis</em>.</p> <p><em><span>Techne</span></em><span> is universal knowledge: things like the boiling point of water, Pythagoras’ theorem, the rule that all <a href="https://sre.google/sre-book/addressing-cascading-failures/">RPCs should have deadlines</a>, or that we should probably alert if no instances of our jobs are running. <em>Techne</em> is very useful. We can write books about <em>techne</em>, and embed some of it in our tooling and infrastructure, like liveness checks in Kubernetes or deadlines in service meshes.</span></p> <p>The other kind of knowledge, <em>metis</em>, is local, specific, and practical. It’s won from experience. It can’t be codified in the same way that <em>techne</em> can. The comparison that Scott gives is between navigation and piloting. Deepwater navigation is a general skill, but a pilot knows a specific port — a ‘local and situated knowledge,’ as Scott puts it, including tides, currents, seasonal changes, shifting sandbars, and wind patterns. A pilot cannot move to another port and expect to have the same level of skill and local knowledge. When I joined a team that ran edge routers and switches, I had some useful <em>techne</em> — knowledge of network protocols and routing protocols and so on — but I definitely wasn’t a network engineer. I had to learn an awful lot of <em>metis</em> in the form of network debugging techniques, how to roll back to a previous router configuration, what a <a href="https://en.wikipedia.org/wiki/QSFP">small form-factor pluggable transceiver</a> implied for maintenance (my new colleagues laughed at me for not knowing what those were, the jerks — but they did explain it), how to deal with a DDOS, and so on.</p> <p>Knowledge of a specific software system is <em>metis</em>, rather than <em>techne</em>. This is why there is a learning curve when we start working on a new system, and why we don’t put our new teammates on call right away. More standardisation in infrastructure, better runbooks and so on are the software equivalent of dredging the shipping channel and putting markers on obstructions — they can somewhat reduce the amount of <em>metis</em> we need to have, but not eliminate the need for a local pilot entirely.</p> <p>To return to where I started this article, this is why generic checklists always fall a bit flat, and why it’s very difficult to run a thorough production readiness review for a system that you aren’t deeply familiar with. Both of these are an attempt to substitute <em>techne</em> for <em>metis</em>, which just doesn’t work.</p> <p><span>This, perhaps, is the source of some of the antipathy that some old-school sysadmins have for SRE (as exemplified by the reaction to Todd Underwood’s LISA 2013 talk, <a href="https://www.usenix.org/conference/lisa13/technical-sessions/plenary/underwood">PostOps: A Non-Surgical Tale of Software, Fragility, and Reliability</a>). SRE is seen as a high modernist project, intent on scientifically managing their systems, all <em>techne</em> and no <em>metis</em>; all SLOs and Kubernetes and no systems knowledge and craft. That view is not entirely wrong. Some in the SRE movement do see it that way — things like consulting SRE teams, SRE software platforms, SLOs, and Kubernetes are popular for a reason, and they do have their uses. But call it what you will – craft, <em>metis</em> – specific systems knowledge isn’t going away anytime soon. SRE or sysadmin, <em>metis </em></span><span>is the one aspect of our jobs that we are unlikely to ever automate away.</span></p></div></div></div> </div>
</div>
</div></div></div></div>
<fieldset class="group-appendix field-group-fieldset form-wrapper"><legend><span class="fieldset-legend">Appendix</span></legend><div class="fieldset-wrapper"><div class="field field-name-field-lvl2-appendix-refs field-type-text field-label-above"><div class="field-label">References:&nbsp;</div><div class="field-items field-items"><div class="field-item odd"><a class="anchor" name="reference-1"></a><p>&nbsp;[1] James Scott, <em>Seeing Like a State</em>. Veritas, 2012.<a name="Reference1"></a>&nbsp;</p>
</div></div></div></div></fieldset>
</div><div class="psuedo-last-updated">Last updated February 8, 2023</div><div class="field-collection-container clearfix"><div class="field field-name-field-authors field-type-field-collection field-label-above"><div class="field-label">Authors:&nbsp;</div><div class="field-items field-items"><div class="field-item odd"><div class="field-collection-view clearfix view-mode-full field-collection-view-final"><div class="entity-plus entity-field-collection-item field-collection-item-field-authors clearfix">
<div class="content">
<a class="anchor" name="Laura Nolan"></a><div class="field field-name-field-collection-author-image field-type-image field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><img src="https://www.usenix.org/sites/default/files/styles/author_bio/public/nolan.jpg" width="138" height="138" alt="" /></div></div></div><div class="field field-name-field-collection-author-bio field-type-text-long field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><p>Laura Nolan is an SRE and software engineer by day and a postgraduate student in Ethics at Dublin City University by night. She is a contributor to Site Reliability Engineering: How Google Runs Production Systems and Seeking SRE, published by O’Reilly, and is a member of the USENIX SREcon Steering Committee.</p>
</div></div></div><div class="field field-name-field-collection-author-email field-type-email field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><a href="/cdn-cgi/l/email-protection#96faf7e3e4f7b8f8f9faf7f8d6f1fbf7fffab8f5f9fbb6"><span class="__cf_email__" data-cfemail="264a475354470848494a474866414b474f4a0845494b">[email&#160;protected]</span> </a></div></div></div> </div>
</div>
</div></div></div></div></div> </div>
<ul class="links inline"><li class="comment-forbidden odd first last"><span><span class="comment-1"><a id="comments" href="/user/login?destination=comment/reply/273658%23comment-form">Log in</a> to post comments</span></span></li></ul>
</article>
</div>
</main>
</div>
<footer role="contentinfo">
<div class="layout-footer container">
<div class="block block-block-motherboard-footer-logo">
<div class="block-content">
<a href="/"><img src="/themes/motherboard/images/usenix-logo-white.png" style="width: 160px; height: auto;" alt="USENIX logo" /></a> </div>
</div>
<div class="block block-menu-menu-motherboard-footer-menu" role="navigation">
<div class="block-content">
<ul class="menu-tree menu" data-menu-style="tree" data-clickdown="0" data-collapse="default"><li class="first leaf menu-mlid-48511"><a href="/contact">Contact USENIX</a></li>
<li class="last leaf menu-mlid-48512"><a href="/privacy-policy">Privacy Policy</a></li>
</ul> </div>
</div>
<div class="block block-block-motherboard-footer-copyright">
<div class="block-content">
<p>© USENIX 2025<br />
EIN 13-3055038
<br />
Website designed and built by <a href="https://www.giantrabbit.com" target="_blank">Giant Rabbit LLC</a><br />
Powered by <a href="https://backdropcms.org/" target="_blank">Backdrop CMS</a></p>
</div>
</div>
</div>
</footer>
</div>
<noscript aria-hidden="true"><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-WQSPGJT"
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript> <script data-cfasync="false" src="/cdn-cgi/scripts/5c5dd728/cloudflare-static/email-decode.min.js"></script><script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" data-cf-beacon='{"version":"2024.11.0","token":"04ec313fe1c44bdeb95db4ee6e679a22","r":1,"spa":2}' crossorigin="anonymous"></script>
<script>(function(){function c(){var b=a.contentDocument||(a.contentWindow&&a.contentWindow.document);if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a39b356d18681bcd',t:'MTc4OTE3NzI3NA=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();</script></body>
</html>