Files
nexus/sreweekly/articles/310/08-lessons-learned-in-10-years-of-sre-part-1-starting-sre.html
2026-09-12 17:23:01 +08:00

274 lines
42 KiB
HTML
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en" dir="ltr" prefix="og: http://ogp.me/ns#">
<head>
<meta charset="utf-8" />
<link rel="shortcut icon" href="https://www.usenix.org/themes/motherboard/favicon.ico" type="image/vnd.microsoft.icon" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta content="general" name="rating" />
<meta content="Drupal 7 (http://drupal.org)" name="generator" />
<link rel="canonical" href="https://www.usenix.org/publications/loginonline/lessons-learned-10-years-sre-part-1-starting-sre" />
<link rel="shortlink" href="https://www.usenix.org/node/278934" />
<meta content="website" property="og:type" />
<meta content="USENIX" property="og:site_name" />
<meta content="Lessons Learned in 10 Years of SRE: Part 1 - Starting SRE" property="og:title" />
<meta content="https://www.usenix.org/publications/loginonline/lessons-learned-10-years-sre-part-1-starting-sre" property="og:url" />
<meta content="2023-02-08T15:35:50-08:00" property="og:updated_time" />
<meta content="https://www.usenix.org/sites/default/files/usenix_2025_og_1200x630.png" property="og:image" />
<meta content="https://www.usenix.org/sites/default/files/usenix_2025_og_1200x630.png" property="og:image:url" />
<meta content="https://www.usenix.org/sites/default/files/usenix_2025_og_1200x630.png" property="og:image:secure_url" />
<meta content="image/png" property="og:image:type" />
<meta content="2022-02-02T12:34:51-08:00" property="article:published_time" />
<meta content="2023-02-08T15:35:50-08:00" property="article:modified_time" />
<title>Lessons Learned in 10 Years of SRE: Part 1 - Starting SRE | USENIX</title>
<link rel="stylesheet" href="https://www.usenix.org/sites/default/files/css/css_nCq0Tqg9eqZPa5fXfe0k9wAX8jnxbXlZ75LfGb5rq7A.css" media="all" />
<link rel="stylesheet" href="https://www.usenix.org/sites/default/files/css/css_2i5hNf7GrbmdIRSKLAmKbqqhLsXPdxtef5MzSaLSGkY.css" media="all" />
<style media="all">#backtotop{bottom:20px;right:20px;}
</style>
<link rel="stylesheet" href="https://www.usenix.org/sites/default/files/css/css_K9stUlVccdAb6O9D60Lwdlif4rvmqiwmXCQESGsrHis.css" media="all" />
<link rel="stylesheet" href="https://www.usenix.org/sites/default/files/css/css_qFgpkJD9LCbzFq1MQrtO4IkZu4u0bPvQ41XA-NJVJPM.css" media="all and (min-width: 60em)" />
<link rel="stylesheet" href="https://www.usenix.org/sites/default/files/css/css_B99K36ryG5rmvImk01aGYNU6pRjl9ax1F1V5MSwJ1l0.css" media="all" />
<link rel="stylesheet" href="https://www.usenix.org/sites/default/files/css/css_cU3KfR4jRp9zSJ4c3qKFtdvl9cZqjJmX6xjYh4peo_E.css" media="all and (min-width: 60em)" />
<link rel="stylesheet" href="https://www.usenix.org/sites/default/files/css/css_gKxwQGISVq7A4HqYbamWQZ1bWNK34i4N81lwb2mAo6Q.css" media="all" />
<script>window.Backdrop = {settings: {"basePath":"\/","pathPrefix":"","drupalCompatibility":true,"ajaxPageState":{"theme":"motherboard","theme_token":"-wdEfDy2WaYeXw-0ouJyR61Y4Me2gGLbnZAYMROEgnw","css":{"core\/misc\/normalize.css":1,"core\/modules\/system\/css\/system.css":1,"core\/modules\/system\/css\/system.theme.css":1,"core\/modules\/system\/css\/messages.theme.css":1,"modules\/contrib\/back_to_top\/css\/back_to_top.css":1,"modules\/custom\/adamm\/css\/base.css":1,"modules\/custom\/adamm\/css\/dropdown.css":1,"core\/modules\/comment\/css\/comment.css":1,"core\/modules\/date\/css\/date.css":1,"core\/modules\/field\/css\/field.css":1,"modules\/contrib\/og\/css\/og.css":1,"modules\/contrib\/paragraphs\/css\/paragraphs.css":1,"core\/modules\/search\/search.theme.css":1,"modules\/contrib\/special_menu_items\/css\/special_menu_items.css":1,"modules\/custom\/usenix_conference\/css\/timezone-picker.css":1,"modules\/custom\/usenix_logo_preview\/css\/usenix_logo_preview.css":1,"core\/modules\/user\/css\/user.css":1,"modules\/contrib\/workflow\/workflow_admin_ui\/workflow_admin_ui.css":1,"core\/modules\/views\/css\/views.css":1,"modules\/contrib\/rules\/rules.css":1,"0":1,"modules\/contrib\/geshifilter\/geshifilter.css":1,"modules\/custom\/biblio\/biblio.css":1,"layouts\/motherboard\/motherboard.css":1,"modules\/contrib\/field_collection\/field_collection.theme.css":1,"core\/modules\/system\/css\/menu-dropdown.theme.breakpoint.css":1,"core\/modules\/system\/css\/menu-dropdown.theme.breakpoint-queries.css":1,"core\/modules\/system\/css\/menu-toggle.theme.breakpoint.css":1,"core\/modules\/system\/css\/menu-toggle.theme.breakpoint-queries.css":1,"themes\/motherboard\/core-css-overrides\/system.theme.css":1,"themes\/motherboard\/css\/style.css":1},"js":{"public:\/\/google_tag\/google_tag.script.js":1,"modules\/contrib\/jquery_update\/replace\/jquery\/1.12\/jquery.min.js":1,"core\/misc\/jquery.once.js":1,"core\/misc\/backdrop.js":1,"modules\/contrib\/behavior_weights\/js\/behavior_weights.js":1,"modules\/contrib\/jquery_update\/js\/jquery_browser.js":1,"modules\/contrib\/beautytips\/js\/jquery.bt.min.js":1,"modules\/contrib\/beautytips\/js\/beautytips.js":1,"modules\/contrib\/back_to_top\/js\/back_to_top.js":1,"modules\/custom\/adamm\/js\/accessible-menu\/dist\/disclosure-menu.iife.js":1,"modules\/custom\/adamm\/js\/contentexpander\/contentExpander.js":1,"modules\/custom\/adamm\/js\/adamm.js":1,"modules\/contrib\/views_slideshow\/js\/views_slideshow.js":1,"modules\/contrib\/views_slideshow\/contrib\/views_slideshow_cycle\/js\/formoptions.js":1,"modules\/contrib\/views_slideshow\/contrib\/views_slideshow_cycle\/js\/views_slideshow_cycle.js":1,"modules\/contrib\/field_group\/js\/field_group.js":1,"modules\/contrib\/field_group\/js\/field_groups.js":1,"core\/modules\/system\/js\/menus.js":1,"modules\/contrib\/matomo\/matomo.js":1,"0":1,"themes\/motherboard\/js\/scripts.js":1}},"back_to_top":{"distance":"100","text":"Back to top","title":1,"type":"image","disable_mobile":0},"beautytipStyles":{"default":[],"plain":[],"netflix":{"positions":["right","left"],"fill":"#FFF","padding":5,"shadow":true,"shadowBlur":12,"strokeStyle":"#B9090B","spikeLength":50,"spikeGirth":20,"cornerRadius":10,"centerPointY":0.1,"overlap":-8,"cssStyles":{"fontSize":"12px","fontFamily":"arial,helvetica,sans-serif"}},"facebook":{"fill":"#F7F7F7","padding":8,"strokeStyle":"#B7B7B7","cornerRadius":0,"cssStyles":{"fontFamily":"\u0022lucida grande\u0022,tahoma,verdana,arial,sans-serif","fontSize":"11px"}},"transparent":{"fill":"rgba(0, 0, 0, .8)","padding":20,"strokeStyle":"#CC0","strokeWidth":3,"spikeLength":40,"spikeGirth":40,"cornerRadius":40,"cssStyles":{"color":"#FFF","fontWeight":"bold"}},"big-green":{"fill":"#00FF4E","padding":20,"strokeWidth":0,"spikeLength":40,"spikeGirth":40,"cornerRadius":15,"cssStyles":{"fontFamily":"\u0022lucida grande\u0022,tahoma,verdana,arial,sans-serif","fontSize":"14px"}},"google-maps":{"positions":["top","bottom"],"fill":"#FFF","padding":15,"strokeStyle":"#ABABAB","strokeWidth":1,"spikeLength":65,"spikeGirth":40,"cornerRadius":25,"centerPointX":0.9,"cssStyles":{"fontSize":"12px","fontFamily":"arial,helvetica,sans-serif"}},"hulu":{"fill":"#F4F4F4","strokeStyle":"#666666","spikeLength":20,"spikeGirth":10,"width":350,"overlap":0,"centerPointY":1,"cornerRadius":0,"cssStyles":{"fontFamily":"\u0022Lucida Grande\u0022,Helvetica,Arial,Verdana,sans-serif","fontSize":"12px","padding":"10px 14px"},"shadow":true,"shadowColor":"rgba(0,0,0,.5)","shadowBlur":8,"shadowOffsetX":4,"shadowOffsetY":4}},"beautytips":{".beautytips":{"cssSelect":".beautytips","style":null}},"field_group":{"fieldset":"full","div":"full"},"matomo":{"trackMailto":0}}};</script>
<script defer="defer" src="https://www.usenix.org/sites/default/files/google_tag/google_tag.script.js?tl8j82"></script>
<script src="https://www.usenix.org/sites/default/files/js/js_bZlYCeUOgUn_Zy3vTWwKIeA3h8k-K96O5_iBg4r0XoU.js"></script>
<script src="https://www.usenix.org/sites/default/files/js/js_nMqxMI6GSYBj21S6Dfarn5Pj4k7mEy4GyxtpkWgmSvM.js"></script>
<script src="https://www.usenix.org/sites/default/files/js/js_Hlb3Ws6cGMVcY0OsEf_ANFK6ZxjxYMG37Nel5e0OEqc.js"></script>
<script src="https://www.usenix.org/sites/default/files/js/js_GvooabyPOa7oSKo06TPm2BWTZj6uwA6fzxtfNPLI3ps.js"></script>
<script>var _paq = _paq || [];(function(){var u=(("https:" == document.location.protocol) ? "https://usenix.matomo.cloud/" : "http://usenix.matomo.cloud/");_paq.push(["setSiteId", "2"]);_paq.push(["setTrackerUrl", u+"matomo.php"]);_paq.push(["setDocumentTitle", "Lessons%20Learned%20in%2010%20Years%20of%20SRE%3A%20Part%201%20-%20Starting%20SRE"]);_paq.push(["setDownloadExtensions", "pdf|epub|mobi|zip|7z|tar|tgz|gz|gzip"]);_paq.push(["setDoNotTrack", 1]);_paq.push(["trackPageView"]);_paq.push(["setIgnoreClasses", ["no-tracking","colorbox"]]);_paq.push(["enableLinkTracking"]);var d=document,g=d.createElement("script"),s=d.getElementsByTagName("script")[0];g.type="text/javascript";g.defer=true;g.async=true;g.src=u+"matomo.js";s.parentNode.insertBefore(g,s);})();</script>
<script src="https://www.usenix.org/sites/default/files/js/js_lGXVUO6ublGZPqBOtfA6PeEk9UZEK29Dq376dDoap4I.js"></script>
</head>
<body class="page node-type-login-online user-is-non-member">
<div class="layout--motherboard layout path--publications-loginonline-lessons-learned-10-years-sre-part-1-starting-sre node--lessons-learned-in-10-years-of-sre-part-1---starting-sre node-type--login-online layout--no-sidebars">
<div class="layout-top-hat-wrapper">
<div class="layout-top-hat container">
<div class="block block-block-motherboard-top-banner">
<div class="block-content">
<!--
<p>SREcon25 Europe/Middle East/Africa registration is open!<br />
<a href="/conference/srecon25emea">Register Now</a></p>
--> </div>
</div>
<div class="block block-menu-menu-motherboard-utility-menu" role="navigation">
<div class="block-content">
<ul class="menu-top-only menu" data-menu-style="top_only" data-clickdown="0" data-collapse="default"><li class="first leaf menu-mlid-48513"><a href="https://www.usenix.org/donate" class="motherboard-donate">Donate</a></li>
<li class="last leaf menu-mlid-48514"><a href="/user/login">Log In</a></li>
</ul> </div>
</div>
</div>
</div>
<div class="layout-header__top-indicator"></div>
<header class="layout-header-wrapper">
<div class="layout-header container">
<div class="block block-system-header">
<div class="block-content">
<a href="/" title="Home" rel="home" class="logo">
<img src="https://www.usenix.org/themes/motherboard/logo.png" alt="Home" />
</a>
</div>
</div>
<div class="block block-menu-menu-motherboard-main-menu" role="navigation">
<div class="block-content">
<input id="menu-toggle-state" class="menu-toggle-state element-invisible" type="checkbox" aria-controls="menu-toggle-state" /><label class="menu-toggle-button" for="menu-toggle-state"><span class="menu-toggle-button-icon"></span><span class="menu-toggle-button-text">Menu</span><span class="menu-toggle-assistive-text element-invisible">Toggle menu visibility</span></label><ul class="menu-tree menu" data-menu-style="tree" data-clickdown="0" data-collapse="default" data-menu-toggle-id="menu-toggle-state"><li class="first expanded has-children menu-mlid-48716"><a href="/about">About</a><ul><li class="first last expanded has-children menu-mlid-48755"><span class="nolink" tabindex="0">About</span><ul><li class="first leaf menu-mlid-48728"><a href="/about">About Us</a></li>
<li class="leaf menu-mlid-48722"><a href="/board">Our Board of Directors</a></li>
<li class="leaf menu-mlid-48726"><a href="/board-meeting-minutes">Board Meeting Minutes</a></li>
<li class="leaf menu-mlid-48725"><a href="/board/elections26">Board Elections</a></li>
<li class="leaf menu-mlid-48759"><a href="/blog">Updates &amp; Announcements</a></li>
<li class="leaf menu-mlid-48723"><a href="/staff">Our Staff</a></li>
<li class="leaf menu-mlid-48724"><a href="/about/governance-financials">Governance &amp; Financials</a></li>
<li class="last leaf menu-mlid-48729"><a href="/about/awards/flame">Lifetime Achievement Award</a></li>
</ul></li>
</ul></li>
<li class="expanded has-children menu-mlid-48717"><a href="/conferences">Events</a><ul><li class="first last expanded has-children menu-mlid-48756"><span class="nolink" tabindex="0">Events</span><ul><li class="first leaf menu-mlid-48731"><a href="/conferences/upcoming">Upcoming</a></li>
<li class="leaf menu-mlid-48740"><a href="/conferences/past">Past</a></li>
<li class="leaf menu-mlid-48736"><a href="/conferences/faq">Conference FAQ</a></li>
<li class="leaf menu-mlid-48737"><a href="/conferences/policies-resources">Conference Policies</a></li>
<li class="leaf menu-mlid-48738"><a href="/conferences/coc">Code of Conduct</a></li>
<li class="leaf menu-mlid-48732"><a href="/conferences/calls-for-papers">Calls for Papers</a></li>
<li class="leaf menu-mlid-48739"><a href="/conferences/author-resources">Author Resources</a></li>
<li class="leaf menu-mlid-48733"><a href="/conferences/grants">Grant Opportunities</a></li>
<li class="leaf menu-mlid-48734"><a href="/conferences/best-papers">Best Papers</a></li>
<li class="last leaf menu-mlid-48735"><a href="/conferences/test-of-time-awards">Test of Time Awards</a></li>
</ul></li>
</ul></li>
<li class="expanded has-children menu-mlid-48719"><a href="/membership">Join &amp; Support</a><ul><li class="first last expanded has-children menu-mlid-48757"><span class="nolink" tabindex="0">Join & Support</span><ul><li class="first leaf menu-mlid-48753"><a href="/membership">Become a Member</a></li>
<li class="leaf menu-mlid-48754"><a href="/donate">Ways to Give</a></li>
<li class="leaf menu-mlid-48750"><a href="/supporters">Our Supporters</a></li>
<li class="leaf menu-mlid-48751"><a href="/students">Student Opportunities</a></li>
<li class="last leaf menu-mlid-48752"><a href="/conferences/sponsorship">Sponsorship Opportunities</a></li>
</ul></li>
</ul></li>
<li class="expanded has-children menu-mlid-48718"><a href="/publications">Archive</a><ul><li class="first last expanded has-children menu-mlid-48758"><span class="nolink" tabindex="0">Archive</span><ul><li class="first leaf menu-mlid-48743"><a href="/publications/proceedings">Proceedings</a></li>
<li class="leaf menu-mlid-48742"><a href="/conferences/multimedia">Multimedia</a></li>
<li class="leaf menu-mlid-48744"><a href="/publications/login">;login: Archive</a></li>
<li class="leaf menu-mlid-48746 dropdown-menu-item--new-column-at-desktop"><a href="/short-topics">Short Topics in System Administration Series</a></li>
<li class="leaf menu-mlid-48745"><a href="/jesa">Journal of Education in System Administration (JESA)</a></li>
<li class="leaf menu-mlid-48748"><a href="/journal/jets">Journal of Election Technology and Systems (JETS)</a></li>
<li class="last leaf menu-mlid-48747"><a href="/publications/compsystems/computing-systems">Computing Systems Journal</a></li>
</ul></li>
</ul></li>
<li class="last leaf menu-mlid-48720"><a href="/search/site" class="motherboard-search">Search</a></li>
</ul> </div>
</div>
</div>
</header>
<div class="layout-subheader-wrapper shadow-bottom">
<div class="layout-subheader container">
<div class="block block-block-162">
<div class="block-content">
<div class="login-v2-article-header">
<div class="login-v2-discussion">
<div class="login-v2-discussion-text">
<a href="#comments"><span class="login-icon-chat"></span>Join the conversation</a><br>
<a href="/publications/loginonline">Back to ;login: Online</a>
</div>
</div>
</div>
<div class="login-v2-logo-title"></div> </div>
</div>
</div>
</div>
<div class="layout-highlighted-wrapper">
<div class="layout-highlighted centered container">
<div class="block block-system-title">
<div class="block-content">
<h1 class="page-title">Lessons Learned in 10 Years of SRE: Part 1 - Starting SRE</h1>
</div>
</div>
</div>
</div>
<div class="layout-page-wrapper row">
<main role="main" class="layout-content-wrapper">
<a id="main-content" tabindex="-1"></a>
<div class="layout-content container">
<div class="l-page-title">
</div>
<article id="node-278934" class="node node-login-online view-mode-full clearfix">
<div class="content clearfix">
<div class="group-article-body-wrapper field-group-div"><div class="field field-name-field-lv2-publication-date field-type-datetime field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><span class="date-display-single">February 2, 2022</span></div></div></div><div class="field field-name-field-lv2-article-type field-type-taxonomy-term-reference field-label-hidden"><div class="field-items field-items"><div class="field-item odd">Opinion</div></div></div><div class="field field-label-inline clearfix field-type-text-long field-pseudo-field field-pseudo-field--author-list"><div class="field-label">Authors:&nbsp;</div><div class="field-items"><a href="#Andrea Spadaccini" title="Andrea Spadaccini">Andrea Spadaccini</a></div></div><div class="field field-name-field-lv2-shepherds field-type-user-reference field-label-inline clearfix"><div class="field-label">Article shepherded by:&nbsp;</div><div class="field-items field-items"><div class="field-item odd"><span class="usenix-user-reference-names">Laura Nolan</span></div></div></div>
<div class="paragraphs-items paragraphs-items-field-lv2-body paragraphs-items-field-lv2-body-full paragraphs-items-full">
<div class="field field-name-field-lv2-body field-type-paragraphs field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><div class="paragraphs-item paragraphs-item-single-column-text paragraphs-item-single-column-text paragraphs-item-full paragraphs-item-5739">
<div class="content">
<div class="field field-name-field-single-column-text field-type-text-long field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><p><span><span>The technical side of Site Reliability Engineering is reasonably well-understood and documented. There are now a significant number of books, articles, and conferences that deal with how to reason about, operate and improve the reliability of large-scale distributed systems. However, the dynamics of SRE teams themselves are less discussed [1]. Effectively operating an SRE team or organization is very rewarding, but can also be very challenging.</span></span></p><p><span><span>In this article, I will share five lessons I learned about starting SRE teams (or engagements, or organizations). There will be a follow-up piece containing five lessons I learned about sustaining healthy SRE engagements at steady state.  I presented a talk version of these articles at SRECon21 [2].</span></span></p></div></div></div> </div>
</div>
</div><div class="field-item even"><div class="paragraphs-item paragraphs-item-callout paragraphs-item-callout paragraphs-item-full paragraphs-item-5740">
<div class="content">
<div class="field field-name-field-callout-text field-type-text-long field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><p>Effectively operating an SRE team or organization is very rewarding, but can also be very challenging.</p></div></div></div> </div>
</div>
</div><div class="field-item odd"><div class="paragraphs-item paragraphs-item-single-column-text paragraphs-item-single-column-text paragraphs-item-full paragraphs-item-5741">
<div class="content">
<div class="field field-name-field-single-column-sub field-type-text field-label-hidden"><div class="field-items field-items"><div class="field-item odd">Know what you want from SRE</div></div></div><div class="field field-name-field-single-column-text field-type-text-long field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><p>The very first question I usually ask to anyone who is planning to start an SRE team or organization is: why do you want to "do SRE"?</p><p>I find that this question is very powerful. It is not easy to articulate a universal definition of SRE, and there are many possible ways of implementing SRE, including different team topologies and focus areas. Asking why probes into the motivations of the person you are talking to. It will also help surface possible differences in interpretation of what exactly SREs do and what an SRE team or organization can provide. It will help understand the goals, assess feasibility and suggest an appropriate SRE setup strategy.</p><p>SRE can be pretty expensive when implemented as a separate team (or organization) that shares work and ownership with a separate product development team (or organization), focusing on aspects related to reliability. This implementation, which is often used in large organizations, provides the following benefits (among others):</p><ul><li>dedicated planning for reliability work</li><li>a growing set of best practices and tools</li><li>an organization (SRE) which will grow to be more impactful than the sum of its parts, influencing more areas of the company than those it directly engages with.</li></ul><p>In order for this approach be effective, the organizational context has to be mature enough to support the separate SRE organization, and the investment needs to be tailored to the needs, the budget and the timeline.</p><p>Finally, I find it beneficial to write down in the very early days the goals of the SRE team and how success will be measured. Having in mind some clear goals, ideally tied to measurable results, will sharpen the focus of your SRE journey, and will allow you to demonstrate return on investment.</p></div></div></div> </div>
</div>
</div><div class="field-item even"><div class="paragraphs-item paragraphs-item-callout paragraphs-item-callout paragraphs-item-full paragraphs-item-5742">
<div class="content">
<div class="field field-name-field-callout-text field-type-text-long field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><p><span><span>Ideally, there should be two feedback loops: one from the product development team to SRE, as mentioned, and one from SRE to product development teams. This allows SREs to influence the product roadmap and make sure reliability and customer experience are top concerns in work planning sessions.</span></span></p></div></div></div> </div>
</div>
</div><div class="field-item odd"><div class="paragraphs-item paragraphs-item-single-column-text paragraphs-item-single-column-text paragraphs-item-full paragraphs-item-5743">
<div class="content">
<div class="field field-name-field-single-column-sub field-type-text field-label-hidden"><div class="field-items field-items"><div class="field-item odd">Align with business goals and customer needs</div></div></div><div class="field field-name-field-single-column-text field-type-text-long field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><p><span>SRE teams operate in a broader business context, and therefore must serve the company's business needs. They must do so in cooperation with their product development counterparts.</span></p><p><span>SRE teams are usually less subject to business pressure to deliver features, and enjoy greater freedom by design. But this freedom has a specific purpose: it is there to allow the team to identify reliability gaps and act on them. </span><span>This freedom demands discipline in identifying and executing impactful projects that are meaningful for the business and, ultimately, for customers. It is fundamental to set up feedback loops between the SRE team and the product development team to make sure SRE keeps serving business goals.</span></p><p><span>Ideally, there should be two feedback loops: one from the product development team to SRE, as mentioned, and one from SRE to product development teams, to influence the product roadmap and make sure reliability and customer experience are top concerns in work planning sessions.</span></p><p><span>The following table summarizes the most common planning documents</span><span> I've seen SRE teams share and agree with their development counterparts [3]. Depending on the degree of structure needed and the stage of the SRE engagement, you may want to use one or more of these documents. </span></p><div align="left"><table><colgroup><col /><col /><col /></colgroup><tbody><tr><td><p><strong>Type of document</strong></p></td><td><p><strong>Time horizon</strong></p></td><td><p><strong>Frequency of production/review</strong></p></td></tr><tr><td><p><span>Team Charter / Strategy</span></p></td><td><p><span>years</span></p></td><td><p><span>yearly or less frequently</span></p></td></tr><tr><td><p><span>Team Roadmap</span></p></td><td><p><span>months (small multiple of the planning cycle - e.g., a few quarters)</span></p></td><td><p><span>once or twice a year</span></p></td></tr><tr><td><p><span>Team planning / backlog</span></p></td><td><p><span>weeks to months</span></p></td><td><p><span>once or twice per planning cycle (e.g., quarter or semester)</span></p></td></tr></tbody></table></div><p> </p><p><span>A team charter is necessary to establish the basics of the SRE engagement and a high-level team strategy. The other documents may appear as the engagement matures and needs more direction and structure. However, w</span><span>riting documents is not sufficient. You need to make sure that the contents are reviewed with the product development team and agree on the given direction and specifics. Bidirectional communication and influence is vital.</span></p><p> </p></div></div></div> </div>
</div>
</div><div class="field-item even"><div class="paragraphs-item paragraphs-item-callout paragraphs-item-callout paragraphs-item-full paragraphs-item-5744">
<div class="content">
<div class="field field-name-field-callout-subtitle field-type-text field-label-hidden"><div class="field-items field-items"><div class="field-item odd">Anti-pattern: SRE roadmap drifts from product development team roadmap.</div></div></div><div class="field field-name-field-callout-text field-type-text-long field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><p><span><span>Anti-pattern: SRE roadmap drifts from product development team roadmap. Over time, SRE teams will grow confident in their operation and may - in good faith! - pursue objectives that are increasingly disconnected from the business goals.</span></span></p><p><span><span><br /></span></span></p><p><span><span>For example, after successfully delivering projects to reduce manual operational work (toil) it can be tempting to bring things to the next level and embark in a longer project that will further reduce toil. This kind of project may have a much lower return on investment.</span></span></p></div></div></div> </div>
</div>
</div><div class="field-item odd"><div class="paragraphs-item paragraphs-item-single-column-text paragraphs-item-single-column-text paragraphs-item-full paragraphs-item-5745">
<div class="content">
<div class="field field-name-field-single-column-sub field-type-text field-label-hidden"><div class="field-items field-items"><div class="field-item odd">Expertise matters</div></div></div><div class="field field-name-field-single-column-text field-type-text-long field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><p>This is something I learned very recently. And it goes very much against who I am: I am a generalist. I am not an expert in any of the areas I have worked on. I have shipped production code in multiple languages (C++, C#, etc.) but I can't call myself an expert in those languages. My approach is to have solid fundamentals and approach any problem as something I can learn how to solve.</p><p><span>While this can work in large organizations, or when there are very loose time constraints, it is not desirable in smaller contexts when it's important to deliver results reasonably quickly.</span></p><p>What works best is to seed teams with a few “T-shaped” experts: people with broad general knowledge as well as deep vertical knowledge about the technology stack that the SRE team will work with. They will speak the same language as the product development team and help establish credibility, in addition to multiplying the impact of the rest of the team by helping them learn the stack much quicker than if they were alone. If the SRE team doesn't have temporal or geographic proximity with the development team then engineers with relevant deep expertise will be even more crucial.</p><p><span>This is less important if there is a homogeneous production interface or toolset in the company - meaning that lots of knowledge is transferable - but this is not always the case.</span></p></div></div></div> </div>
</div>
</div><div class="field-item even"><div class="paragraphs-item paragraphs-item-single-column-text paragraphs-item-single-column-text paragraphs-item-full paragraphs-item-5746">
<div class="content">
<div class="field field-name-field-single-column-sub field-type-text field-label-hidden"><div class="field-items field-items"><div class="field-item odd">You cannot declare &quot;SRE&quot;</div></div></div><div class="field field-name-field-single-column-text field-type-text-long field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><p><span>How do you tell whether you are "doing SRE?". Just saying that you have successfully implemented SRE doesn't mean it is true. </span>Introducing SRE is a cultural change, and technology is only one part of the equation. As mentioned before, dedicated SRE organizations are complex and expensive. It takes time to implement a transition to SRE, both for a team and - on a larger scale, of course - for a division or a company. </p><p>How can we, then, judge success of an SRE transition?</p><p>Following from the first lesson, success can be measured as progress towards the stated goals. Did you improve (or set up!) the indicators you wanted to influence? Can you tell whether customers are receiving a more satisfactory level of service? Is the production posture of the services better?</p><p>Of course, not everything can be measured. There are some meta-indicators that I use to understand the health of the SRE transition, that could be considered proxies for SRE success:</p><ol><li><span>How is the product dev team interacting with the SRE team? Look for shared work, code reviews and design reviews across teams, meetings organized without managerial pressure.</span></li><li>What type of projects is the SRE team executing? Does the SRE team mostly do operational work, or are they undertaking increasingly more impactful projects, moving up as appropriate in the Dickerson Service Reliability Hierarchy [4]?</li><li>In case of larger SRE orgs, is there an SRE community, or just a set of SRE teams? The value of an SRE organization is greater than the sum of its parts: look for (and encourage) cross-team collaboration in any shape or form.</li></ol></div></div></div> </div>
</div>
</div><div class="field-item odd"><div class="paragraphs-item paragraphs-item-callout paragraphs-item-callout paragraphs-item-full paragraphs-item-5747">
<div class="content">
<div class="field field-name-field-callout-text field-type-text-long field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><p>The value of an SRE organization is greater than the sum of its parts: look for (and encourage) cross-team collaboration in any shape or form.</p></div></div></div> </div>
</div>
</div><div class="field-item even"><div class="paragraphs-item paragraphs-item-single-column-text paragraphs-item-single-column-text paragraphs-item-full paragraphs-item-5748">
<div class="content">
<div class="field field-name-field-single-column-sub field-type-text field-label-hidden"><div class="field-items field-items"><div class="field-item odd">Build Trust</div></div></div><div class="field field-name-field-single-column-text field-type-text-long field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><p><span>SRE relies heavily on shared ownership. </span>Trust between the SRE team and the product development team is vital. </p><p><span>At the executive level, trust is necessary to gain support, sponsorship and funding. If you are starting an SRE initiative, most likely you were trusted with funds to do so. At the executive-level trust can best be maintained by setting and delivering business goals. As a corollary, trust can be lost by diverging from business goals and delivering projects whose importance is not understood by product development execs. This is why it is crucial to make the SRE impact obvious and to communicate it in a way that can be broadly understood. If that is difficult, it may be time to revise the SRE strategy or roadmap.</span></p><p><span>At the level of senior individual contributors and management, SREs need to establish relationships with their peers in the product dev organization. This lets SREs gauge the product developers' level of understanding of SRE, and to establish common conceptual grounds. Managing expectations and delivering impactful, concrete and meaningful projects is key to establish trust with these stakeholders.</span></p><p><span>At the individual contributor level, SREs gain trust by taking the time to understand the product and its stack. Be curious as to why things are the way they are and suspend judgement when you ask questions. Improvements that may seem obvious to you are clearly not obvious to others, and there will be reasons for why things are the way they are. Start small and increase the scope of your ideas over time. </span></p><p>In general, I see two patterns for establishing trust:</p><ul><li><span>build alignment and exercise SRE advocacy, through continuous bidirectional feedback loops.</span></li><li><span>deliver complete, incrementally more impactful projects over time.</span></li></ul></div></div></div> </div>
</div>
</div><div class="field-item odd"><div class="paragraphs-item paragraphs-item-single-column-text paragraphs-item-single-column-text paragraphs-item-full paragraphs-item-5749">
<div class="content">
<div class="field field-name-field-single-column-sub field-type-text field-label-hidden"><div class="field-items field-items"><div class="field-item odd">Conclusion</div></div></div><div class="field field-name-field-single-column-text field-type-text-long field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><p><span>There are lots of factors to consider when starting an SRE team or organization, and this article is by no means exhaustive. </span></p><p>The basics are having clear motivations and goals, aligning with business needs, having the right kind of expertise to successfully bootstrap the team, paying attention to the cultural aspects of SRE and, last but not least, building trust at all levels in the organization.</p><p>In the second article in this series, we will cover 5 more items that I think every SRE team should take into account in order to sustainably deliver high-impact work.</p></div></div></div> </div>
</div>
</div><div class="field-item even"><div class="paragraphs-item paragraphs-item-callout paragraphs-item-callout paragraphs-item-full paragraphs-item-5750">
<div class="content">
<div class="field field-name-field-callout-subtitle field-type-text field-label-hidden"><div class="field-items field-items"><div class="field-item odd">References</div></div></div><div class="field field-name-field-callout-text field-type-text-long field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><p>[1] Matt Brown and Gustavo Franco's 'How SRE teams are organized, and how to get started' (<em>Google Cloud Blog</em>) is a useful introduction to SRE team types and topologies. See: <a href="https://cloud.google.com/blog/products/devops-sre/how-sre-teams-are-organized-and-how-to-get-started">https://cloud.google.com/blog/products/devops-sre/how-sre-teams-are-organized-and-how-to-get-started</a></p><p>[2] Andrea Spadaccini, '10 Lessons Learned in 10 Years of SRE', <em>USENIX SREcon. </em>See h<a href="//www.usenix.org/conference/srecon21/presentation/spadaccini">ttps://www.usenix.org/conference/srecon21/presentation/spadaccini</a></p><p>[3] Shylaja Nukala and Vivek Rau, 'Why SRE Documents Matter', <em>ACM Queue </em>16:4 (October 2018). See <a href="https://queue.acm.org/detail.cfm?id=3283589">https://queue.acm.org/detail.cfm?id=3283589</a></p><p>[4] Betsy Beyer, Chris Jones, Jennifer Petoff and Niall Murphy, 'Part III: Practices' in <em>Site Reliability Engineering: How Google Runs Production Systems </em>(O'Reilly, 2018). See <a href="https://sre.google/sre-book/part-III-practices/#fig_part-practices_reliability-hierarchy">https://sre.google/sre-book/part-III-practices/#fig_part-practices_reliability-hierarchy</a></p></div></div></div> </div>
</div>
</div></div></div></div>
</div><div class="field field-name-field-lv2-tags field-type-taxonomy-term-reference field-label-above"><div class="field-label">Article Categories:&nbsp;</div><div class="field-items field-items"><div class="field-item odd">SRE</div></div></div><div class="psuedo-last-updated">Last updated February 8, 2023</div><div class="field-collection-container clearfix"><div class="field field-name-field-authors field-type-field-collection field-label-above"><div class="field-label">Authors:&nbsp;</div><div class="field-items field-items"><div class="field-item odd"><div class="field-collection-view clearfix view-mode-full field-collection-view-final"><div class="entity-plus entity-field-collection-item field-collection-item-field-authors clearfix">
<div class="content">
<a class="anchor" name="Andrea Spadaccini"></a><div class="field field-name-field-collection-author-image field-type-image field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><img src="https://www.usenix.org/sites/default/files/styles/author_bio/public/screenshot_2022-02-02_at_20.19.21.png" width="138" height="138" alt="" /></div></div></div><div class="field field-name-field-collection-author-bio field-type-text-long field-label-hidden"><div class="field-items field-items"><div class="field-item odd">Andrea is a Principal Software Engineer in SRE at Microsoft Azure, where he currently acts as a tech lead for all the Azure SRE teams. He works on cross-team projects (currently, SLOs for all Azure products), while also focusing on SRE for Azure SQL products.
He joined Microsoft in 2018. Before that, he worked as a Site Reliability Engineer for Google since 2011, in various technical and management roles across SRE teams in CorpEng, Ads, and Google Cloud Platform. He's been lucky enough to contribute to the first and second SRE books, mostly to the chapters about on-call.
He received his Ph.D. in Computer Engineering from the University of Catania in 2012, with a thesis on novel traits for biometric recognition. Andrea is the maintainer of the free CPU simulator EduMIPS64.
Twitter: @lupino3. Github: @lupino3</div></div></div><div class="field field-name-field-collection-author-email field-type-email field-label-hidden"><div class="field-items field-items"><div class="field-item odd"><a href="/cdn-cgi/l/email-protection#b9d8d7ddcbdcd897cac9d8ddd8dadad0d7d0f9d4d0dacbd6cad6dfcd97dad6d4"><span class="__cf_email__" data-cfemail="92f3fcf6e0f7f3bce1e2f3f6f3f1f1fbfcfbd2fffbf1e0fde1fdf4e6bcf1fdff">[email&#160;protected]</span></a></div></div></div> </div>
</div>
</div></div></div></div></div> </div>
<ul class="links inline"><li class="comment-forbidden odd first last"><span><span class="comment-1"><a id="comments" href="/user/login?destination=comment/reply/278934%23comment-form">Log in</a> to post comments</span></span></li></ul>
</article>
</div>
</main>
</div>
<footer role="contentinfo">
<div class="layout-footer container">
<div class="block block-block-motherboard-footer-logo">
<div class="block-content">
<a href="/"><img src="/themes/motherboard/images/usenix-logo-white.png" style="width: 160px; height: auto;" alt="USENIX logo" /></a> </div>
</div>
<div class="block block-menu-menu-motherboard-footer-menu" role="navigation">
<div class="block-content">
<ul class="menu-tree menu" data-menu-style="tree" data-clickdown="0" data-collapse="default"><li class="first leaf menu-mlid-48511"><a href="/contact">Contact USENIX</a></li>
<li class="last leaf menu-mlid-48512"><a href="/privacy-policy">Privacy Policy</a></li>
</ul> </div>
</div>
<div class="block block-block-motherboard-footer-copyright">
<div class="block-content">
<p>© USENIX 2025<br />
EIN 13-3055038
<br />
Website designed and built by <a href="https://www.giantrabbit.com" target="_blank">Giant Rabbit LLC</a><br />
Powered by <a href="https://backdropcms.org/" target="_blank">Backdrop CMS</a></p>
</div>
</div>
</div>
</footer>
</div>
<noscript aria-hidden="true"><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-WQSPGJT"
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript> <script data-cfasync="false" src="/cdn-cgi/scripts/5c5dd728/cloudflare-static/email-decode.min.js"></script><script>(function(){function c(){var b=a.contentDocument||(a.contentWindow&&a.contentWindow.document);if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'a39c70596d9f2f1a',t:'MTc4OTE5MDE3Mw=='};var a=document.createElement('script');a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();</script><script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" data-cf-beacon='{"version":"2024.11.0","token":"04ec313fe1c44bdeb95db4ee6e679a22","r":1,"spa":2}' crossorigin="anonymous"></script>
</body>
</html>