965 lines
61 KiB
HTML
965 lines
61 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<!-- Google Tag Manager -->
|
|
<script>
|
|
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
|
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
|
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
|
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
|
ga('create', 'UA-75468017-1', 'auto');
|
|
ga('send', 'pageview');
|
|
</script>
|
|
<script>
|
|
(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});
|
|
var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;
|
|
j.src= 'https://www.googletagmanager.com/gtag/js?id='+i+dl;f.parentNode.insertBefore(j,f); })
|
|
(window,document,'script','dataLayer','GTM-WVF23W3');
|
|
</script>
|
|
<!-- End Google Tag Manager -->
|
|
|
|
<meta charset="utf-8">
|
|
<meta content="initial-scale=1, minimum-scale=1, width=device-width" name="viewport">
|
|
<title>Google SRE lessons - key principles of site reliability engineering</title>
|
|
<meta name="description" content="Learn about the eleven lessons, from two decades, shared by site reliability engineers at Google, essential SRE lessons and core principles of SRE.">
|
|
<link rel="canonical" href="https://sre.google/resources/practices-and-processes/twenty-years-of-sre-lessons-learned/">
|
|
<link rel="apple-touch-icon-precomposed" sizes="180x180" href="https://lh3.googleusercontent.com/gWsjPZC9Fbk97ga1qFDlK88JynGoiUIF_mFTx-DocUWT94qi8OI7WIkD-5_VXfIvMQk1hpgdLZEDK9UJgjrevhz85GRmU60wJ9Mx">
|
|
<link rel="icon" type="image/png" sizes="32x32" href="https://lh3.googleusercontent.com/gWsjPZC9Fbk97ga1qFDlK88JynGoiUIF_mFTx-DocUWT94qi8OI7WIkD-5_VXfIvMQk1hpgdLZEDK9UJgjrevhz85GRmU60wJ9Mx">
|
|
<link rel="icon" type="image/png" sizes="16x16" href="https://lh3.googleusercontent.com/gWsjPZC9Fbk97ga1qFDlK88JynGoiUIF_mFTx-DocUWT94qi8OI7WIkD-5_VXfIvMQk1hpgdLZEDK9UJgjrevhz85GRmU60wJ9Mx">
|
|
<link rel="shortcut icon" href="https://lh3.googleusercontent.com/gWsjPZC9Fbk97ga1qFDlK88JynGoiUIF_mFTx-DocUWT94qi8OI7WIkD-5_VXfIvMQk1hpgdLZEDK9UJgjrevhz85GRmU60wJ9Mx">
|
|
<link rel="preconnect" href="https://lh3.googleusercontent.com/" crossorigin>
|
|
<link rel="preconnect" href="https://kstatic.googleusercontent.com/" crossorigin>
|
|
<link rel="preconnect" href="https://fonts.googleapis.com/" crossorigin>
|
|
<link rel="preconnect" href="https://www.gstatic.com/" crossorigin>
|
|
<link rel="preconnect" href="https://www.google-analytics.com/" crossorigin>
|
|
<link rel="preconnect" href="https://www.googletagmanager.com" crossorigin>
|
|
<link rel="preconnect" href="https://www.youtube.com/" crossorigin>
|
|
|
|
<link rel="preload" as="style" href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700|Google+Sans:400,500|Product+Sans:400&lang=en&display=swap">
|
|
<link rel="preload" as="style" href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
|
|
<link href="/static/css/index.min.css?cache=4b4f0ea" rel="stylesheet" rel="preload" as="style">
|
|
<link as="style" href="https://www.gstatic.com/glue/cookienotificationbar/cookienotificationbar.min.css" rel="stylesheet">
|
|
<script defer src="/static/js/detect.min.js?cache=8d84ad0"></script>
|
|
</head>
|
|
|
|
<body class="glue-body">
|
|
<!-- Google Tag Manager (noscript) -->
|
|
<noscript>
|
|
<iframe class="no-script-iframe" src="https://www.googletagmanager.com/ns.html?id=GTM-WVF23W3"></iframe>
|
|
</noscript>
|
|
<!-- End Google Tag Manager (noscript) -->
|
|
|
|
<header class="glue-header glue-header--single header" glue-header-options='{"steppedNav": "true"}'>
|
|
<div class="glue-header__bar glue-header__bar--desktop glue-header__drawer" id="header-drawer">
|
|
<div class="glue-header__tier">
|
|
<div class="glue-header__container">
|
|
<div class="glue-header__lock-up">
|
|
<div class="glue-header__logo">
|
|
<a class="glue-header__logo-link" href="/" aria-label="Google" title="Google">
|
|
<div class="glue-header__logo-container">
|
|
<svg role="presentation" aria-hidden="true" class="glue-header__logo-svg" alt="Google">
|
|
<use xlink:href="/static/img/glue-icons.svg#google-color-logo"></use>
|
|
</svg>
|
|
</div>
|
|
<p class="glue-header__logo--product header-product-logo">Site Reliability Engineering</p>
|
|
</a>
|
|
</div>
|
|
<a href="#page-content" class="glue-header__skip-content" aria-label="Jump to Content" title="Jump to Content">
|
|
<p class="glue-header__skip-content-text">Jump to Content</p>
|
|
</a>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="glue-header__container glue-header__stepped-nav">
|
|
<div class="glue-header__stepped-nav-controls-container">
|
|
<div class="glue-header__stepped-nav-controls" tabindex="0" role="tabpanel">
|
|
<div class="glue-header__stepped-nav-controls-arrow">
|
|
<svg role="presentation" class="glue-icon glue-icon--18px" alt="Previous">
|
|
<use xlink:href="/static/img/glue-icons.svg#chevron-left"></use>
|
|
</svg>
|
|
<svg role="presentation" class="glue-icon glue-icon--18px glue-header__stepped-nav-subnav-icon" alt="Next">
|
|
<use xlink:href="/static/img/glue-icons.svg#chevron-right"></use>
|
|
</svg>
|
|
</div>
|
|
<div class="glue-header__stepped-nav-controls-title glue-header__link"></div>
|
|
</div>
|
|
</div>
|
|
<div class="glue-header__stepped-nav-menus"></div>
|
|
</div>
|
|
|
|
<div class="glue-header__container glue-header__container--flex-auto">
|
|
<nav class="glue-header__link-bar">
|
|
<ul id="list-1" class="glue-header__list glue-header__link-bar--arrow">
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/"
|
|
title="Home"
|
|
aria-label="Home"
|
|
>
|
|
Home
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link mega-menu-button"
|
|
href="#"
|
|
title="Resources"
|
|
aria-label="Resources"
|
|
aria-expanded="false">
|
|
<span class="no-event">
|
|
Resources
|
|
<i class="material-icons new-release-icon" data-release-date="2022-03-31"></i>
|
|
</span>
|
|
<svg role="presentation" class="glue-icon glue-icon--20px mega-menu-expand-icon no-event" alt="Resources">
|
|
<use xlink:href="/static/img/glue-icons.svg#expand-more"></use>
|
|
</svg>
|
|
</a>
|
|
<section class="glue-mod-elevation-1 mega-menu js-hide">
|
|
<div class="glue-page">
|
|
<div class="glue-grid glue-mod-spacer-4-top glue-mod-spacer-4-bottom">
|
|
<div class="glue-grid__col glue-grid__col--span-3 glue-mod-spacer-4-bottom">
|
|
<ul class="mega-menu-container">
|
|
<div class="mega-menu-image">
|
|
<svg fill="none" height="24" viewBox="0 0 22 24" width="22"><g fill="#1a73e8"><path d="m11.9736 7.42627c-1.0507 0-1.9019.85333-1.9019 1.90667 0 1.05336.8512 1.90666 1.9019 1.90666s1.9019-.8533 1.9019-1.90666c0-1.05334-.8512-1.90667-1.9019-1.90667z"></path><path d="m11.9726 0c-4.98758 0-9.04417 3.92-9.28357 8.85333l-2.553663 3.41337c-.332507.44-.0133 1.0666.532013 1.0666h1.99505v4c0 1.4667 1.19702 2.6667 2.66006 2.6667h1.33003v4h9.31018v-6.24c3.1389-1.4933 5.3202-4.7067 5.3202-8.42667 0-5.16-4.163-9.33333-9.3103-9.33333zm3.9901 9.33333c0 .17334-.0133.34667-.0266.52l1.104.87997c.1064.08.133.2134.0665.3334l-1.0641 1.8533c-.0665.12-.2128.16-.3192.12l-1.3167-.5333c-.2793.2133-.5719.3866-.8911.52l-.2128 1.4133c-.0133.1333-.133.2267-.266.2267h-2.1281c-.133 0-.2394-.0934-.266-.2267l-.1995-1.4133c-.3325-.1334-.62511-.3067-.90442-.52l-1.31673.5333c-.1197.04-.266 0-.3325-.12l-1.06403-1.8533c-.0665-.1067-.0399-.2534.0665-.3334l1.11723-.87997c-.0133-.17333-.0266-.34666-.0266-.52 0-.17333.0266-.36.0532-.52l-1.13053-.88c-.1064-.08-.133-.21333-.0665-.34666l1.06403-1.84c.0665-.12.1995-.16.3192-.12l1.33003.53333c.26601-.2.57192-.38667.89112-.52l.1995-1.41333c.0266-.13334.133-.22667.266-.22667h2.1281c.133 0 .2394.09333.266.22667l.1995 1.41333c.3192.13333.6118.30667.8911.52l1.33-.53333c.1197-.04.266 0 .3192.12l1.0641 1.84c.0665.12.0399.26666-.0665.34666l-1.1306.88c.0399.16.0532.33334.0532.52z"></path></g></svg>
|
|
</div>
|
|
<li class="mega-menu-tile">
|
|
<p class="glue-cta mega-menu-title">
|
|
<a class="text-decoration"
|
|
href="/resources/"
|
|
title="Latest resources"
|
|
aria-label="Latest resources">
|
|
Latest resources
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption glue-mod-spacer-3-top">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/resources/practices-and-processes/ai-engineering-reliable-operations/"
|
|
title="AI Engineering Reliable Operations"
|
|
aria-label="AI Engineering Reliable Operations"
|
|
>
|
|
AI Engineering Reliable Operations
|
|
</a>
|
|
<span class="glue-eyebrow new-link-text">New!</span>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/resources/videos/"
|
|
title="Google SRE Video Gallery"
|
|
aria-label="Google SRE Video Gallery"
|
|
>
|
|
Google SRE Video Gallery
|
|
</a>
|
|
<span class="glue-eyebrow new-link-text">New!</span>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/resources/practices-and-processes/product-focused-reliability-for-sre/"
|
|
title="Product-Focused Reliability for SRE"
|
|
aria-label="Product-Focused Reliability for SRE"
|
|
>
|
|
Product-Focused Reliability for SRE
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/20/"
|
|
title="Twentieth Anniversary"
|
|
aria-label="Twentieth Anniversary"
|
|
>
|
|
Twentieth Anniversary
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="https://goo.gle/sre-20-ll"
|
|
title="Twenty years of SRE lessons learned"
|
|
aria-label="Twenty years of SRE lessons learned"
|
|
>
|
|
Twenty years of SRE lessons learned
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/prodverbs/"
|
|
title="Prodverbs"
|
|
aria-label="Prodverbs"
|
|
>
|
|
Prodverbs
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/resources/practices-and-processes/sre-fundamentals-course/"
|
|
title="SRE Fundamentals"
|
|
aria-label="SRE Fundamentals"
|
|
>
|
|
SRE Fundamentals
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/resources/practices-and-processes/measuring-reliability/"
|
|
title="Measuring Reliability"
|
|
aria-label="Measuring Reliability"
|
|
>
|
|
Measuring Reliability
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/resources/practices-and-processes/no-heroes/"
|
|
title="Why Heroism is Bad"
|
|
aria-label="Why Heroism is Bad"
|
|
>
|
|
Why Heroism is Bad
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/resources/practices-and-processes/stpa/"
|
|
title="System Theoretic Process Analysis"
|
|
aria-label="System Theoretic Process Analysis"
|
|
>
|
|
System Theoretic Process Analysis
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/next26/"
|
|
title="Ask an SRE at Next '26"
|
|
aria-label="Ask an SRE at Next '26"
|
|
>
|
|
Ask an SRE at Next '26
|
|
</a>
|
|
</p>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
<div class="glue-grid__col glue-grid__col--span-3 glue-mod-spacer-4-bottom">
|
|
<ul class="mega-menu-container">
|
|
<div class="mega-menu-image">
|
|
<svg fill="none" height="23" viewBox="0 0 22 23" width="22"><path d="m0 .087402h21.9868v22h-21.9868z" fill="#fff"></path><path d="m17.0624.0874023h-13.05471c-1.19668 0-2.17578.9899977-2.17578 2.1999977v17.6c0 1.21.9791 2.2 2.17578 2.2h13.05471c1.1966 0 2.1757-.99 2.1757-2.2v-17.6c0-1.21-.9791-2.1999977-2.1757-2.1999977zm-13.05471 2.1999977h5.43945v8.8l-2.71973-1.65-2.71972 1.65z" fill="#1a73e8"></path></svg>
|
|
</div>
|
|
<li class="mega-menu-tile">
|
|
<p class="glue-cta mega-menu-title">
|
|
<a class="text-decoration"
|
|
href="/books/"
|
|
title="Books"
|
|
aria-label="Books">
|
|
Books
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption glue-mod-spacer-3-top">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/books/building-secure-reliable-systems/"
|
|
title="Building Secure & Reliable Systems"
|
|
aria-label="Building Secure & Reliable Systems"
|
|
>
|
|
Building Secure & Reliable Systems
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/workbook/table-of-contents/"
|
|
title="The Site Reliability Workbook"
|
|
aria-label="The Site Reliability Workbook"
|
|
>
|
|
The Site Reliability Workbook
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/sre-book/table-of-contents/"
|
|
title="Site Reliability Engineering"
|
|
aria-label="Site Reliability Engineering"
|
|
>
|
|
Site Reliability Engineering
|
|
</a>
|
|
</p>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
<div class="glue-grid__col glue-grid__col--span-3 glue-mod-spacer-4-bottom">
|
|
<ul class="mega-menu-container">
|
|
<div class="mega-menu-image">
|
|
<svg fill="none" height="23" viewBox="0 0 22 23" width="22"><path d="m0 .087402h21.9354v22h-21.9354z" fill="#fff"></path><path d="m19.4981.0874023h-17.06084c-1.34049 0-2.43726 1.0999977-2.43726 2.4444477v17.11115c0 1.3444 1.09677 2.4444 2.43726 2.4444h17.06084c1.3405 0 2.4373-1.1 2.4373-2.4444v-17.11115c0-1.34445-1.0968-2.4444477-2.4373-2.4444477zm-6.0932 17.1110977h-8.53037v-2.4444h8.53037zm3.6559-4.8889h-12.18627v-2.44442h12.18627zm0-4.88886h-12.18627v-2.44445h12.18627z" fill="#1a73e8"></path></svg>
|
|
</div>
|
|
<li class="mega-menu-tile">
|
|
<p class="glue-cta mega-menu-title">
|
|
<a class="text-decoration"
|
|
href="/mobaa/"
|
|
title="Mobaa"
|
|
aria-label="Mobaa">
|
|
Mobaa
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption glue-mod-spacer-3-top">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/mobaa/2024/"
|
|
title="2024 Gallery"
|
|
aria-label="2024 Gallery"
|
|
>
|
|
2024 Gallery
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/mobaa/2022/"
|
|
title="2022 Gallery"
|
|
aria-label="2022 Gallery"
|
|
>
|
|
2022 Gallery
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/mobaa/frayed/"
|
|
title="2020 Gallery"
|
|
aria-label="2020 Gallery"
|
|
>
|
|
2020 Gallery
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="https://goo.gle/mobaa-vector"
|
|
title="Vector Methods"
|
|
aria-label="Vector Methods"
|
|
>
|
|
Vector Methods
|
|
</a>
|
|
</p>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
<div class="glue-grid__col glue-grid__col--span-3 glue-mod-spacer-4-bottom">
|
|
<ul class="mega-menu-container">
|
|
<div class="mega-menu-image">
|
|
<svg fill="none" height="28" viewBox="0 0 28 28" width="28"><path d="m.000977 0h27.9958v27.8329h-27.9958z" fill="#fff"></path><path d="m5.09014 15.4092v5.0606l8.90776 4.8328 8.9078-4.8328v-5.0606l-8.9078 4.8329zm8.90776-12.87899-13.9979 7.59079 13.9979 7.5908 11.4528-6.2118v8.7421h2.5451v-10.1211z" fill="#1a73e8"></path></svg>
|
|
</div>
|
|
<li class="mega-menu-tile">
|
|
<p class="glue-cta mega-menu-title">
|
|
<a class="text-decoration"
|
|
href="https://goo.gle/sre-classroom"
|
|
title="Classroom"
|
|
aria-label="Classroom">
|
|
Classroom
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption glue-mod-spacer-3-top">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="https://goo.gle/sre-classroom-distributed-pubsub"
|
|
title="Distributed PubSub"
|
|
aria-label="Distributed PubSub"
|
|
>
|
|
Distributed PubSub
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="https://goo.gle/sre-classroom-imageserver"
|
|
title="Distributed Image Server"
|
|
aria-label="Distributed Image Server"
|
|
>
|
|
Distributed Image Server
|
|
</a>
|
|
</p>
|
|
<p class="mega-menu-caption">
|
|
<a class="glue-caption text-decoration mega-menu-variant"
|
|
href="/resources/practices-and-processes/art-of-slos/"
|
|
title="The Art of SLO"
|
|
aria-label="The Art of SLO"
|
|
>
|
|
The Art of SLO
|
|
</a>
|
|
</p>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
<button class="mega-menu-close-btn btn-override" title="close" type="button" tabindex="0">
|
|
<svg role="presentation" class="glue-icon glue-icon--24px" alt="Close">
|
|
<use xlink:href="/static/img/glue-icons.svg#close"></use>
|
|
</svg>
|
|
</button>
|
|
</div>
|
|
</section> <ul id="list-2" class="glue-header__list glue-header__list--nested">
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="#"
|
|
title="Latest resources"
|
|
aria-label="Latest resources">
|
|
Latest resources
|
|
</a>
|
|
<ul id="list-3" class="glue-header__list glue-header__list--nested">
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/resources/"
|
|
title="Resources overview"
|
|
aria-label="Resources overview"
|
|
>
|
|
Resources overview
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/resources/practices-and-processes/ai-engineering-reliable-operations/"
|
|
title="AI Engineering Reliable Operations"
|
|
aria-label="AI Engineering Reliable Operations"
|
|
>
|
|
AI Engineering Reliable Operations
|
|
</a>
|
|
<span class="glue-eyebrow new-link-text">New!</span>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/resources/videos/"
|
|
title="Google SRE Video Gallery"
|
|
aria-label="Google SRE Video Gallery"
|
|
>
|
|
Google SRE Video Gallery
|
|
</a>
|
|
<span class="glue-eyebrow new-link-text">New!</span>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/resources/practices-and-processes/product-focused-reliability-for-sre/"
|
|
title="Product-Focused Reliability for SRE"
|
|
aria-label="Product-Focused Reliability for SRE"
|
|
>
|
|
Product-Focused Reliability for SRE
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/20/"
|
|
title="Twentieth Anniversary"
|
|
aria-label="Twentieth Anniversary"
|
|
>
|
|
Twentieth Anniversary
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item glue-header--is-active">
|
|
<a class="glue-header__link"
|
|
href="https://goo.gle/sre-20-ll"
|
|
title="Twenty years of SRE lessons learned"
|
|
aria-label="Twenty years of SRE lessons learned"
|
|
>
|
|
Twenty years of SRE lessons learned
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/prodverbs/"
|
|
title="Prodverbs"
|
|
aria-label="Prodverbs"
|
|
>
|
|
Prodverbs
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/resources/practices-and-processes/sre-fundamentals-course/"
|
|
title="SRE Fundamentals"
|
|
aria-label="SRE Fundamentals"
|
|
>
|
|
SRE Fundamentals
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/resources/practices-and-processes/measuring-reliability/"
|
|
title="Measuring Reliability"
|
|
aria-label="Measuring Reliability"
|
|
>
|
|
Measuring Reliability
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/resources/practices-and-processes/no-heroes/"
|
|
title="Why Heroism is Bad"
|
|
aria-label="Why Heroism is Bad"
|
|
>
|
|
Why Heroism is Bad
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/resources/practices-and-processes/stpa/"
|
|
title="System Theoretic Process Analysis"
|
|
aria-label="System Theoretic Process Analysis"
|
|
>
|
|
System Theoretic Process Analysis
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/next26/"
|
|
title="Ask an SRE at Next '26"
|
|
aria-label="Ask an SRE at Next '26"
|
|
>
|
|
Ask an SRE at Next '26
|
|
</a>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="#"
|
|
title="Books"
|
|
aria-label="Books">
|
|
Books
|
|
</a>
|
|
<ul id="list-3" class="glue-header__list glue-header__list--nested">
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/books/"
|
|
title="Books overview"
|
|
aria-label="Books overview"
|
|
>
|
|
Books overview
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/books/building-secure-reliable-systems/"
|
|
title="Building Secure & Reliable Systems"
|
|
aria-label="Building Secure & Reliable Systems"
|
|
>
|
|
Building Secure & Reliable Systems
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/workbook/table-of-contents/"
|
|
title="The Site Reliability Workbook"
|
|
aria-label="The Site Reliability Workbook"
|
|
>
|
|
The Site Reliability Workbook
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/sre-book/table-of-contents/"
|
|
title="Site Reliability Engineering"
|
|
aria-label="Site Reliability Engineering"
|
|
>
|
|
Site Reliability Engineering
|
|
</a>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="#"
|
|
title="Mobaa"
|
|
aria-label="Mobaa">
|
|
Mobaa
|
|
</a>
|
|
<ul id="list-3" class="glue-header__list glue-header__list--nested">
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/mobaa/"
|
|
title="Mobaa overview"
|
|
aria-label="Mobaa overview"
|
|
>
|
|
Mobaa overview
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/mobaa/2024/"
|
|
title="2024 Gallery"
|
|
aria-label="2024 Gallery"
|
|
>
|
|
2024 Gallery
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/mobaa/2022/"
|
|
title="2022 Gallery"
|
|
aria-label="2022 Gallery"
|
|
>
|
|
2022 Gallery
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/mobaa/frayed/"
|
|
title="2020 Gallery"
|
|
aria-label="2020 Gallery"
|
|
>
|
|
2020 Gallery
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="https://goo.gle/mobaa-vector"
|
|
title="Vector Methods"
|
|
aria-label="Vector Methods"
|
|
>
|
|
Vector Methods
|
|
</a>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="#"
|
|
title="Classroom"
|
|
aria-label="Classroom">
|
|
Classroom
|
|
</a>
|
|
<ul id="list-3" class="glue-header__list glue-header__list--nested">
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="https://goo.gle/sre-classroom"
|
|
title="Classroom overview"
|
|
aria-label="Classroom overview"
|
|
>
|
|
Classroom overview
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="https://goo.gle/sre-classroom-distributed-pubsub"
|
|
title="Distributed PubSub"
|
|
aria-label="Distributed PubSub"
|
|
>
|
|
Distributed PubSub
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="https://goo.gle/sre-classroom-imageserver"
|
|
title="Distributed Image Server"
|
|
aria-label="Distributed Image Server"
|
|
>
|
|
Distributed Image Server
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/resources/practices-and-processes/art-of-slos/"
|
|
title="The Art of SLO"
|
|
aria-label="The Art of SLO"
|
|
>
|
|
The Art of SLO
|
|
</a>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/books/"
|
|
title="Books"
|
|
aria-label="Books"
|
|
>
|
|
Books
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/careers/"
|
|
title="Careers"
|
|
aria-label="Careers"
|
|
>
|
|
Careers
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/cloud/"
|
|
title="Cloud"
|
|
aria-label="Cloud"
|
|
>
|
|
Cloud
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/local/"
|
|
title="Local"
|
|
aria-label="Local"
|
|
>
|
|
Local
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/prodcast/"
|
|
title="Prodcast"
|
|
aria-label="Prodcast"
|
|
>
|
|
Prodcast
|
|
</a>
|
|
</li>
|
|
<li class="glue-header__item">
|
|
<a class="glue-header__link"
|
|
href="/spotlight/"
|
|
title="Spotlight"
|
|
aria-label="Spotlight"
|
|
>
|
|
Spotlight
|
|
</a>
|
|
</li>
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="glue-header__bar glue-header__bar--mobile header-bar-mobile">
|
|
<div class="glue-header__tier">
|
|
<div class="glue-header__container">
|
|
<div class="glue-header__lock-up">
|
|
<div class="glue-header__hamburger glue-header__hamburger--first-tier">
|
|
<div class="glue-header__hamburger-wrapper">
|
|
<button type="button" class="glue-header__drawer-toggle-btn"
|
|
aria-controls="header-drawer" aria-expanded="false" aria-label="Open the navigation drawer">
|
|
<svg role="presentation" class="glue-icon glue-icon--24px" alt="Header hamburger" aria-hidden="true">
|
|
<use xlink:href="/static/img/glue-icons.svg#menu"></use>
|
|
</svg>
|
|
</button>
|
|
</div>
|
|
</div>
|
|
<div class="glue-header__logo">
|
|
<a class="glue-header__logo-link" href="/" aria-label="Google" title="Google">
|
|
<div class="glue-header__logo-container">
|
|
<svg role="presentation" aria-hidden="true" class="glue-header__logo-svg" alt="Google">
|
|
<use xlink:href="/static/img/glue-icons.svg#google-color-logo"></use>
|
|
</svg>
|
|
</div>
|
|
<p class="glue-header__logo--product header-product-logo">Site Reliability Engineering</p>
|
|
</a>
|
|
</div>
|
|
<a href="#page-content" class="glue-header__skip-content" aria-label="Jump to Content" title="Jump to Content">
|
|
<p class="glue-header__skip-content-text">Jump to Content</p>
|
|
</a>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="glue-header__drawer-backdrop"></div>
|
|
</header> <main class="main-content" id="page-content">
|
|
<section class="glue-page glue-mod-spacer-5-bottom twenty-years-lessons">
|
|
<div class="glue-grid">
|
|
<div class="glue-grid__col glue-grid__col--span-12">
|
|
<h1 class="glue-headline glue-headline--headline-3 glue-mod-spacer-5-top glue-mod-text-center glue-mod-font-weight-bold">
|
|
Lessons Learned from Twenty Years of Site Reliability Engineering
|
|
</h1>
|
|
<h2 class="glue-headline glue-headline--headline-4 glue-mod-spacer-4-top glue-mod-text-center">
|
|
Or, Eleven things we have learned as Site Reliability Engineers at Google
|
|
</h2>
|
|
<h3 class="glue-headline glue-headline--headline-4 glue-mod-spacer-5-top">
|
|
Authors
|
|
</h3>
|
|
<p class="glue-mod-spacer-2-top">
|
|
Adrienne Walcer, Kavita Guliani, Mikel Ward, Sunny Hsiao, and Vrai Stacey
|
|
</p>
|
|
<h3 class="glue-headline glue-headline--headline-4 glue-mod-spacer-3-top">
|
|
Contributors
|
|
</h3>
|
|
<p class="glue-mod-spacer-2-top">
|
|
Ali Biber, Guy Nadler, Luisa Fearnside, Thomas Holdschick, and Trevor Mattson-Hamilton
|
|
</p>
|
|
<h3 class="glue-headline glue-headline--headline-4 glue-mod-spacer-3-top">Foreword</h3>
|
|
<p class="glue-mod-spacer-2-top">A lot can happen in twenty years, especially when you're busy growing.</p>
|
|
<p class="glue-mod-spacer-2-top">Two decades ago, Google had a pair of small datacenters, each housing a few thousand servers, connected in a ring by a pair of 2.4G network links. We ran our private cloud (though we didn't call it that at the time) using Python scripts such as "Assigner" and "Autoreplacer" and "Babysitter" which operated on config files full of individual server names. We had a small database of the machines (MDB) which helped keep information about individual servers organized and durable. Our small team of engineers used scripts and configs to solve some common problems automatically, and to reduce the manual labor required to manage our little fleet of servers.</p>
|
|
<p class="glue-mod-spacer-2-top">Time passed, Google's users came for the search and stayed for the free GB of Gmail, and our fleet and network grew with it. Today, in terms of computing power, we are over 1,000 times as large as we were 20 years ago; in network, over 10,000 times as large, and we spend far less effort per server than we used to while enjoying much better reliability from our service stack. Our tools have evolved from a collection of Python scripts, to integrated ecosystems of services, to a unified platform which offers reliability by default. And our understanding of the problems and failure modes of distributed systems also evolved, as we experienced new classes of outages. We created the <a href="https://sre.google/sre-book/accelerating-sre-on-call/" title="Wheel of Misfortune">Wheel of Misfortune</a>, we wrote <a href="https://sre.google/sre-book/service-best-practices/" title="Service Best Practices guides">Service Best Practices guides</a>, we published Google's Greatest Hits, and today are <s>horrified</s> delighted to present:</p>
|
|
<p class="glue-mod-spacer-3-top glue-mod-text-right twenty-years-lessons-learned-italic">— Benjamin Treynor Sloss, Creator of Google SRE</p>
|
|
<p class="glue-headline glue-headline--headline-5 glue-mod-spacer-6-top glue-mod-text-center twenty-years-lessons-learned-italic">Lessons learned from two decades of Site Reliability Engineering</p>
|
|
<p class="glue-mod-spacer-4-top">Let's start back in 2016, when YouTube was offering your favorite videos such as "Carpool Karaoke with Adele" and the ever-catchy "Pen-Pineapple-Apple-Pen." YouTube experienced a fifteen-minute global outage, due to a bug in YouTube's distributed memory caching system, disrupting YouTube's ability to serve videos. Here are three lessons we learned from this incident.</p>
|
|
<div class="glue-mod-spacer-4-top twenty-years-lessons-learned">
|
|
<h2 class="glue-headline glue-headline--headline-2 twenty-years-lessons-learned-number">
|
|
1
|
|
</h2>
|
|
<p class="glue-headline glue-headline--headline-4 twenty-years-lessons-learned-content">
|
|
The riskiness of a mitigation should scale with the severity of the outage
|
|
</p>
|
|
</div>
|
|
<p class="glue-mod-spacer-2-top">There's a meme where one person posts a picture of a spider seen in their house, and the captain says, "TIME 2 MOVE 2 A NEW HOUSE!". The joke is that the incident (seeing a scary spider) would be responded to with a severe mitigation (abandon your current home and move to a new one). We, here in SRE, have had some interesting experiences in choosing a mitigation with more risks than the outage it's meant to resolve. During the aforementioned YouTube outage, a risky load-shedding process didn't fix the outage... it instead created a cascading failure.</p>
|
|
<p class="glue-mod-spacer-2-top">We learned the hard way that during an incident, we should monitor and evaluate the severity of the situation and choose a mitigation path whose riskiness is appropriate for that severity. In a best case scenario, a risky mitigation resolves an outage. In a worst case scenario, the risky mitigation misfires and the outage is prolonged by something that was intended to fix it. Additionally, if everything is broken, you can make an informed decision to bypass standard procedures.</p>
|
|
<div class="glue-mod-spacer-4-top twenty-years-lessons-learned">
|
|
<h2 class="glue-headline glue-headline--headline-2 twenty-years-lessons-learned-number">
|
|
2
|
|
</h2>
|
|
<p class="glue-headline glue-headline--headline-4 twenty-years-lessons-learned-content">
|
|
Recovery mechanisms should be fully tested before an emergency
|
|
</p>
|
|
</div>
|
|
<p class="glue-mod-spacer-2-top">An emergency fire evacuation in a tall city building is a terrible opportunity to use a ladder for the first time. Similarly, an outage is a terrible opportunity to try a risky load-shedding process for the first time. To keep your cool during a high-risk and high-stress situation, it's important to practice recovery mechanisms and mitigations beforehand and verify that:</p>
|
|
<ul>
|
|
<li>they'll do what you need them to do</li>
|
|
<li>you know how to do them</li>
|
|
</ul>
|
|
<p>Testing recovery mechanisms has a fun side effect of reducing the risk of performing some of these actions. Since this messy outage, we've doubled down on testing.</p>
|
|
<div class="glue-mod-spacer-4-top twenty-years-lessons-learned">
|
|
<h2 class="glue-headline glue-headline--headline-2 twenty-years-lessons-learned-number">
|
|
3
|
|
</h2>
|
|
<p class="glue-headline glue-headline--headline-4 twenty-years-lessons-learned-content">
|
|
Canary all changes
|
|
</p>
|
|
</div>
|
|
<p class="glue-mod-spacer-2-top">At one point, we wanted to push a caching configuration change. We were pretty sure that it would not lead to anything bad. But pretty sure is not 100% sure. Turns out, caching was a pretty critical feature for YouTube, and the config change had some unintended consequences that fully hobbled the service for 13 minutes. Had we <a href="https://sre.google/workbook/canarying-releases/" title="canaried those global changes">canaried those global changes</a> with a progressive rollout strategy, this outage could have been curbed before it had global impact. <a href="https://storage.googleapis.com/pub-tools-public-publication-data/pdf/24017e52c907294589604a29a86f158828eda078.pdf" rel="noopener noreferrer" target="_blank" title="Read more">Read more about the canary strategy in this paper</a>, and <a href="https://www.usenix.org/conference/srecon18europe/presentation/davidovic" rel="noopener noreferrer" target="_blank" title="learn more">learn more in this video</a>.</p>
|
|
<p class="glue-mod-spacer-2-top">Around the same timeframe, YouTube's slightly younger sibling, Google Calendar, also experienced an outage which serves as the backdrop for the next two lessons.</p>
|
|
<div class="glue-mod-spacer-4-top twenty-years-lessons-learned">
|
|
<h2 class="glue-headline glue-headline--headline-2 twenty-years-lessons-learned-number">
|
|
4
|
|
</h2>
|
|
<p class="glue-headline glue-headline--headline-4 twenty-years-lessons-learned-content">
|
|
Have a "Big Red Button"
|
|
</p>
|
|
</div>
|
|
<p class="glue-mod-spacer-2-top">A "Big Red Button" is a unique but highly practical safety feature: it should kick off a simple, easy-to-trigger action that reverts whatever triggered the undesirable state to (ideally) shut down whatever's happening. "Big Red Buttons" come in many shapes and sizes—and it's important to identify what those big red buttons might be before you submit a potentially risky action. We once narrowly missed a major outage because the engineer who submitted the would-be-triggering change unplugged their desktop computer before the change could propagate. So when planning your major rollouts, consider What is my big red button? Ensure every service dependency has a "big red button" to exercise in an emergency. See <a href="https://www.oreilly.com/content/generic-mitigations/" rel="noopener noreferrer" target="_blank" title="Generic Mitigations">"Generic Mitigations"</a> for more!</p>
|
|
<div class="glue-mod-spacer-4-top twenty-years-lessons-learned">
|
|
<h2 class="glue-headline glue-headline--headline-2 twenty-years-lessons-learned-number">
|
|
5
|
|
</h2>
|
|
<p class="glue-headline glue-headline--headline-4 twenty-years-lessons-learned-content">
|
|
Unit tests alone are not enough - integration testing is also needed
|
|
</p>
|
|
</div>
|
|
<p class="glue-mod-spacer-2-top">Ahh.... unit tests. They verify that an individual component can perform the way we need it to. Unit tests have intentionally limited scope, and are super helpful, but they also don't fully replicate the runtime environment and productionized demands that might exist. For this reason, we are big advocates of integration testing! We can use integration tests to verify that jobs and tasks can perform a cold start. Will things work the way we want them to? Will components work together the way we want them too? Will these components successfully create the system we want them to? This lesson was learned during a Calendar outage in which our testing didn't follow the same path as real use, resulting in plenty of testing... that didn't help us assess how a change would perform in reality.</p>
|
|
<p class="glue-mod-spacer-2-top">Shifting to an incident that happened in February 2017, we find our next two lessons.</p>
|
|
<p class="glue-mod-spacer-2-top">First, unavailable OAuth tokens caused millions of users to be logged out of devices and services, and 32,000 OnHub and Google WiFi devices to perform a factory reset. Manual account recovery claims jumped by 10x because of failed logins. It took Google about 12 hours to fully recover from the outage.</p>
|
|
<div class="glue-mod-spacer-4-top twenty-years-lessons-learned">
|
|
<h2 class="glue-headline glue-headline--headline-2 twenty-years-lessons-learned-number">
|
|
6
|
|
</h2>
|
|
<p class="glue-headline glue-headline--headline-4 twenty-years-lessons-learned-content">
|
|
COMMUNICATION CHANNELS! AND BACKUP CHANNELS!! AND BACKUPS FOR THOSE BACKUP CHANNELS!!!
|
|
</p>
|
|
</div>
|
|
<p class="glue-mod-spacer-2-top">Yes, it was a bad time. You want to know what made it worse? Teams were expecting to be able to use Google Hangouts and Google Meet to manage the incident. But when 350M users were logged out of their devices and services... relying on these Google services was, in retrospect, kind of a bad call. Ensure that you have non-dependent backup communication channels, and that you have tested them.</p>
|
|
<p class="glue-mod-spacer-2-top">Then, the same 2017 incident led us to better understand <a href="https://sre.google/sre-book/addressing-cascading-failures/#:~:text=Graceful%20degradation%20takes%20the%20concept,decreasing%20the%20quality%20of%20responses." title="graceful degradation">graceful degradation:</a></p>
|
|
<div class="glue-mod-spacer-4-top twenty-years-lessons-learned">
|
|
<h2 class="glue-headline glue-headline--headline-2 twenty-years-lessons-learned-number">
|
|
7
|
|
</h2>
|
|
<p class="glue-headline glue-headline--headline-4 twenty-years-lessons-learned-content">
|
|
Intentionally degrade performance modes
|
|
</p>
|
|
</div>
|
|
<p class="glue-mod-spacer-2-top">It's easy to think of availability as either "fully up" or "fully down" ... but being able to offer a continuous minimum functionality with a degraded performance mode helps to offer a more consistent user experience. So we've built degraded performance modes carefully and intentionally—so during rough patches, it might not even be user-visible (it might be happening right now!). Services should degrade gracefully and continue to function under exceptional circumstances.</p>
|
|
<p class="glue-mod-spacer-2-top">This next lesson is a recommendation to ensure that your last-line-of-defense system works as expected in extreme scenarios, such as natural disasters or cyber attacks, that result in loss of productivity or service availability.</p>
|
|
<div class="glue-mod-spacer-4-top twenty-years-lessons-learned">
|
|
<h2 class="glue-headline glue-headline--headline-2 twenty-years-lessons-learned-number">
|
|
8
|
|
</h2>
|
|
<p class="glue-headline glue-headline--headline-4 twenty-years-lessons-learned-content">
|
|
Test for Disaster resilience
|
|
</p>
|
|
</div>
|
|
<p class="glue-mod-spacer-2-top">Besides unit testing and integration testing, there are other types of very important testing: disaster resilience and recovery testing. While resilience testing verifies that your service or system could survive in the event of faults, latency, or disruptions, recovery testing verifies that your service can transition back to homeostasis after a full shutdown. Both should be critical pieces of your business continuity strategy—as described in <a href="https://queue.acm.org/detail.cfm?id=2371516" rel="noopener noreferrer" target="_blank" title="Weathering the Unexpected">"Weathering the Unexpected"</a>. A useful activity can also be sitting your team down and working through how some of these scenarios could theoretically play out—tabletop game style. This can also be a fun opportunity to explore those terrifying "What Ifs", for example, "What if part of your network connectivity gets shut down unexpectedly?".</p>
|
|
<div class="glue-mod-spacer-4-top twenty-years-lessons-learned">
|
|
<h2 class="glue-headline glue-headline--headline-2 twenty-years-lessons-learned-number">
|
|
9
|
|
</h2>
|
|
<p class="glue-headline glue-headline--headline-4 twenty-years-lessons-learned-content">
|
|
Automate your mitigations
|
|
</p>
|
|
</div>
|
|
<p class="glue-mod-spacer-2-top">In March of 2023, a near-simultaneous failure of multiple networking devices occurred in a few datacenters, resulting in a widespread packet loss. In this 6-day outage, an estimated 70% of services experienced varied levels of impact, depending on the location, service load, and configuration at the time of network failure.</p>
|
|
<p class="glue-mod-spacer-2-top">In such instances, you can reduce your mean time to resolution (MTTR), by automating mitigating measures done by hand. If there's a clear signal that a particular failure is occurring, then why can't that mitigation be kicked off in an automated way? Sometimes it is better to use an automated mitigation first and save the root-causing for after user impact has been avoided.</p>
|
|
<div class="glue-mod-spacer-4-top twenty-years-lessons-learned">
|
|
<h2 class="glue-headline glue-headline--headline-2 twenty-years-lessons-learned-number">
|
|
10
|
|
</h2>
|
|
<p class="glue-headline glue-headline--headline-4 twenty-years-lessons-learned-content">
|
|
Reduce the time between rollouts, to decrease the likelihood of the rollout going wrong
|
|
</p>
|
|
</div>
|
|
<p class="glue-mod-spacer-2-top">In March of 2022, a widespread outage in the payments system prevented customers from completing transactions, resulting in the Pokémon GO community day being postponed. The cause was the removal of a single database field, which should have been safe as all uses of that field were removed from the code beforehand. Unfortunately, a slow rollout cadence of one part of the system meant that the field was still being used by the live system.</p>
|
|
<p class="glue-mod-spacer-2-top">Having long delays between rollouts, especially in complex, multiple component systems, makes it extremely difficult to reason out the safety of a particular change. <a href="https://cloud.google.com/blog/products/devops-sre/using-the-four-keys-to-measure-your-devops-performance" rel="noopener noreferrer" target="_blank" title="Frequent rollouts">Frequent rollouts</a>—with the proper testing in place— lead to fewer surprises from this class of failure.</p>
|
|
<div class="glue-mod-spacer-4-top twenty-years-lessons-learned">
|
|
<h2 class="glue-headline glue-headline--headline-2 twenty-years-lessons-learned-number">
|
|
11
|
|
</h2>
|
|
<p class="glue-headline glue-headline--headline-4 twenty-years-lessons-learned-content">
|
|
A single global hardware version is a single point of failure
|
|
</p>
|
|
</div>
|
|
<p class="glue-mod-spacer-2-top">Having only one particular model of device to perform a critical function can make for simpler operations and maintenance. However, it means that if that model turns out to have a problem, that critical function is no longer being performed.</p>
|
|
<p class="glue-mod-spacer-2-top">This happened in March 2020 when a networking device that had an undiscovered zero-day bug, encountered a change in traffic patterns that triggered that bug. As the same model and version of the device was being used across the network, a substantial regional outage ensued. What prevented this from being a total outage was the presence of multiple network backbones that allowed high-priority traffic to be routed via a still working alternative.</p>
|
|
<p class="glue-mod-spacer-2-top">Latent bugs in critical infrastructure can lurk undetected until a seemingly innocuous event triggers them. Maintaining a diverse infrastructure, while incurring costs of its own, can mean the difference between a troublesome outage and a total one.</p>
|
|
<p class="glue-mod-spacer-5-top">
|
|
So there you have it! Eleven <a href="../../../20/">lessons learned, from two decades of Site Reliability Engineering at Google</a>. Why eleven? Well, you see, Google Site Reliability, with our rich history, is still in our <span class="twenty-years-lessons-learned-italic">prime</span>.
|
|
</p>
|
|
<div class="glue-mod-text-center glue-mod-spacer-5-top">
|
|
<a class="glue-button glue-button--high-emphasis" title="Download as PDF" aria-label="Download as PDF" href="/static/pdf/LessonsLearnedFromTwoDecades.pdf" target="_blank" rel="noopener noreferrer">
|
|
Download as PDF
|
|
</a>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
</main>
|
|
<footer class="glue-footer">
|
|
<section class="glue-footer__upper">
|
|
<div class="glue-social glue-social__group">
|
|
<p class="glue-social__title glue-social__title--inline">Follow us</p>
|
|
<ul class="glue-social__list glue-no-bullet">
|
|
<li class="glue-social__item">
|
|
<a class="glue-social__link" href="https://www.twitter.com/googlesre/" target="_blank" rel="noopener noreferrer" title="Follow us on Twitter">
|
|
<svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--24px glue-icon--social" alt="twitter-X">
|
|
<use xlink:href="/static/img/glue-icons.svg#twitter-X"></use>
|
|
</svg>
|
|
</a>
|
|
</li>
|
|
<li class="glue-social__item">
|
|
<a class="glue-social__link" href="https://www.youtube.com/google" target="_blank" rel="noopener noreferrer" title="Follow us on YouTube">
|
|
<svg role="presentation" aria-hidden="true" class="glue-icon glue-icon--24px glue-icon--social" alt="video-youtube">
|
|
<use xlink:href="/static/img/glue-icons.svg#video-youtube"></use>
|
|
</svg>
|
|
</a>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</section>
|
|
<section class="glue-footer__global">
|
|
<div class="glue-footer__logo">
|
|
<a href="https://www.google.com" target="_blank" rel="noopener noreferrer" aria-label="Google" title="Google">
|
|
<svg role="img" aria-hidden="true" class="glue-footer__logo-img" alt="Google">
|
|
<use xlink:href="/static/img/glue-icons.svg#google-solid-logo"></use>
|
|
</svg>
|
|
</a>
|
|
</div>
|
|
<ul class="glue-footer__global-links glue-no-bullet">
|
|
<li class="glue-footer__global-links-list-item">
|
|
<a class="glue-footer__link" href="https://about.google/" target="_blank" rel="noopener noreferrer" aria-label="About Google" title="About Google">About Google</a>
|
|
</li>
|
|
<li class="glue-footer__global-links-list-item">
|
|
<a class="glue-footer__link" href="https://about.google/products/" target="_blank" rel="noopener noreferrer" aria-label="Google products" title="Google products">Google products</a>
|
|
</li>
|
|
<li class="glue-footer__global-links-list-item">
|
|
<a class="glue-footer__link" href="https://policies.google.com/privacy" target="_blank" rel="noopener noreferrer" aria-label="Privacy" title="Privacy">Privacy</a>
|
|
</li>
|
|
<li class="glue-footer__global-links-list-item">
|
|
<a class="glue-footer__link" href="https://policies.google.com/terms" target="_blank" rel="noopener noreferrer" aria-label="Terms" title="Terms">Terms</a>
|
|
</li>
|
|
</ul>
|
|
<ul class="glue-footer__global-links glue-footer__global-links--extra glue-no-bullet">
|
|
<li class="glue-footer__global-links-list-item glue-footer__global-links-list-item--extra">
|
|
<a class="glue-footer__link" href="https://support.google.com/?hl=en" aria-label="Help" title="Help" target="_blank" rel="noopener noreferrer">
|
|
<svg role="img" aria-hidden="true" class="glue-icon glue-icon--24px glue-icon--footer-help" alt="Help">
|
|
<use xlink:href="/static/img/glue-icons.svg#help"></use>
|
|
</svg>
|
|
Help
|
|
</a>
|
|
</li>
|
|
</ul>
|
|
</section>
|
|
</footer>
|
|
<script defer src="https://www.gstatic.com/external_hosted/hammerjs/v2_0_2/hammer.min.js"></script>
|
|
<script defer src="/static/js/index.min.js?cache=50d0546"></script>
|
|
<script src="https://www.gstatic.com/glue/cookienotificationbar/cookienotificationbar.min.js"
|
|
data-glue-cookie-notification-bar-category="2B"
|
|
data-glue-cookie-notification-bar-language="en"
|
|
data-glue-cookie-notification-bar-site-id="sre.google">
|
|
</script>
|
|
</body>
|
|
</html> |