SRE weekly 所有文章

This commit is contained in:
2026-09-12 17:23:01 +08:00
parent 409b40ddcb
commit af7633f9dc
8486 changed files with 4489990 additions and 7 deletions

View File

@@ -0,0 +1,364 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>PagerDuty Post-Incident Reviews Howie</title>
<!-- Author and License -->
<meta name="dcterms.license" content="http://www.apache.org/licenses/LICENSE-2.0" />
<meta name="author" content="PagerDuty, Inc." />
<meta name="dcterms.rightsHolder" content="PagerDuty, Inc." />
<meta name="dcterms.rights" content="Copyright &copy; PagerDuty, Inc." />
<!-- Page Description -->
<meta name="description" content="A guide to the post-incident review process, outlining how to collect data, hold meetings, and publish learnings. Originally by the folks at Jeli." />
<meta name="robots" content="index, follow, archive" />
<meta name="generator" content="mkdocs-1.6.1, mkdocs-theme-pagerduty" />
<!-- Canonical Link -->
<link rel="canonical" href="https://howie-guide.pagerduty.com/">
<!-- Favicon -->
<link rel="shortcut icon" href="/assets/images/favicon.png" type="image/png" />
<link rel="icon" href="/assets/images/favicon.png" type="image/png" />
<link rel="apple-touch-icon" href="/assets/images/favicon.png" />
<!-- Mobile -->
<meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0" />
<meta name="theme-color" content="#005a24" />
<!-- Apple -->
<meta name="apple-mobile-web-app-title" content="PagerDuty Post-Incident Reviews Howie" />
<meta name="apple-mobile-web-app-capable" content="yes" />
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
<!-- Open Graph -->
<meta property="og:url" content="https://howie-guide.pagerduty.com/" />
<meta property="og:title" content="PagerDuty Post-Incident Reviews Howie" />
<meta property="og:site_name" content="PagerDuty Post-Incident Reviews Howie" />
<meta property="og:description" content="A guide to the post-incident review process, outlining how to collect data, hold meetings, and publish learnings. Originally by the folks at Jeli." />
<meta property="og:image" content="https://howie-guide.pagerduty.com/assets/images/covers/default.png" />
<meta property="og:locale" content="en_US" />
<meta property="og:type" content="website" />
<!-- Twitter -->
<meta name="twitter:card" content="summary_large_image" />
<meta name="twitter:title" content="PagerDuty Post-Incident Reviews Howie" />
<meta name="twitter:description" content="A guide to the post-incident review process, outlining how to collect data, hold meetings, and publish learnings. Originally by the folks at Jeli." />
<meta name="twitter:image" content="https://howie-guide.pagerduty.com/assets/images/covers/default.png" />
<!-- Style -->
<link rel="stylesheet" href="assets/stylesheets/fonts.css" />
<link rel="stylesheet" href="assets/stylesheets/reset.css" />
<link rel="stylesheet" href="assets/stylesheets/pagerduty.css" />
<link rel="stylesheet" href="assets/stylesheets/code.css" />
<link rel="stylesheet" href="authors.css" />
</head>
<body class="">
<header>
<div class="content-wrapper">
<div id="logo">
<a href="." title="Go to Post-Incident Howie homepage.">
<img src="./assets/images/logo.png" title="PagerDuty" />
</a>
<span class="subtitle">
Post-Incident Howie
</span>
</div>
<!-- This is a cheat for mobile search, allowing us to store state in a checkbox. -->
<input id="mobile-search-link" type="checkbox" />
<label id="mobile-search-link-selector" for="mobile-search-link" role="button"></label>
<div id="search">
<input name="q" id="mkdocs-search-query" type="search" placeholder="Search..." />
<div id="mkdocs-search-results"><p>No results found</p></div>
</div>
<!-- This is used for MkDocs search. Hardcoding to '/' so CSP is easier. -->
<script>var base_url = '/';</script>
</div>
</header>
<nav id="breadcrumbs">
<div class="content-wrapper">
<p><a href=".">Post-Incident Howie</a></p>
<h1>Home</h1>
</div>
</nav>
<main>
<!-- This is a cheat for mobile navigation, allowing us to store state in a checkbox. -->
<input id="mobile-nav-link" type="checkbox" />
<label id="mobile-nav-link-selector" for="mobile-nav-link" role="button"></label>
<nav>
<div id="nav-title">Post-Incident Howie</div>
<ul>
<li class="active">
<a title="Home" href=".">Home</a>
<ul id="toc">
<li class="anchor">
<a title="Welcome to the Howie Guide to post‑incident investigations!" href="#welcome-to-the-howie-guide-to-postincident-investigations">Welcome to the Howie Guide to post‑incident investigations!</a>
</li>
<li class="anchor">
<a title="A brief word about language in incident investigations" href="#a-brief-word-about-language-in-incident-investigations">A brief word about language in incident investigations</a>
</li>
<li class="anchor">
<a title="Glossary" href="#glossary">Glossary</a>
</li>
</ul>
</li>
<li>
<a title="Assign" href="assign/">Assign</a>
</li>
<li>
<a title="Identify" href="identify/">Identify</a>
</li>
<li>
<a title="Analyze" href="analyze/">Analyze</a>
</li>
<li>
<a title="Interview" href="interview/">Interview</a>
</li>
<li>
<a title="Calibrate" href="calibrate/">Calibrate</a>
</li>
<li>
<a title="Meet" href="meet/">Meet</a>
</li>
<li>
<a title="Report" href="report/">Report</a>
</li>
<li>
<a title="Distribute" href="distribute/">Distribute</a>
</li>
<li>
<a title="Closing Thoughts" href="thoughts/">Closing Thoughts</a>
</li>
<li>
<a title="Continued Learning" href="continue/">Continued Learning</a>
</li>
<li>
<a title="Appendices" href="appendices/">Appendices</a>
</li>
<li>
<a title="Authors & Acknowledgements" href="authack/">Authors & Acknowledgements</a>
</li>
</ul>
</nav>
<article>
<h1>Home</h1>
<section id="downloads">
</section>
<div class="callout">The Howie Guide to Post-Incident Investigations was originally published by Dr. Laura Maguire, Nora Jones, Vanessa Huerta Granda in collaboration with the Jeli team. PagerDuty acquired Jeli in 2023 and we're proud to rehome the Howie Guide as a part of the PagerDuty Ops Guide library.</div>
<h2 id="welcome-to-the-howie-guide-to-postincident-investigations">Welcome to the Howie Guide to post‑incident investigations!<a class="headerlink" href="#welcome-to-the-howie-guide-to-postincident-investigations" title="Permanent link">#</a></h2>
<p>The guide you’re about to read will provide you with an explanation of how to get the most out of your incidents. This process has been developed by a number of leading experts in the field and shows the steps to conduct an in-depth investigation.</p>
<p>We affectionately call this process the “how we got here process” (or “Howie process” for short), and it is applicable to companies of different sizes and stages of maturity with investigation programs.</p>
<h3 id="why-focus-on-learning-from-incidents">Why focus on learning from incidents?<a class="headerlink" href="#why-focus-on-learning-from-incidents" title="Permanent link">#</a></h3>
<p>Recent events have shown how critical digital services are to individuals, organizations and society as a whole. Now more than ever, learning from incidents is crucial to helping maintain service reliability requirements so companies can continue delivering on their commitments to clients and keep employees connected and productive</p>
<p>As Nora Jones notes:</p>
<blockquote>
<p>We’re at an age in software where learning from incidents is pivotal to our companies’ continued successes. There is a massive opportunity for Software Engineers to learn more about the applications of Resilience Engineering, Human Factors, and Systems Safety to their everyday work with the goal of learning how we can extract value and build expertise from incidents and surprises.<sup><a href="https://howie-guide.pagerduty.com/authack/#references">1</a></sup></p>
</blockquote>
<p>When we think about learning from our incidents, we often don’t have a good framework for what we can get out of this work. As noted cognitive psychologist and researcher Gary Klein explains in his book on human performance in everyday work conditions.<sup><a href="https://howie-guide.pagerduty.com/authack/#references">2</a></sup></p>
<div class="admonition tip">
<p class="admonition-title">Tip</p>
<p>Performance Improvement = Error Reduction + Insight Generation</p>
</div>
<p>So far, the industry has over-indexed in the “error reduction” part of the equation by emphasizing incident metrics like mean time to respond and not much on generating insights. By investing in learning and in generating quality insights from each individual incident, you will be able to provide context around your incident metrics and show a more complete picture of performance improvements.<sup><a href="https://howie-guide.pagerduty.com/authack/#references">3</a></sup></p>
<p>This guide provides some concrete strategies to help you begin to develop skills in generating insights and to help your company in developing an incident analysis program. It will start by walking you through the stages of an investigation: how to assign and accept an investigation, identify your incident data for analysis, prepare for interviews, and write a calibration document. As you wrap up your investigation, you will also learn how to lead a learning review meeting, complete an incident report, and integrate any additional findings and action items before you finalize and distribute your learnings within your organization.</p>
<p>While the steps outlined here represent a well-rounded way to review an investigation, our research shows that:</p>
<ol>
<li>Every organization has different needs, strengths, limitations, and goals for their incident investigation process, so some of the steps may not be applicable to every organization.</li>
<li>Many investigators have constraints on how much time they can spend on an investigation, and some investigators are currently the only ones doing investigations, so it has to be efficient.</li>
<li>Incidents have differing levels of value for investigating incidents. Fundamentally, we believe you can learn from every incident but, realistically, we know that organizations are balancing tradeoffs between time spent in development or operations versus time spent in learning.</li>
</ol>
<p>This means that an investigation process needs to be flexible and adaptable to the goals, interests and needs of the team or organization. We built Howie to be customizable for different sizes of organizations, skills of investigators, or levels of severity of incidents.</p>
<p><img alt="Toolbox" src="assets/images/toolbox.png" /></p>
<p>In this way, an investigator should think about this process as a set of tools in a toolbox. As you progress as an investigator and your company progresses with its incident investigation program, we encourage you to seek more tools to add! This will help you increase the number of insights gained in your investigations.</p>
<p>At the start of each investigation, you’ll consider how much time you have, the availability of participants, and how much value the findings from the investigation will have for the company. High severity, very public events, or substantial near misses would use more of the tools provided here. We encourage you to take the parts that make sense to you at this specific time.</p>
<p>What we hope you take away from this is how to structure your post-incident processes and train others on them. We believe our guide will set you up for success in helping your workplace become a learning organization, one investigation at a time.</p>
<p>—Your friends at Jeli</p>
<h2 id="a-brief-word-about-language-in-incident-investigations">A brief word about language in incident investigations<a class="headerlink" href="#a-brief-word-about-language-in-incident-investigations" title="Permanent link">#</a></h2>
<p>We introduce a number of terms in this guide that may be different from what is used in your organization. Generally speaking, we believe that getting the process right is more important than arguing over language. At the same time, many of the legacy terms used in the industry (such as “post-mortem,” “root-cause analysis,” etc.) come with a history of negativity. Using positive, specific language (such as “learning review” or “blame-aware”) is an important piece of building a culture of learning where folks are excited to be part of this transformation. We believe that language will always be evolving; in this guide we use a number of terms interchangeably (“incident investigation” and “incident analysis”) to reflect the way we use different terminology in our day-to-day work. We expect future iterations of this guide to present new terms and retire some of the ones we see here.</p>
<h2 id="glossary">Glossary<a class="headerlink" href="#glossary" title="Permanent link">#</a></h2>
<p><strong>Blame-aware</strong> – An evolution from “blameless,” we recognize that everyone works with constraints and sometimes those don’t appear until after an incident; we acknowledge our tendency to blame and name names and move past it in order to be productive.</p>
<p><strong>Investigator</strong> – The person who will own the responsibility for the investigation all the way through the process (from initial analysis through distribution of findings); we outline a number of recommendations for what makes someone a good candidate for this role.</p>
<p><strong>Knowledge elicitation</strong> – The process of gathering an expert’s tacit knowledge (expertise and experience) underlying their performance. This is a sub-process of knowledge acquisition.</p>
<p><strong>Calibration document</strong> – A document of initial findings put together by the investigator that will be shared prior to a review meeting to ensure proper alignment with interviewees and participants. This document should prevent any surprises from coming up during the review meeting. A calibration document can also be a draft to the final incident report.</p>
<p><strong>Incident report/ how we got here report</strong> – A document to be sent out following the review meeting to all participants. This should be a learning document different from a post-mortem report in that its goal is learning. It is focused on the story of what happened and how events came to be.</p>
<p><strong>Review meeting</strong> – A facilitated meeting where the investigator guides participants through the incident in order to uncover and share insights.</p>
<p><strong>Action items meeting</strong> – A facilitated meeting or section of a meeting to focus on follow-up items uncovered during the investigation and where you can generate and assign any next steps.</p>
</article>
</main>
<nav id="pagination">
<div class="content-wrapper">
<div class="previous">
</div>
<div class="next">
<a href="assign/" title="Assign">
<span class="label">Next</span>
<div class="page">
<div class="title">
Assign
</div>
<div class="button" role="button" aria-label="Next">
<i class="icon icon-next"></i>
</div>
</div>
</a>
</div>
</div>
</nav>
<footer>
<div class="content-wrapper">
<ul id="ops-guides">
<h2>Ops Guides</h2>
<p>Explore our other guides!</p>
<li id="og-incident-response" class="ops-guide">
<a href="https://response.pagerduty.com">Incident Response</a>
<p>A detailed outline of response processes for technical incidents practices by PagerDuty and our leading customers.</p>
</li>
<li id="og-security-training" class="ops-guide">
<a href="https://sudo.pagerduty.com">Security Training</a>
<p>Open source security training used at PagerDuty - adaptable for your own technical and non-technical teams.</p>
</li>
<li id="og-postmortems" class="ops-guide">
<a href="https://postmortems.pagerduty.com/">Postmortems</a>
<p>Comprehensive guide on how to conduct effective postmortems. Learn how to build a culture of blamelessness.</p>
</li>
<li id="og-operational-reviews" class="ops-guide">
<a href="https://reviews.pagerduty.com/">Operational Reviews</a>
<p>Gauge incident impact using data-driven regularly scheduled reviews to better manage the hidden cost of real-time ops.</p>
</li>
<li id="og-business-response" class="ops-guide">
<a href="https://business-response.pagerduty.com/">Business Response</a>
<p>Learn how to align the business needs with technical needs when severe technical incidents occur.</p>
</li>
<li id="og-stakeholder-comms" class="ops-guide">
<a href="https://stakeholders.pagerduty.com/">Stakeholder Comms</a>
<p>Build an effective communiction strategy for your internal stakeholders during major incidents.</p>
</li>
<li id="og-retrospectives" class="ops-guide">
<a href="https://retrospectives.pagerduty.com/">Retrospectives</a>
<p>Iteratively learn from working processes and behaviors while cultivating a culture of continuous improvement.</p>
</li>
<li id="og-ownership" class="ops-guide">
<a href="https://ownership.pagerduty.com">Ownership</a>
<p>Ensure the reliability of systems & services through a deeper understanding of how code functions in production.</p>
</li>
<li id="og-devsecops" class="ops-guide">
<a href="https://devsecops.pagerduty.com/">DevSecOps</a>
<p>New to DevSecOps, or wondering what it is and how to implement it? This guide will help you get started.</p>
</li>
<li id="og-goingoncall" class="ops-guide">
<a href="https://goingoncall.pagerduty.com/">Going On-Call</a>
<p>The complete resource to going on call for teams and managers.</p>
</li>
<li id="og-autoremediation" class="ops-guide">
<a href="https://autoremediation.pagerduty.com/">Auto Remediation</a>
<p>This guide will help you to leverage automation in your Incident Response process.</p>
</li>
<li id="og-customerservice" class="ops-guide">
<a href="https://customerserviceops.pagerduty.com/">Customer Service</a>
<p>Excellent Customer Service means excellent customer experience, even during incidents.</p>
</li>
</ul>
<p id="outro">
Made with <span class="material-icons love" aria-label="Love">favorite</span> at PagerDuty.
</p>
<p id="github-repo-link">
<a target="_blank"
title="View the source for this documentation on GitHub."
href="https://github.com/pagerduty/postincident-howie-docs">View Source on GitHub</a>
</p>
<!-- It's our theme, so we can just hardcode these :p -->
<ul id="social-links">
<li class="twitter">
<a title="View the @PagerDuty Twitter feed."
href="https://twitter.com/PagerDuty">Twitter</a>
</li>
<li class="github">
<a title="View PagerDuty on GitHub"
href="https://github.com/PagerDuty">GitHub</a>
</li>
<li class="pagerduty">
<a title="View the PagerDuty website."
href="https://www.pagerduty.com/">PagerDuty</a>
</li>
</ul>
</div>
</footer>
<script src="search/main.js"></script>
<script>
/* Google Analytics */
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
ga('create', 'UA-8759953-1', 'auto');
ga('send', 'pageview');
/* Track outbound links */
var buttons = document.querySelectorAll('a');
Array.prototype.map.call(buttons, function(item) {
if (item.host != document.location.host) {
item.addEventListener('click', function() {
var action = item.getAttribute('data-action') || 'follow';
ga('send', 'event', 'outbound', action, item.href);
});
}
});
/* Register handler to log search on blur */
var query = document.querySelector('#mkdocs-search-query');
query.addEventListener('blur', function() {
if (this.value) {
var path = document.location.pathname;
ga('send', 'pageview', path + '?q=' + this.value);
}
});
</script>
</body>
</html>
<!-- Built at: 2026-04-09 17:45:36.063717+00:00 UTC -->