155 lines
16 KiB
HTML
155 lines
16 KiB
HTML
<!doctype html><html lang=en-us prefix="og: http://ogp.me/ns# fb: http://ogp.me/ns/fb#"><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge,chrome=1"><meta name=HandheldFriendly content="True"><meta name=MobileOptimized content="320"><meta name=viewport content="width=device-width,initial-scale=1"><meta name=description content="The urgency often obscures the fact that incidents cause more incidents. Incidents are not isolated events but links in a chain, each capable of setting off a cascade of further issues. The key lies in resisting these impulses, favoring a methodical exploration of safe, effective and reversible solutions."><meta name=keywords content="incidents,devops,"><meta property="og:type" content="article"><meta property="og:description" content="The urgency often obscures the fact that incidents cause more incidents. Incidents are not isolated events but links in a chain, each capable of setting off a cascade of further issues. The key lies in resisting these impulses, favoring a methodical exploration of safe, effective and reversible solutions."><meta property="og:title" content="Incidents and the requirement of slowing down"><meta property="og:site_name" content="Shayon Mukherjee"><meta property="og:image:type" content="image/png"><meta property="og:url" content="https://www.shayon.dev/post/2024/89/incidents-and-the-requirement-of-slowing-down/"><meta property="og:locale" content="en-us"><meta property="article:published_time" content="2024-03-29
|
|
"><meta property="article:modified_time" content="2024-03-29 "><meta property="article:tag" content="incidents"><meta property="article:tag" content="devops"><meta name=twitter:card content="summary_large_image"><meta name=twitter:site content="@shayonj"><meta name=twitter:creator content="@shayonj"><meta name=twitter:title content="Incidents and the requirement of slowing down"><meta name=twitter:description content="The urgency often obscures the fact that incidents cause more incidents. Incidents are not isolated events but links in a chain, each capable of setting off a cascade of further issues. The key lies in resisting these impulses, favoring a methodical exploration of safe, effective and reversible solutions."><meta property="og:image" content="https://www.shayon.dev/incidents.png"><meta property="og:image:width" content="1200"><meta property="og:image:height" content="630"><meta name=twitter:image content="https://www.shayon.dev/incidents.png"><base href=https://www.shayon.dev/><title>Incidents and the requirement of slowing down</title>
|
|
<link rel=canonical href=https://www.shayon.dev/post/2024/89/incidents-and-the-requirement-of-slowing-down/><link href rel=alternate type=application/rss+xml title="Incidents and the requirement of slowing down"><link rel=stylesheet href=https://unpkg.com/tachyons@4.10.0/css/tachyons.min.css><link rel=stylesheet href=https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css><link id=hljs-dark rel=stylesheet href=https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/atom-one-dark.min.css><link id=hljs-light rel=stylesheet href=https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/atom-one-light.min.css disabled><script>(function(){try{var e,t=localStorage.getItem("theme")||"dark-plastic";document.documentElement.setAttribute("data-theme",t),e=t==="light",document.getElementById("hljs-dark").disabled=e,document.getElementById("hljs-light").disabled=!e}catch{}})()</script><script src=https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js></script><script src=https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/go.min.js></script><script src=https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/ruby.min.js></script><script src=https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/rust.min.js></script><script>hljs.highlightAll()</script><link rel=stylesheet href="https://www.shayon.dev/css/style.css?v=14"><script type=module>
|
|
import mermaid from 'https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs';
|
|
|
|
function getThemeVars() {
|
|
var s = getComputedStyle(document.documentElement);
|
|
var v = function(n) { return s.getPropertyValue(n).trim(); };
|
|
var dark = (localStorage.getItem('theme') || 'dark-plastic') !== 'light';
|
|
return {
|
|
darkMode: dark,
|
|
background: v('--bg-secondary'),
|
|
primaryColor: v('--bg'),
|
|
primaryTextColor: v('--text'),
|
|
primaryBorderColor: v('--text-secondary'),
|
|
lineColor: v('--text-secondary'),
|
|
secondaryColor: v('--bg'),
|
|
secondaryTextColor: v('--text'),
|
|
secondaryBorderColor: v('--text-secondary'),
|
|
tertiaryColor: v('--bg'),
|
|
tertiaryTextColor: v('--text-secondary'),
|
|
tertiaryBorderColor: v('--text-secondary'),
|
|
noteBkgColor: v('--bg'),
|
|
noteTextColor: v('--text'),
|
|
noteBorderColor: v('--text-secondary'),
|
|
edgeLabelBackground: v('--bg-secondary'),
|
|
clusterBkg: v('--bg-secondary'),
|
|
clusterBorder: v('--text-secondary'),
|
|
actorBkg: v('--bg'),
|
|
actorTextColor: v('--text'),
|
|
actorBorder: v('--text-secondary'),
|
|
actorLineColor: v('--text-secondary'),
|
|
signalColor: v('--text-secondary'),
|
|
signalTextColor: v('--text'),
|
|
activationBkgColor: v('--bg'),
|
|
activationBorderColor: v('--text-secondary'),
|
|
labelBoxBkgColor: v('--bg'),
|
|
labelBoxBorderColor: v('--text-secondary'),
|
|
labelTextColor: v('--text'),
|
|
loopTextColor: v('--text'),
|
|
};
|
|
}
|
|
|
|
function addExpandButtons() {
|
|
document.querySelectorAll('.mermaid').forEach(function(el) {
|
|
if (el.querySelector('.mermaid-expand-btn')) return;
|
|
var btn = document.createElement('button');
|
|
btn.className = 'mermaid-expand-btn';
|
|
btn.setAttribute('aria-label', 'Expand diagram');
|
|
btn.innerHTML = '<svg width="14" height="14" viewBox="0 0 14 14" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"><polyline points="9 1 13 1 13 5"/><polyline points="5 13 1 13 1 9"/><line x1="13" y1="1" x2="8" y2="6"/><line x1="1" y1="13" x2="6" y2="8"/></svg>';
|
|
btn.addEventListener('click', function(e) {
|
|
e.stopPropagation();
|
|
openOverlay(el);
|
|
});
|
|
el.appendChild(btn);
|
|
});
|
|
}
|
|
|
|
function openOverlay(mermaidEl) {
|
|
var overlay = document.createElement('div');
|
|
overlay.className = 'mermaid-overlay';
|
|
|
|
var close = document.createElement('button');
|
|
close.className = 'mermaid-overlay-close';
|
|
close.setAttribute('aria-label', 'Close');
|
|
close.innerHTML = '<svg width="18" height="18" viewBox="0 0 18 18" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><line x1="4" y1="4" x2="14" y2="14"/><line x1="14" y1="4" x2="4" y2="14"/></svg>';
|
|
|
|
var content = document.createElement('div');
|
|
content.className = 'mermaid-overlay-content';
|
|
var svg = mermaidEl.querySelector('svg');
|
|
if (svg) {
|
|
var clone = svg.cloneNode(true);
|
|
clone.removeAttribute('style');
|
|
var vb = clone.getAttribute('viewBox');
|
|
if (vb) {
|
|
var parts = vb.split(/[\s,]+/);
|
|
clone.setAttribute('width', parseFloat(parts[2]));
|
|
clone.setAttribute('height', parseFloat(parts[3]));
|
|
}
|
|
content.appendChild(clone);
|
|
}
|
|
|
|
overlay.appendChild(close);
|
|
overlay.appendChild(content);
|
|
document.body.appendChild(overlay);
|
|
document.body.style.overflow = 'hidden';
|
|
|
|
requestAnimationFrame(function() { overlay.classList.add('active'); });
|
|
|
|
function closeOverlay() {
|
|
overlay.classList.remove('active');
|
|
document.body.style.overflow = '';
|
|
setTimeout(function() { overlay.remove(); }, 200);
|
|
}
|
|
|
|
close.addEventListener('click', closeOverlay);
|
|
overlay.addEventListener('click', function(e) {
|
|
if (e.target === overlay) closeOverlay();
|
|
});
|
|
var escHandler = function(e) {
|
|
if (e.key === 'Escape') {
|
|
closeOverlay();
|
|
document.removeEventListener('keydown', escHandler);
|
|
}
|
|
};
|
|
document.addEventListener('keydown', escHandler);
|
|
}
|
|
|
|
function renderMermaid() {
|
|
document.querySelectorAll('.mermaid[data-processed]').forEach(function(el) {
|
|
el.removeAttribute('data-processed');
|
|
el.innerHTML = el.getAttribute('data-source');
|
|
});
|
|
|
|
mermaid.initialize({
|
|
startOnLoad: false,
|
|
theme: 'base',
|
|
themeVariables: getThemeVars(),
|
|
flowchart: { curve: 'basis', padding: 24, nodeSpacing: 60, rankSpacing: 70 },
|
|
sequence: { mirrorActors: false, noteMargin: 16, messageMargin: 50, actorMargin: 80, width: 220 },
|
|
fontSize: 16,
|
|
fontFamily: '-apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif'
|
|
});
|
|
mermaid.run().then(addExpandButtons);
|
|
}
|
|
|
|
document.addEventListener('DOMContentLoaded', function() {
|
|
document.querySelectorAll('pre code.language-mermaid').forEach(function(el) {
|
|
var pre = el.parentElement;
|
|
var div = document.createElement('div');
|
|
div.className = 'mermaid';
|
|
div.textContent = el.textContent;
|
|
div.setAttribute('data-source', el.textContent);
|
|
pre.parentNode.replaceChild(div, pre);
|
|
});
|
|
renderMermaid();
|
|
|
|
var observer = new MutationObserver(function(mutations) {
|
|
mutations.forEach(function(m) {
|
|
if (m.attributeName === 'data-theme') renderMermaid();
|
|
});
|
|
});
|
|
observer.observe(document.documentElement, { attributes: true });
|
|
});
|
|
</script><link rel="shortcut icon" href=/favicon.ico type=image/x-icon><link rel=apple-touch-icon href=/apple-touch-icon.png></head></html><body lang=en-us class="sans-serif w-90 w-80-m w-60-ns center mv2 mv5-ns" itemscope itemtype=http://schema.org/Article><nav class=f5 style=color:var(--text-muted)><a href=https://www.shayon.dev/ class=no-underline style=color:var(--text-secondary);font-weight:500>Shayon Mukherjee</a>
|
|
<span style=color:var(--border)>/ </span><a href=/post class=no-underline style=color:var(--text-muted)>blog</a></nav><section id=main class=mt4><h1 itemprop=name id=title>Incidents and the requirement of slowing down</h1><div class="f6 mb4" style=color:var(--text-muted)><span>March 29, 2024</span>
|
|
<span class=mh1>·</span>
|
|
<span>3 min read</span></div><div class=icon-bar><a href="https://twitter.com/share?text=Incidents%20and%20the%20requirement%20of%20slowing%20down&url=https%3a%2f%2fwww.shayon.dev%2fpost%2f2024%2f89%2fincidents-and-the-requirement-of-slowing-down%2f" class=twitter target=_blank><i class="fa fa-twitter"></i></a>
|
|
<a href="https://www.facebook.com/sharer.php?u=https%3a%2f%2fwww.shayon.dev%2fpost%2f2024%2f89%2fincidents-and-the-requirement-of-slowing-down%2f&p[title]=Incidents%20and%20the%20requirement%20of%20slowing%20down" class=facebook target=_blank><i class="fa fa-facebook"></i></a></div><article itemprop=articleBody id=content class="w-90 lh-copy"><p>Speed is key, except when its not. This adage, while seemingly prudent, overlooks a fundamental truth of complex systems: the paradox that slowing down can often be the fastest route to resolution. This contemplative approach to incident management is not merely a tactical choice but a philosophical stance on navigating the intricacies of system failures. It’s an acknowledgment that incidents have a tendency to breed more incidents.</p><p>Imagine a scenario in which a production cluster experiences a sudden blip, leading to saturated connections and dropped requests for critical workflows. The knee-jerk reaction might be to initiate an immediate upgrade of the cluster in an attempt to rectify the issue. However, this decision comes fraught with risks. For example, if your application isn’t used to sudden upgrades, introducing them during a stressful situation can lead to unforeseen complications. And even if upgrading the cluster is the answer to the solution, it’s crucial to understand the implications of the upgrade. Will it make matters worse? Will it introduce new issues? Will the upgrade process itself fail, leading to further disruptions?</p><p>The urgency often obscures the fact that incidents cause more incidents. Incidents are not isolated events but links in a chain, each capable of setting off a cascade of further issues. The key lies in resisting these impulses, favoring a methodical exploration of safe, effective and reversible solutions.</p><p>One thing that I try to practice is to see how much time I can buy before making wide-impacting, irreversible changes. If we are able to buy time and quickly explore + implement less lethal changes, we can avoid making the situation worse. And while this slowdown might seem counterintuitive, it’s a necessary step to ensure that we are not just reacting to the symptoms.</p><p>Central to this approach is the principle of tackling one issue at a time. In the chaos of an ongoing incident, it might seem counterintuitive to narrow one’s focus. Yet, this singularity of purpose allows us to precisely understand the effects of our actions. By methodically addressing one aspect of the problem, confirming its resolution, and then moving on to the next, we construct a clear path through the complexity.</p><p>This philosophy extends beyond the immediate tactical benefits. Slowing down in the midst of an incident fosters a culture of thoughtful response over reactive haste. It encourages a deeper understanding of the systems we manage and a commitment to resolving issues in a manner that is not just quick but lasting. This is the true benefit of deliberation during incidents—it not only reduces the likelihood of subsequent issues but also enhances our capacity to manage complex systems more effectively.</p></article><span class="f6 mv3" style=color:var(--text-muted) title="Lastmod: March 29, 2024. Published at: 2024-03-29."><em>last modified</em> March 29, 2024</span></section><footer><div class="mt5 pt3" style="border-top:1px solid var(--border)"><p class="f6 lh-copy" style=color:var(--text-muted)>© 2026 Shayon Mukherjee</p></div></footer><div id=theme-switcher><button id=theme-btn aria-label="Switch theme">
|
|
<svg viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><circle cx="8" cy="8" r="7" fill="none" stroke="currentcolor" stroke-width="1.5"/><path d="M8 1a7 7 0 000 14z" fill="currentcolor"/></svg><span id=theme-label></span></button><div id=theme-menu style=display:none><button data-set-theme=light>Light</button>
|
|
<button data-set-theme=dark-plastic>Dark</button>
|
|
<button data-set-theme=carbon>Carbon</button>
|
|
<button data-set-theme=peppercorn>Peppercorn</button>
|
|
<button data-set-theme=dusk>Dusk</button></div></div><script>(function(){var s={light:"Light","dark-plastic":"Dark",carbon:"Carbon",peppercorn:"Peppercorn",dusk:"Dusk"},o=document.getElementById("theme-btn"),e=document.getElementById("theme-menu"),i=document.getElementById("theme-label"),t="dark-plastic";try{t=localStorage.getItem("theme")||"dark-plastic"}catch{}function n(n){t=n,document.documentElement.setAttribute("data-theme",n);try{localStorage.setItem("theme",n)}catch{}i.textContent=s[n]||n;var o,a,r=n==="light",c=document.getElementById("hljs-dark"),l=document.getElementById("hljs-light");c&&(c.disabled=r),l&&(l.disabled=!r);for(a=e.querySelectorAll("button"),o=0;o<a.length;o++)a[o].getAttribute("data-set-theme")===n?a[o].className="active":a[o].className=""}n(t),o.addEventListener("click",function(t){t.stopPropagation(),e.style.display=e.style.display==="none"?"block":"none"}),e.addEventListener("click",function(t){var o=t.target,s=o.getAttribute("data-set-theme");s&&(n(s),e.style.display="none")}),document.addEventListener("click",function(){e.style.display="none"})})()</script><script src=//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.14.2/highlight.min.js></script><script>hljs.initHighlightingOnLoad()</script><script async src="https://www.googletagmanager.com/gtag/js?id=G-RBYHPPZF2P"></script><script>window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments)}gtag("js",new Date),gtag("config","G-RBYHPPZF2P")</script><script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" data-cf-beacon='{"version":"2024.11.0","token":"5ff3b0d8a31e4425adef284c498ea68a","r":1,"spa":2}' crossorigin="anonymous"></script>
|
|
</body></html></body> |