Files
nexus/sreweekly/articles/284/01-alerting-on-slos-like-pros.html
2026-09-12 17:23:01 +08:00

243 lines
74 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="x-ua-compatible" content="ie=edge"/><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"/><style data-href="/blog/styles.99789929b311b6c1ffbd.css" id="gatsby-global-css">:root{--surface-color:#121212;--primary-color:#fff;--secondary-color:#999;--highlight-color:#303030;--special-color:#f50;--white-color:#fff;--black-color:#000;--link-color:#699fff;--font-main:"S\F6hne";--font-sans-serif-family:var(--font-main),system-ui,-apple-system,"Segoe UI",Roboto,sans-serif;--font-monospace-family:"Azeret Mono","Monaco","Courier New",monospace;--font-primary-color:#fafafa;--font-secondary-color:var(--secondary-color);--background-surface-color:var(--surface-color);--background-dark-color:#000;--spacing-1x:8px;--spacing-2x:16px;--spacing-3x:24px;--spacing-4x:32px;--spacing-5x:40px;--spacing-6x:48px;--spacing-8x:64px;--breakpoint-sm:768px;--breakpoint-lg:1440px}@font-face{font-family:S\\F6hne;font-style:normal;font-weight:400;font-display:swap;src:url(/blog/static/soehne-buch-a345998020ddc13cd8ca36703b932d68.woff2) format("woff2")}@font-face{font-family:S\\F6hne;font-style:normal;font-weight:600;font-display:swap;src:url(/blog/static/soehne-halbfett-98214b7847e20a37d4cd13292b2d58d7.woff2) format("woff2")}@font-face{font-family:S\\F6hne Breit;font-style:normal;font-weight:900;font-display:swap;font-stretch:expanded;src:url(/blog/static/soehne-breit-extrafett-49752ba2971aac5f0725005fee4dec58.woff2) format("woff2")}@font-face{font-family:Azeret Mono;font-style:normal;font-weight:400;font-display:swap;src:url(/blog/static/azeret-mono-regular-54aca46592769fd2fefe889e10bfc829.woff2) format("woff2")}body,html{margin:0;padding:0;font-family:var(--font-sans-serif-family);font-size:14px;line-height:1.5;color:var(--font-primary-color);background-color:var(--surface-color);-webkit-font-smoothing:antialiased}.sc-redesign a{color:inherit}.sc-redesign img{max-width:100%}.sc-header{background-color:#000;width:100%;position:-webkit-sticky;position:sticky;top:0;z-index:50}.sc-nav{display:flex;align-items:center;justify-content:space-between;box-sizing:border-box;width:100%;margin:0;padding:11px 40px;height:46px}.sc-logo{display:flex;flex-shrink:0;line-height:0}.sc-logo img{display:block;width:170px;height:22px;object-fit:contain}.sc-nav__links{display:flex;list-style:none;gap:24px;margin:0;padding:0}.sc-nav__links li{margin:0}.sc-nav__links a{font-family:var(--font-sans-serif-family);font-size:16px;font-weight:600;line-height:24px;letter-spacing:.96px;text-transform:uppercase;color:#fff;text-decoration:none}.sc-nav__links a:hover{color:var(--font-secondary-color);border-bottom:2px solid var(--font-secondary-color);padding-bottom:2px}.sc-nav__links a.sc-nav__current{border-bottom:2px solid #fff;padding-bottom:2px}.sc-nav__hamburger{display:none;flex-direction:column;justify-content:center;gap:5px;background:none;border:none;cursor:pointer;padding:4px}.sc-nav__hamburger span{display:block;width:22px;height:2px;background:var(--font-primary-color);transition:transform .2s,opacity .2s}.sc-nav__hamburger[aria-expanded=true] span:first-child{transform:translateY(7px) rotate(45deg)}.sc-nav__hamburger[aria-expanded=true] span:nth-child(2){opacity:0}.sc-nav__hamburger[aria-expanded=true] span:nth-child(3){transform:translateY(-7px) rotate(-45deg)}@media screen and (max-width:768px){.sc-nav{padding:11px 12px}.sc-nav__links{display:none;position:fixed;top:0;left:0;right:0;bottom:0;background:var(--surface-color);flex-direction:column;justify-content:center;align-items:flex-start;padding:var(--spacing-8x) var(--spacing-4x);gap:var(--spacing-4x);z-index:100}.sc-nav__links.is-open{display:flex}.sc-nav__links a{font-size:1.5rem;letter-spacing:.05em}.sc-nav__hamburger{display:flex;z-index:101}}.sc-footer{background-color:#000;display:flex;align-items:flex-start;justify-content:space-between;padding:32px 40px 40px;box-sizing:border-box;width:100%;margin:0}.sc-footer__socials{display:flex;flex-direction:column;gap:8px}.sc-footer__follow{font-family:var(--font-sans-serif-family);font-size:16px;font-weight:400;line-height:24px;letter-spacing:.16px;color:#fafafa;margin:0}.sc-footer__icons{display:flex;align-items:center;gap:17px}.sc-footer__icons a{display:flex;align-items:center;opacity:.9;transition:opacity .15s}.sc-footer__icons a:hover{opacity:1}.sc-footer__links{display:flex;align-items:center;gap:32px;list-style:none;margin:8px 0 0;padding:0}.sc-footer__cookieManager,.sc-footer__links a{font-family:var(--font-sans-serif-family);font-size:16px;font-weight:400;line-height:24px;letter-spacing:.16px;color:#fafafa;text-decoration:none;background:none;border:none;padding:0;cursor:pointer}.sc-footer__cookieManager:hover,.sc-footer__links a:hover{color:var(--font-secondary-color)}@media screen and (max-width:768px){.sc-footer{flex-direction:column;gap:24px;padding:32px 12px 40px}.sc-footer__links{flex-wrap:wrap;gap:16px}}.sc-main{width:100%;margin:0;padding:0 40px;box-sizing:border-box}.sc-footer-wrap{background-color:#000;width:100%;margin-top:var(--spacing-8x)}@media screen and (max-width:768px){.sc-main{padding:0 12px}}.postList{list-style:none;margin:0;padding:0}.postList__item{border-bottom:1px solid hsla(0,0%,98%,.15)}.postList__item:last-child{border-bottom:none}.postList__link{display:grid;grid-template-columns:minmax(0,1fr) 314px;align-items:center;gap:var(--spacing-3x);padding:10px 0;text-decoration:none;color:var(--font-primary-color);transition:color .15s}.postList__link:hover{color:var(--font-secondary-color)}.postList__title{font-family:var(--font-sans-serif-family);font-size:24px;font-weight:400;line-height:30px;letter-spacing:.24px;color:#fafafa;min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.postList__meta{display:flex;align-items:center;justify-content:space-between;gap:var(--spacing-2x);min-width:0}.postList__badge{display:inline-flex;align-items:center;justify-content:center;font-family:var(--font-monospace-family);letter-spacing:1.4px;text-transform:uppercase;border:1px solid #fafafa;border-radius:60px;padding:2px var(--spacing-1x)}.postList__badge,.postList__date{font-size:14px;font-weight:400;line-height:20px;color:#fafafa;white-space:nowrap;flex-shrink:0}.postList__date{font-family:var(--font-sans-serif-family);letter-spacing:.14px}.postList__sentinel{height:1px;margin:0;padding:0;border:none;list-style:none}@media screen and (max-width:768px){.postList__link{grid-template-columns:1fr;gap:var(--spacing-1x)}.postList__title{white-space:normal}.postList__meta{justify-content:flex-start}}.sidebar__categories{list-style:none;padding:6px 0 0;margin:0;display:flex;flex-direction:column;gap:2px}.sidebar__category{margin:0;border-left:2px solid transparent}.sc-redesign .sidebar__link,.sc-redesign .sidebar__link:visited{display:block;font-family:var(--font-sans-serif-family);font-size:11px;font-weight:600;line-height:16px;letter-spacing:1.1px;text-transform:uppercase;text-decoration:none;color:hsla(0,0%,98%,.6);padding:2px 6px}.sc-redesign .sidebar__link.m-current,.sc-redesign .sidebar__link:hover{color:#fafafa}.sidebar__category.m-current{border-left-color:#f50}.blogPosts__hero{padding:80px 0 0}.blogPosts__title{font-family:S\\F6hne Breit,var(--font-sans-serif-family);font-size:76px;font-weight:900;line-height:76px;letter-spacing:0;color:#fafafa;text-transform:uppercase;margin:0}.blogPosts__category{margin:var(--spacing-2x) 0 0;font-size:16px;color:var(--font-secondary-color)}.blogPosts__category span{color:var(--font-primary-color);text-transform:uppercase;letter-spacing:.05em}.blogPosts__grid{display:grid;grid-template-columns:214px 1fr;gap:71px;align-items:start;padding:80px 0 var(--spacing-8x)}.blogPosts__main{min-width:0}.blogPosts__clearFilter{padding:24px 0 0}.blogPosts__clearFilterBtn{display:inline-flex;align-items:center;justify-content:center;padding:12px 24px;background:#303030;border-radius:1000px;font-family:var(--font-sans-serif-family);font-size:14px;font-weight:600;line-height:20px;letter-spacing:.28px;color:#fafafa;text-decoration:none;transition:background .15s}.blogPosts__clearFilterBtn:hover{background:#3d3d3d}@media screen and (max-width:768px){.blogPosts__hero{padding:40px 0 0}.blogPosts__title{font-size:40px;line-height:44px}.blogPosts__grid{grid-template-columns:1fr;gap:var(--spacing-4x);padding-top:var(--spacing-4x)}.blogPosts__sidebar{display:none}}.blogPost__content .gatsby-highlight{margin:0 0 24px;background:#1f1f1f;border-radius:16px;overflow:hidden}.blogPost__content .gatsby-highlight pre[class*=language-]{margin:0;padding:10px 16px;background:#1f1f1f;border-radius:16px;overflow-x:auto}.blogPost__content .gatsby-highlight code[class*=language-],.blogPost__content .gatsby-highlight pre[class*=language-]{font-family:var(--font-monospace-family);font-size:14px;font-weight:400;line-height:1.5;letter-spacing:normal;color:#e6e6e6}.blogPost__content .gatsby-highlight code[class*=language-]{background:none;text-shadow:none}.blogPost__content .gatsby-highlight .token{font-family:inherit}.sc-main.blogPost{padding-top:80px}.blogPost__grid{display:grid;grid-template-columns:repeat(24,minmax(0,1fr));align-items:start;box-sizing:border-box;width:100%;max-width:1360px;margin:0 auto;padding:0 0 var(--spacing-8x)}.blogPost__sidebar{grid-column:1/span 4;grid-row:1;display:flex;flex-direction:column;gap:243px;min-width:226px;padding-top:8px}.blogPost__back{font-family:var(--font-sans-serif-family);font-size:20px;font-weight:400;line-height:16px;letter-spacing:.2px;text-transform:uppercase;color:#fafafa;text-decoration:none;padding:0 10px}.blogPost__back:hover{color:var(--font-secondary-color)}.blogPost__adjacent{display:flex;flex-direction:column;gap:23px}.blogPost__adjacentBlock{display:flex;flex-direction:column;gap:10px}.blogPost__adjacentLabel{font-family:var(--font-monospace-family);font-size:20px;font-weight:400;line-height:16px;letter-spacing:.2px;text-transform:uppercase;color:#fafafa;margin:0}.blogPost__adjacentTitle{font-family:var(--font-sans-serif-family);font-size:16px;font-weight:400;line-height:20px;letter-spacing:.16px;color:hsla(0,0%,98%,.6);text-decoration:none}.blogPost__adjacentTitle:hover{color:#fafafa}.blogPost__article{grid-column:8/span 13;grid-row:1;justify-self:start;min-width:0;max-width:728px;width:100%}.blogPost__title{font-size:40px;font-weight:600;line-height:42px;letter-spacing:.8px;margin:0}.blogPost__metadata,.blogPost__title{font-family:var(--font-sans-serif-family);color:#fafafa}.blogPost__metadata{display:flex;flex-wrap:wrap;align-items:center;gap:10px;margin:24px 0 0;padding:0;font-size:16px;font-weight:400;line-height:16px;letter-spacing:.32px;text-transform:uppercase}.blogPost__metadataMobile{display:none}.blogPost__metadataSep{letter-spacing:.16px;text-transform:none}.sc-redesign .blogPost__metadata a{color:#fafafa;text-decoration:underline}.sc-redesign .blogPost__metadata a:hover{color:var(--font-secondary-color)}.blogPost__tags{display:flex;flex-wrap:wrap;gap:12px;list-style:none;margin:16px 0 0;padding:0}.blogPost__tag{display:inline-flex;align-items:center;justify-content:center;font-family:var(--font-monospace-family);font-size:14px;font-weight:400;line-height:20px;letter-spacing:1.4px;text-transform:uppercase;color:#fafafa;border:1px solid #fafafa;border-radius:60px;padding:2px 8px;text-decoration:none;white-space:nowrap}.blogPost__tag:hover{color:var(--font-secondary-color);border-color:var(--font-secondary-color)}.blogPost__content{padding-top:40px;min-width:0;max-width:100%;font-family:var(--font-sans-serif-family);font-size:18px;font-weight:400;line-height:30px;letter-spacing:.18px;color:hsla(0,0%,98%,.6)}.blogPost__content>:first-child{margin-top:0}.blogPost__content p{margin:0 0 24px}.blogPost__emoji{color:#fafafa;font-style:normal}.blogPost__content h2{font-size:26px;font-weight:600;line-height:36px;letter-spacing:.26px;color:#fafafa;margin:40px 0 24px}.blogPost__content h3{font-size:20px;font-weight:600;line-height:28px;letter-spacing:.2px;color:#fafafa;margin:32px 0 16px}.blogPost__content h4,.blogPost__content h5,.blogPost__content h6{color:#fafafa;margin:24px 0 12px}.sc-redesign .blogPost__content a{color:#fafafa;text-decoration:underline}.sc-redesign .blogPost__content a:hover{color:var(--font-secondary-color)}.blogPost__content ol,.blogPost__content ul{margin:0 0 24px;padding-left:24px}.blogPost__content li{margin-bottom:8px}.blogPost__content li>ol,.blogPost__content li>ul{margin-top:8px;margin-bottom:0}.blogPost__content blockquote{margin:24px 0;padding-left:16px;border-left:2px solid hsla(0,0%,98%,.3);color:hsla(0,0%,98%,.8)}.blogPost__content img{max-width:100%;height:auto;border-radius:8px}.blogPost__content code{font-family:var(--font-monospace-family);font-size:16px;font-weight:400;line-height:26px;letter-spacing:.16px;color:#fafafa;background:#272727;border-radius:6px;padding:2px 8px}.blogPost__content pre{margin:0 0 24px;padding:10px 16px;background:#1f1f1f;border-radius:16px;overflow-x:auto}.blogPost__content pre code{display:block;padding:0;background:none;border-radius:0;font-size:14px;font-weight:400;line-height:1.5;letter-spacing:normal;color:#e6e6e6}.blogPost__content .token.cdata,.blogPost__content .token.comment,.blogPost__content .token.doctype,.blogPost__content .token.prolog{color:#6a737d}.blogPost__content .token.punctuation{color:#e6e6e6}.blogPost__content .token.boolean,.blogPost__content .token.constant,.blogPost__content .token.number,.blogPost__content .token.property,.blogPost__content .token.symbol,.blogPost__content .token.tag{color:#79b8ff}.blogPost__content .token.attr-name,.blogPost__content .token.builtin,.blogPost__content .token.char,.blogPost__content .token.selector,.blogPost__content .token.string{color:#b392f0}.blogPost__content .token.entity,.blogPost__content .token.operator,.blogPost__content .token.url,.blogPost__content .token.variable{color:#f97583}.blogPost__content .token.atrule,.blogPost__content .token.attr-value,.blogPost__content .token.class-name,.blogPost__content .token.function{color:#b392f0}.blogPost__content .token.keyword{color:#f97583}.blogPost__tableWrap{overflow-x:auto;max-width:100%;margin:0 0 24px;background:#1f1f1f;border-radius:16px;-webkit-overflow-scrolling:touch}.blogPost__tableWrap table{width:auto;min-width:100%;margin:0;border-spacing:0;border-collapse:collapse;border-radius:16px;overflow:hidden}.blogPost__content thead th{font-family:var(--font-sans-serif-family);font-size:11px;font-weight:600;letter-spacing:.88px;text-transform:uppercase;color:hsla(0,0%,98%,.5);background-color:#1f1f1f;padding:10px 14px;white-space:nowrap;border-right:1px solid hsla(0,0%,98%,.08);border-bottom:1px solid hsla(0,0%,98%,.08)}.blogPost__content thead th:last-child{border-right:none}.blogPost__content td,.blogPost__content th{padding:10px 14px;border-right:1px solid hsla(0,0%,98%,.08);border-bottom:1px solid hsla(0,0%,98%,.08);color:hsla(0,0%,98%,.8);font-size:14px;line-height:1.5;vertical-align:top}.blogPost__content td:last-child,.blogPost__content th:last-child{border-right:none}.blogPost__content tbody tr:last-child td,.blogPost__content tbody tr:last-child th{border-bottom:none}.blogPost__content table code{white-space:normal;word-break:break-word}.blogPost__content thead th code{font-size:11px;line-height:1.4;padding:1px 5px}.blogPost__content .footnotes{margin-top:40px;padding-top:24px;border-top:1px solid hsla(0,0%,98%,.15);font-size:14px}.blogPost__content .footnotes ol li{position:relative}.blogPost__content .footnotes .footnote-backref{position:absolute;top:3px;left:-40px}@media screen and (max-width:768px){.sc-main.blogPost{padding-top:0}.blogPost__grid{grid-template-columns:1fr;grid-template-rows:auto auto;gap:48px;padding-top:0}.blogPost__sidebar{grid-column:1;grid-row:2;min-width:0;gap:48px}.blogPost__article{grid-column:1;grid-row:1;max-width:none}.blogPost__back{display:none}.blogPost__adjacent{display:flex;flex-direction:column;gap:48px;width:100%}.blogPost__adjacentBlock{max-width:300px}.blogPost__adjacentBlock:last-child{align-self:flex-end;text-align:right}.blogPost__adjacentLabel{font-size:16px;line-height:16px}.blogPost__adjacentTitle{font-size:16px;line-height:24px}.blogPost__header{padding-top:32px}.blogPost__title{font-size:32px;line-height:36px;letter-spacing:.64px}.blogPost__metadataDesktop{display:none}.blogPost__metadataMobile{display:flex;flex-direction:column;align-items:flex-start;gap:10px;margin:24px 0 0;font-size:14px;line-height:20px;letter-spacing:.14px;text-transform:none}.blogPost__metadataMobileDetails{display:flex;align-items:center;gap:10px}.blogPost__tag{font-size:12px;line-height:16px;letter-spacing:1.2px}.blogPost__content{font-size:16px;line-height:24px;letter-spacing:.16px}}</style><meta name="generator" content="Gatsby 2.32.13"/><link rel="alternate" type="application/rss+xml" title="SoundCloud Backstage Blog" href="/blog/blog.rss"/><title data-react-helmet="true">Alerting on SLOs like Pros | SoundCloud Backstage Blog</title><link data-react-helmet="true" rel="shortcut icon" type="image/ico" href="data:image/vnd.microsoft.icon;base64,AAABAAIAEBAAAAEAIABoBAAAJgAAACAgAAABACAAqBAAAI4EAAAoAAAAEAAAACAAAAABACAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAACL//wAi//8AIv//ACL//wAi//8AIv//ACL//wAi//8AIv//ACL//wAi//8AI///ACL//wAj//8AI///ACL//wAj//8AI///ACP//wAj//8AI///ACP//wAj//8AI///ACP//wAj//8AI///ACP//wAj//8AI///ACP//wAj//8AJf//ACX//wAl//8AJf//ACX//wAl//8AJf//ACX//wAl//8AJf//ACX//wAl//8AJf//ACX//wAl//8AJf//ACn//wAp//8AKf//ACj//wAp//8AKP//ACj//wAp//8AKP//ACn//wAp//8AKf//ACn//wAp//8AKf//ACn//wAu//8ALf//AC3//wAu//8ALf//AC7//wAu//8ALf//AC7//wAt//8ALf//AC7//wAt//8ALv//AC7//wAt//8ANP//ADX//5mu//8ANf//2uH//wA1///Z4P//ADX//9be///+/v///f3///z8///q7v//XX7//wA0//8ANf//AD3//wA9////////AD3///////8APf///////wA9//////////////////////////////Dz//8APv//AD7//wBH//8AR////////wBI////////AEj///////8AR//////////////////////////////F1f//AEf//wBH//8AUf//AFL//1mO//8AUv///////wBS////////AFH//////////////////+7z//9wnf//El7//wBS//8AUv//AFz//wBc//8AXP//AFz//1qV//8AXP///////wBb//////////////////+Js///AFv//wBb//8AW///AFz//wBk//8AZf//AGX//wBk//8AZf//AGT//1ma//8AZP//xdv//+jx//+LuP//BWj//wBk//8AZf//AGX//wBk//8AbP//AGv//wBr//8Aa///AGv//wBr//8Aa///AGz//wBr//8AbP//AGz//wBr//8AbP//AGv//wBr//8AbP//AHD//wBx//8Acf//AHH//wBx//8Acf//AHH//wBw//8Acf//AHD//wBw//8AcP//AHD//wBw//8AcP//AHD//wB0//8AdP//AHT//wB0//8AdP//AHT//wB0//8AdP//AHT//wB0//8AdP//AHP//wB0//8Ac///AHP//wB0//8Adv//AHb//wB2//8Adv//AHb//wB2//8Adv//AHf//wB2//8Ad///AHf//wB2//8Ad///AHb//wB2//8Adv//AHf//wB3//8Ad///AHb//wB3//8Adv//AHb//wB3//8Adv//AHf//wB3//8Ad///AHf//wB3//8Ad///AHf//wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoAAAAIAAAAEAAAAABACAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAACL//wAi//8AIv//ACL//wAi//8AIv//ACL//wAi//8AIv//ACL//wAi//8AI///ACL//wAj//8AI///ACL//wAj//8AIv//ACL//wAi//8AIv//ACL//wAi//8AIv//ACL//wAi//8AIv//ACL//wAi//8AIv//ACL//wAi//8AIv//ACP//wAj//8AIv//ACP//wAi//8AIv//ACL//wAi//8AIv//ACL//wAj//8AIv//ACP//wAj//8AIv//ACP//wAi//8AIv//ACP//wAi//8AI///ACP//wAj//8AI///ACP//wAj//8AI///ACP//wAj//8AI///ACL//wAi//8AI///ACP//wAj//8AI///ACP//wAj//8AI///ACP//wAj//8AI///ACL//wAj//8AIv//ACL//wAi//8AIv//ACL//wAi//8AI///ACL//wAj//8AI///ACL//wAj//8AIv//ACL//wAj//8AIv//ACP//wAj//8AI///ACT//wAj//8AI///ACP//wAj//8AI///ACP//wAj//8AI///ACP//wAj//8AJP//ACP//wAk//8AJP//ACT//wAk//8AJP//ACT//wAk//8AJP//ACT//wAk//8AJP//ACT//wAk//8AJP//ACP//wAk//8AI///ACP//wAj//8AJf//ACT//wAk//8AJf//ACT//wAl//8AJf//ACT//wAl//8AJP//ACT//wAl//8AJP//ACX//wAl//8AJP//ACX//wAk//8AJP//ACT//wAk//8AJP//ACT//wAk//8AJP//ACT//wAk//8AJf//ACT//wAl//8AJf//ACT//wAm//8AJv//ACb//wAm//8AJv//ACb//wAm//8AJv//ACb//wAm//8AJv//ACX//wAm//8AJf//ACX//wAm//8AJf//ACb//wAm//8AJv//ACb//wAm//8AJv//ACb//wAm//8AJv//ACb//wAm//8AJv//ACb//wAm//8AJv//ACf//wAo//8AKP//ACf//wAo//8AJ///ACf//wAn//8AJ///ACf//wAn//8AKP//ACf//wAo//8AKP//ACj//wAo//8AKP//ACj//wAn//8AKP//ACf//wAn//8AKP//ACf//wAo//8AKP//ACf//wAo//8AJ///ACf//wAo//8AKv//ACr//wAq//8AKv//ACr//wAq//8AKv//ACr//wAq//8AKv//ACr//wAq//8AKv//ACr//wAq//8AKf//ACr//wAp//8AKf//ACr//wAp//8AKv//ACr//wAq//8AKv//ACr//wAq//8AKf//ACr//wAp//8AKf//ACr//wAs//8ALP//ACz//wAs//8ALP//ACz//wAs//8AK///ACz//wAr//8AK///ACz//wAr//8ALP//ACz//wAs//8ALP//ACz//wAs//8ALP//ACz//wAs//8ALP//ACz//wAs//8ALP//ACz//wAs//8ALP//ACz//wAs//8ALP//ADD//wAv//8AL///ADD//wAv//8AMP//ADD//wAv//8AMP//AC///wAv//8AL///AC///wAv//8AL///AC///wAv//8AL///AC///wAv//8AL///AC///wAv//8AMP//AC///wAw//8AMP//AC///wAw//8AL///AC///wAv//8AM///ADP//wAz//8AMv//ADP//wAy//8AMv//ADP//wAy//8AM///ADP//wAz//8AM///ADP//wAz//8AM///ADP//wAz//8AM///ADP//wAz//8AM///ADP//wAy//8AM///ADL//wAy//8AM///ADL//wAz//8AM///ADP//wA3//8ANv//ADb//wA2//8ANv//ADb//9nh//8AN///2uH//wA3///Z4f//ADb//9rh//8ANv//2eH//wA3//+1xP///v7///39///8/P//+/v///r7///5+v//+Pn///f4///M1v//T3T//wA2//8ANv//ADb//wA2//8AN///ADv//wA7//8AO///ADv//9vj//8AO////////wA7////////ADv///////8AO////////wA7////////ADv////////////////////////////////////////////////////////+/v//T3f//wA7//8AO///ADv//wA7//8AP///AED//wBA//8AQP///////wBA////////AD////////8AP////////wA/////////AD////////8AQP/////////////////////////////////////////////////////////////N2f//AED//wBA//8AQP//AD///wBF//8ARP//AET//wBF////////AEX///////8ARf///////wBF////////AEX///////8ARf///////wBE//////////////////////////////////////////////////////////////P2//8ARf//AET//wBE//8ARf//AEr//wBK//8ASv//AEn//9vl//8ASf///////wBK////////AEr///////8ASv///////wBK////////AEr/////////////////////////////////////////////////////////////y9r//wBK//8ASv//AEr//wBK//8AT///AE///wBP//8AT///AE///wBP////////AE///9rl//8AT////////wBQ////////AFD///////8AT/////////////////////////////////////////////////////////////9Lgv//AE///wBP//8AT///AFD//wBU//8AVf//AFX//wBU//8AVf//AFT//9nl//8AVP//AFT//wBU////////AFX///////8AVf///////wBU//////////////////////////////////////////////b5///L3P//S4f//wBU//8AVf//AFT//wBU//8AVP//AFn//wBZ//8AWf//AFn//wBZ//8AWf//AFn//wBa//8AWf//AFr///////8AWf//2uf//wBZ////////AFn///////////////////////////////////////+70v//AFn//wBZ//8AWf//AFn//wBZ//8AWf//AFn//wBa//8AXv//AF7//wBe//8AXv//AF7//wBe//8AXv//AF7//wBe//8AXv//2ef//wBe//8AXv//AF7///////8AXv///////////////////////////////////////2Kb//8AXv//AF7//wBe//8AXv//AF7//wBe//8AXv//AF7//wBj//8AYv//AGL//wBj//8AYv//AGP//wBj//8AYv//AGP//wBi//8AYv//AGP//wBi//8AY///2ef//wBi///////////////////////////////////A2P//BWX//wBi//8AYv//AGL//wBj//8AYv//AGP//wBj//8AYv//AGb//wBn//8AZ///AGb//wBn//8AZv//AGb//wBm//8AZv//AGb//wBm//8AZv//AGb//wBm//8AZv//AGb///r8////////////////////////yN7//xRy//8AZv//AGb//wBm//8AZv//AGb//wBm//8AZv//AGb//wBm//8Aaf//AGr//wBq//8Aaf//AGr//wBp//8Aaf//AGn//wBp//8Aaf//AGn//wBq//8Aaf//AGr//wBq//8Aav//iLn///j6///o8f//utb//2Kj//8Baf//AGn//wBq//8Aaf//AGr//wBq//8Aaf//AGr//wBp//8Aaf//AGr//wBt//8Abf//AG3//wBt//8Abf//AG3//wBt//8Abf//AG3//wBt//8Abf//AG3//wBt//8Abf//AG3//wBt//8Abf//AG3//wBt//8Abf//AG3//wBt//8Abf//AG3//wBt//8Abf//AG3//wBt//8Abf//AG3//wBt//8Abf//AG///wBv//8Ab///AG///wBv//8Ab///AG///wBv//8Ab///AG///wBv//8Ab///AG///wBv//8Ab///AHD//wBv//8AcP//AHD//wBv//8AcP//AG///wBv//8Ab///AG///wBv//8Ab///AG///wBv//8Ab///AG///wBw//8Acv//AHL//wBy//8Acv//AHL//wBy//8Acv//AHH//wBy//8Acf//AHH//wBx//8Acf//AHH//wBx//8Acv//AHH//wBy//8Acv//AHH//wBy//8Acf//AHH//wBy//8Acf//AHL//wBy//8Acf//AHL//wBx//8Acf//AHL//wBz//8AdP//AHT//wBz//8AdP//AHP//wBz//8Ac///AHP//wBz//8Ac///AHT//wBz//8AdP//AHT//wBz//8AdP//AHP//wBz//8AdP//AHP//wB0//8AdP//AHP//wB0//8Ac///AHP//wB0//8Ac///AHT//wB0//8Ac///AHT//wB0//8AdP//AHT//wB0//8AdP//AHT//wB1//8AdP//AHX//wB1//8AdP//AHX//wB0//8AdP//AHX//wB0//8Adf//AHX//wB1//8Adf//AHX//wB1//8AdP//AHX//wB0//8AdP//AHT//wB0//8AdP//AHT//wB1//8Adf//AHb//wB2//8Adv//AHb//wB2//8Adv//AHX//wB2//8Adf//AHX//wB2//8Adf//AHb//wB2//8Adf//AHb//wB1//8Adf//AHX//wB1//8Adf//AHX//wB2//8Adf//AHb//wB2//8Adf//AHb//wB1//8Adf//AHb//wB2//8Ad///AHf//wB2//8Ad///AHb//wB2//8Adv//AHb//wB2//8Adv//AHb//wB2//8Adv//AHb//wB2//8Adv//AHb//wB2//8Adv//AHb//wB2//8Adv//AHf//wB2//8Ad///AHf//wB3//8Ad///AHf//wB3//8Adv//AHf//wB2//8Adv//AHf//wB2//8Ad///AHf//wB3//8Ad///AHf//wB3//8Ad///AHf//wB3//8Ad///AHb//wB3//8Adv//AHb//wB2//8Adv//AHb//wB2//8Ad///AHb//wB3//8Ad///AHb//wB3//8Adv//AHb//wB2//8Ad///AHf//wB3//8Adv//AHf//wB2//8Adv//AHf//wB2//8Ad///AHf//wB3//8Ad///AHf//wB3//8Ad///AHf//wB3//8Ad///AHf//wB3//8Ad///AHf//wB3//8Ad///AHf//wB3//8Ad///AHf//wB3//8Ad///AHf//wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"/><meta data-react-helmet="true" name="description" content="If there is anything like a silver bullet for creating meaningful and actionable alerts with a high signal-to-noise ratio, it is alerting based on service-level objectives (SLOs). Fulfilling a well-defined SLO is the very definition of meeting your users’ expectations. Conversely, a certain level of service errors is OK as long as you stay within the SLO — in other words, if the SLO grants you an error budget. Burning through this error budget too quickly is the ultimate signal that some rectifying action is needed. The faster the budget is burned, the more urgent it is that engineers get involved. This post describes how we implemented this concept at SoundCloud, enabling us to fulfill our SLOs without flooding our engineers on call with an unsustainable amount of pages."/><meta data-react-helmet="true" property="og:title" content="Alerting on SLOs like Pros"/><meta data-react-helmet="true" property="og:description" content="If there is anything like a silver bullet for creating meaningful and actionable alerts with a high signal-to-noise ratio, it is alerting based on service-level objectives (SLOs). Fulfilling a well-defined SLO is the very definition of meeting your users’ expectations. Conversely, a certain level of service errors is OK as long as you stay within the SLO — in other words, if the SLO grants you an error budget. Burning through this error budget too quickly is the ultimate signal that some rectifying action is needed. The faster the budget is burned, the more urgent it is that engineers get involved. This post describes how we implemented this concept at SoundCloud, enabling us to fulfill our SLOs without flooding our engineers on call with an unsustainable amount of pages."/><meta data-react-helmet="true" property="og:type" content="article"/><meta data-react-helmet="true" property="og:site_name" content="SoundCloud Backstage Blog"/><meta data-react-helmet="true" property="og:url" content="https://developers.soundcloud.com/blog/alerting-on-slos"/><meta data-react-helmet="true" name="twitter:card" content="summary_large_image"/><meta data-react-helmet="true" name="twitter:site" content="@SoundCloudDev"/><meta data-react-helmet="true" name="twitter:creator" content="SoundCloud"/><meta data-react-helmet="true" name="twitter:title" content="Alerting on SLOs like Pros"/><meta data-react-helmet="true" name="twitter:description" content="If there is anything like a silver bullet for creating meaningful and actionable alerts with a high signal-to-noise ratio, it is alerting based on service-level objectives (SLOs). Fulfilling a well-defined SLO is the very definition of meeting your users’ expectations. Conversely, a certain level of service errors is OK as long as you stay within the SLO — in other words, if the SLO grants you an error budget. Burning through this error budget too quickly is the ultimate signal that some rectifying action is needed. The faster the budget is burned, the more urgent it is that engineers get involved. This post describes how we implemented this concept at SoundCloud, enabling us to fulfill our SLOs without flooding our engineers on call with an unsustainable amount of pages."/><meta data-react-helmet="true" name="theme-color" content="#333"/><meta data-react-helmet="true" property="og:image" content="https://developers.soundcloud.com/blog/images/og-default.png"/><meta data-react-helmet="true" name="twitter:image" content="https://developers.soundcloud.com/blog/images/og-default.png"/><meta data-react-helmet="true" property="og:image:width" content="1200"/><meta data-react-helmet="true" property="og:image:height" content="630"/><link rel="preload" as="font" type="font/woff2" href="/blog/fonts/soehne-buch.woff2" crossorigin="anonymous"/><link rel="preload" as="font" type="font/woff2" href="/blog/fonts/soehne-halbfett.woff2" crossorigin="anonymous"/><style>
@font-face {
font-family: 'Söhne';
font-weight: 400;
font-display: block;
src: url('/blog/fonts/soehne-buch.woff2') format('woff2');
}
@font-face {
font-family: 'Söhne';
font-weight: 600;
font-display: block;
src: url('/blog/fonts/soehne-halbfett.woff2') format('woff2');
}
</style><script>
if (!['localhost', '127.0.0.1'].includes(window.location.hostname)) {
var s = document.createElement('script');
s.defer = true;
s.src = 'https://consent.sndcdn.com/v1/consent.js';
document.head.appendChild(s);
}
</script><script type="text/javascript">
(function() {
if (['localhost', '127.0.0.1'].includes(window.location.hostname)) return;
window.dataLayer = window.dataLayer || [];
function gtag() {
dataLayer.push(arguments);
}
gtag("js", new Date());
gtag("config", "UA-79535818-1");
gtag("consent", "default", {
ad_storage: "denied",
analytics_storage: "denied",
functionality_storage: "denied",
personalization_storage: "denied",
security_storage: "granted",
ad_user_data: "denied",
ad_personalization: "denied",
region: [
"BE",
"EL",
"LT",
"PT",
"BG",
"ES",
"LU",
"RO",
"CZ",
"FR",
"HU",
"SI",
"DK",
"HR",
"MT",
"SK",
"DE",
"IT",
"NL",
"FI",
"EE",
"CY",
"AT",
"SE",
"IE",
"LV",
"PL",
"US-CA",
],
});
gtag("consent", "default", {
ad_storage: "granted",
analytics_storage: "granted",
functionality_storage: "granted",
personalization_storage: "granted",
security_storage: "granted",
ad_user_data: "granted",
ad_personalization: "granted",
});
addEventListener("DOMContentLoaded", (event) => {
Promise.all([
window.SCConsent.waitForCategory('C0002'),
window.SCConsent.waitForCategory('C0004'),
]).then(() => {
(function (w, d, s, l, i) {
w[l] = w[l] || [];
w[l].push({ "gtm.start": new Date().getTime(), event: "gtm.js" });
var f = d.getElementsByTagName(s)[0],
j = d.createElement(s),
dl = l != "dataLayer" ? "&l=" + l : "";
j.async = true;
j.src = "https://www.googletagmanager.com/gtag/js?id=" + i;
f.parentNode.insertBefore(j, f);
})(window, document, "script", "dataLayer", "UA-79535818-1");
});
})
})();
</script><link as="script" rel="preload" href="/blog/webpack-runtime-500dd8fe0fe45b062595.js"/><link as="script" rel="preload" href="/blog/framework-ac576c16d7c8bd3ef403.js"/><link as="script" rel="preload" href="/blog/app-3fa20cd6fd93205429a7.js"/><link as="script" rel="preload" href="/blog/styles-0ec71dd62c66cb95665c.js"/><link as="script" rel="preload" href="/blog/commons-05739180ee3b1321cf79.js"/><link as="script" rel="preload" href="/blog/component---src-templates-blog-post-js-95a2dd758be9473349e5.js"/><link as="fetch" rel="preload" href="/blog/page-data/alerting-on-slos/page-data.json" crossorigin="anonymous"/><link as="fetch" rel="preload" href="/blog/page-data/sq/d/659219053.json" crossorigin="anonymous"/><link as="fetch" rel="preload" href="/blog/page-data/app-data.json" crossorigin="anonymous"/></head><body><div id="___gatsby"><div style="outline:none" tabindex="-1" id="gatsby-focus-wrapper"><div class="sc-redesign"><header class="sc-header"><nav class="sc-nav"><a href="https://soundcloud.com/" class="sc-logo" title="Go to SoundCloud.com" aria-label="SoundCloud"><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAVQAAAAsCAYAAAA6lECtAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAD3ZJREFUeNrsXQuQFcUV7eXnsiy7uKyCX35GoyFCTESjRD7RYDSJSdRoiSTGT/lDo2gMISqKKEalVEg0QUNKRVFJYqzERCUqfiIoiPEfIyDgIuACwu7qLuwCubfenaK33709Pe/NvDcP+lSdeju9PT0z3Xdu3759u6ds+/btysPDw8Mjf5TtxAq1FtgX2AOfE7gFuBq4ArjZN72Hh4dXqDKqgccDTwKOBPYS8m0F/hf4MvApYpMXBQ8PD69QlToSOA74fWDnHM7fBHwYOBX4gRcJDw+PXVWhPgscEVNZaLn+EfhL4DovGh4eHruaQk3i5lGZngt83IuHh4eHV6jxlDsZODHBa3h4eOxk6FBC9/o14KP0m3hHA7wG+Dv628PDw2OnslDnAYcBnwaOSthC1XGzyvhVPTw8PFKtUAcAlzkqRgxt6gZsBFYVUKEifgSc48XFw8MjrUP+y4FLgLc75K0hZYroDuxd4Hv9PXBvLy4eHh5pVahn0+9II/0o4GzgQC1tHyNPoRXq7jT09/Dw8BDRqYjXrqHfaiP9VlKqaJF+j9L2MvLsUYT7HU1K9d2UtWFH4GDgIK1eMPTrLeBrKhNfm0+HeyjwMK3sT4CvA99Q7i4XLKevkYb32BByHrp2ao20ZULe/sbxGuDnIeVXAvc00pYDtzleQ0crcKPKuKTiRFeVWbzSR2VW/+ES6pXAV4B1DufjyKpcO/6c6iYug4yTj8XANyPIR/+YZENqO1u7tVG7NcRSI+hDTZgnAOcAhxjpm7ZnsMFIX0Lp72tpZ25vjzMpvdCYUYD6cmU5cAJwleV+1wOnAntFLLsT8FLgh5aylwLHUt6w8qqZ8y90OO9C5jwpr4k5DuWfwZxXHeEaHFCunwFeCxyQR/v2Bk4HNlqu9RJweEg5Lxrn/D0G2QvkY3mIfFzsKB9xyUZ1Hu3WAHwWeD3wwFzrphAv/jt0w68ID9lqpNdryiBIGytUeKHRSIqs2Mp0P61eXbCRlIdL2XtTW7liAb38aVOoiNOLoFB1tAFnAmsitu+JJP8u2Aa8EVhWIIWK8vFqhDqY7yAfaVCoOrYC7wfWRq2fQvhQ99CGHvpQS3c7VGjH1ZpLoLPgFuhepOE13vexRR7i4xBwLvCQCOdg/T0InBSSD4dQzwOHRCj7CDqnZwpdWtOL4G833TE/BS5UmYgWF/xAZVbp1TjmxzjpCQ5tGwcC+Tg8wjlH0jk1qnSAenEMucy+kLZJqUrN58EpVF1BVqj2G5z0FPJXFLGyRxh+HBSULgW8/hXAg5j0JvIvrrWcG3af9wIPsPgIJZ/kgSoTCZE2oAK4KwX3gT68fzKGAVePs0gRc2i0+MR/BTwm4efY2eQjDPurzG50uyepUE+hSQkXS60TWVSmQu0mKNQegnWbJoV6GnCByuypijtVrQc2A98D3l0AoT7TOMb7OIPqbgBZZDiJ93Nytge4DjjeUu5xKrP1oQncMGYgTWx0owmwB5h8J6fAepcsvtEJlf0blVm5hxwK/CHwFpocMYGWzk0h5f2Wke0NwIvJuKiidwFlcKmR73OaIEoK3xLk4x7gwYZ8zBL0RlrkI0q79aP/JTYptUhziIfl7SH4OA410g+l9ION9KGUfq+RfleRfKhR/EZHJOA77Uj+HR0PW/LvBVwI/IVD2U8xz3G1Jf91TP4nUuZD1SfneifgQx0v5OsCvJPJv9niTzyCyb8a2FfIj/69FcC1NAFWk7AP9Wnm/iZY8k+KIB+F9qGOt0y23cLkb6V5i0R8qEFM6H5G+mgK5zjHGBIrxj9axYRAKJW9KXS15jdMgw81it/o3+TbSho9LP/DLxRgCNqvHYbG3zTSMOzlRss56LN7h7FiqlPYHuiWmVHA62Fo089UZp9d0+VyqnDOj5m0ywSrCbGO2qwftcWGhF0npnxg2NwUyzk4Inq3ROQjAIZQXUWuDXOkfWpSQ/7Ar2nG711ESvY8i0KtEobsXY2yzfxdiuD7zRcdSSFNi7HMrYyPdBQNsQ63+LbCMIzx2+GwyBZHuI3xi6HgDUlJ/Zv19F1BaSWJq1V2XKS0f++3jWM0TsKWOy9R4bG2cWAY886VunzYcAPTbk7uiqiKCa2hYNKo3PCF7ku/e1t8nd0Ei7NcUKjdhfzdSsixfQnw0hjLm8uk4ejgVZUJ9L6PjqMsfhjIpD2X472k5YW5VmUH2d+hCruEeClZ+joGCRZ0PyPtCSUvMCg0BuUoH8+kWD5swM5ssZH25XwVaj+qtGrD9NcRDNExdKM3M2yXLFHXSakq4d46qdICrv76YkxlTbZYJfuQFTaLLLTXaOTQNaTMPsYxTnR96HAvS5iXvm9K6hyV6TVGGs7W3lPg+3jbON5PeNcU43JJC/bNUT7+x6T1LZF39g3jeK98FSrOLA5X7WeVezK+FUXWUBdteF4rKM7guFJIN/0rlSGKtVSAdTIlprLwu1c4c70pJB92crgkEGeOMQJhqCVvJaOMXJastjFWYEVK6ryMhqXzjfQTVCY2tFAwfZudVfuloJzcI9anSH7NOYsGR+u5lZHTihJ5Zz81jjsqB/+vTaEOpl99Q2dzGBko2N5CenfB4iwXLE7zhnfTXo5SB4ac7B9TWbgn7JdUJkzLZe14HxqeH612LWwl5dlspN/OWF1JKnYT20qsHnfFTdY75NJuNoV6MOM/qRV6LlPR1hhDfLN3qhAsJFefbrcSbKAyxcfx5YpVNJzHNhlFSsK2cQt2YvcJ7pImpl1d2qIT02ly7ojNgtUehvIY6ul9lQl6Ny3Cu1RhJnRMY6NFZaIAzBGBEoySNKCBMYxc5KMzYyS51PluOeaJsz33ZKztUONFqhSshMB5f4j2EtYIClKyXMuFl6icMadtFq0SLNpSw/EJlLmFLNZxZLWiH/VcZqiLGEBuHBMrGWHt73DtAxgZWs7kayH3ANfp2tCTedZccCfwJSMNZ/1PLECbD2FcNmH1r8hdkxbUMfLRz1E+lIN8xCEbbcotosUVRxnHS3M1axW9mHrlHSgouK6C5Vql9VBReqAOO/lQ45ACXONj4B+UHH/Kzdi+xaS5fJ6bCyV51XJfOr7iUP4gxirPBThUOwv4mZF+dsJtMVxlT/i9zORbq7LDvL6j5CWohcYbwrOF4ThH+VgVg2x8HHO7mQbF/HwUamdhaC8NwSoFRVshlNMx5Pwow8JSQlwbRHTUXDI2cOunuYUAz6ts/9DFIR0ays75jJUgKdQ3mZetNsQCOTakjChAC2O8o/zHAZT96Uz6Y0J+M8QIZ5XHhFzjGBqZJD1ie4GRj7ExyscrTEe9R8h7FKds6EAdx31F5C/5KNQ64bhKUHhVjoqwg6BApZ64q5C/skQVahzfwMI6wYDvRSoz229DHyaNmz3GVTf/YiwA28cJMSTJjF9F14MUffAUI7i3CS9lGf3P7MCfzbPuMOLhuQK0M4ZGzWXqZ5HiY3cRs5k0jJuV4h/Rx4c+8alkQY5I8HnqGYU/WNn3hpjIjMgk+ZjHyMZUi2zcorLnZ+Jo133oHgcznfGTcSlUbnWOOSQ3HzyYNIqr59zqqIDTjnUxWLhzSZFWUK+JHGm4UbB+cHXL3UwZrwtl38ak4UqvmeQCKtPcFvgiX8fkt30fDGNjzcmvnwAfUu39cQNIuZxl5EX/6SMxdGi4NDquXfXxBfwqEa3F0VRfH6hsHxxe+zIlzxT/Q2VPKlbT6OECtWPHI2znk8h10FdrE+xsrnS451qy7mwc6CgfuNkLfmr9IE0+sAO4X2UWVrjKx2xGNsaQbPQ3ZOMh1X55O6KZrplru40hmca46m8w+S9R2X5eQcLkhf5raWOAOi1tirFpwAWUfq2wucFVRvo4Sr/VSL+B0mca6fcJGzy8mPLNUSQ8nufmKHNCNsVdA1wGbBbyLAvZQf2vlvJxY48my/8fc7j/myznryFKuCPiJsW2DbXPt1wnzg2mdVzvUD8jmM1vdGygTas5tAAPctgcxQUPFkE+JuchG9MiykYUTIlrg+k6xtlrhk8Ex2a4Qgv9NjI9CcKcjfvM+L+Z3spYK6WIeXmej0PwlZbRRi+y9iRf9xUhPe251EtL/mwpXO0DOjcMaNUuFP7XS2VvjhPgbZW96ikfzKChXSGwje59okNeHLZeZfn/7pbRGW7G8n7Cz5KPfJwTUvYkxpfqIhu4Qc/VCbnnJoW4vZyH/IiPjF/FDP0DhfepoFBNM75JUMyBwtxopG8W0ptLUJm2xTBkRWH+OvDPEc8Ldj96zMElMUzJE0tKmFA4Rrmt7MH7GEXD2Cjlj1TxfvxuOymHTQm3+WLybU6OcA76Di9S7iFArZS/EBs4B/KxMAf52OAgGxjZMD9C2QtJNhpifs63yPUxMeqJNoW61Pg1rVVEEO5gvkzrBZ/heiF/g5C/UXOK69hYggp1toontAPLOIWE7xllXyK6hZQo7kQ1LUL5Q8nft9KSb7nKbPqCeaN8RfNTEtbLjc6au49L6GWsT6A98NrjErBqsNPD7d+OJh/dCzmUg77vw8g/3mqZV3ic2vbuAsrxx/RsScjHOmrvsLJXk4GAZX8Sc7th9MkgleMEqG3S6F6yJqYzCtRUsPVMxSimV/pEUKhB/jVC/jqmQksJ2DHEvTfqE0ScqMLgcYwV7qp2fBZ3OVkHTTmUjS/xndT25meC69WOzwTnuoQS7/EOUvI4ozqQyu+kdnwCe1HE8gcIsmPDTMYN0xjhGqaCW6P4VWG5AN0cJ1P7ogI7gIbVLWTkLFBuk5ynqegrzpoiyMcgko/amOSjTStbko2on0e3tdtnpI/a4mg0m0J9T2XP9i3X/ta/713vqDil9HpBoa4WFOoqVTrA3u885fYN9VyAndaTyjGsIyLwhfgPUSVU/mKVvVVaLlhWgPOWFUF+sH3/lqdFqRJsv9eVHDlS6rIRGVEDmxs1Zahv3baa6alNK2Grls9s5CD/CkGhLrUo9rQDv+30iPLw8NjpkctKkfe0IYmuaJs1JRiYzw1qxwTVas0f9JGhaFdo6bopv5S5FuKdEqhbHDZhfNtUL2YeHl6hShhLSmKCMAxfIVifupO5Re3w/9RpirZVU7Y4A7tWG/YE5TdrJvzcFNYpWu2PqkyA8ywvYh4eXqHagMvcrmQU5wxyB5gfuLqHFOI0phzEIiM9CNlZYKQHoTYPaFbs6cA/aVZwMdBCnQIqd9wmDieHTisxt4SHh0cMKMPo/iIh2ODgSdU+jgyDg/ELi/OMdJypHK4y4QxbfNN5eHh4herh4eHhFaqHh4eHhw3/F2AAclRMcUCUDxEAAAAASUVORK5CYII=" alt="SoundCloud" width="170" height="22"/></a><ul class="sc-nav__links"><li><a href="https://soundcloud.com/you/apps">Your Apps</a></li><li><a href="https://soundcloud.com/jobs">Jobs</a></li><li><a href="https://developers.soundcloud.com">Developers</a></li><li><a class="sc-nav__current" href="/blog/">Blog</a></li></ul><button type="button" class="sc-nav__hamburger" aria-label="Open menu" aria-expanded="false"><span></span><span></span><span></span></button></nav></header><main class="sc-main blogPost"><div class="blogPost__grid"><article class="blogPost__article"><header class="blogPost__header"><h1 class="blogPost__title">Alerting on SLOs like Pros</h1><p class="blogPost__metadata blogPost__metadataDesktop"><span>Jun 4th, 2019</span><span class="blogPost__metadataSep" aria-hidden="true">•</span><span>by <a href="https://soundcloud.com/brabenstein" target="_blank" rel="noopener noreferrer">Björn “Beorn” Rabenstein</a></span><span class="blogPost__metadataSep" aria-hidden="true">•</span><span>16<!-- --> minute<!-- -->s<!-- --> read</span></p><div class="blogPost__metadataMobile"><span>by <a href="https://soundcloud.com/brabenstein" target="_blank" rel="noopener noreferrer">Björn “Beorn” Rabenstein</a></span><div class="blogPost__metadataMobileDetails"><span>Jun 4th, 2019</span><span class="blogPost__metadataSep" aria-hidden="true">•</span><span>16<!-- --> minute<!-- -->s<!-- --> read</span></div></div><ul class="blogPost__tags"><li><a class="blogPost__tag" href="/blog/category/monitoring">Monitoring</a></li><li><a class="blogPost__tag" href="/blog/category/prometheus">Prometheus</a></li><li><a class="blogPost__tag" href="/blog/category/sre">SRE</a></li></ul></header><div class="blogPost__content"><p>If there is anything like a silver bullet for creating meaningful and actionable alerts with a high signal-to-noise ratio, it is alerting based on service-level objectives (SLOs). Fulfilling a well-defined SLO is the very definition of meeting your users’ expectations. Conversely, a certain level of service errors is OK as long as you stay within the SLO — in other words, if the SLO grants you an <em>error budget</em>. Burning through this error budget too quickly is the ultimate signal that some rectifying action is needed. The faster the budget is burned, the more urgent it is that engineers get involved.</p>
<p>This post describes how we implemented this concept at SoundCloud, enabling us to fulfill our SLOs without flooding our engineers on call with an unsustainable amount of pages.</p>
<!-- more -->
<h2>Let’s Talk SRE</h2>
<p>SLOs, alerting, and error budgets are central concepts of <a href="https://landing.google.com/sre/">Site Reliability Engineering (SRE)</a>, a discipline invented by Google in the early 2000s. Although it was initially proprietary to Google, over the last few years, SRE has become a widely discussed topic and is now applied in various flavors throughout the industry. An important milestone in this development was Google’s publication of the bestselling book <a href="https://landing.google.com/sre/sre-book/toc/index.html"><em>Site Reliability Engineering: How Google Runs Production Systems</em></a>. The book is available online for free. If you run anything even remotely at scale, there is really no excuse as to why you shouldn’t read this book in its entirety. Among all of its important chapters, there are three of particular importance for this blog post:</p>
<ul>
<li><a href="https://landing.google.com/sre/sre-book/chapters/service-level-objectives">Chapter 4 — Service Level Objectives</a> explains everything you need to know about SLOs (and SLIs and SLAs, in case you’ve ever wondered about the difference).</li>
<li><a href="https://landing.google.com/sre/sre-book/chapters/monitoring-distributed-systems">Chapter 6 — Monitoring Distributed Systems</a> lays the foundation of “alerting philosophy” and creates an understanding of when it makes sense to page engineers (and when it does not).</li>
<li><a href="https://landing.google.com/sre/sre-book/chapters/practical-alerting">Chapter 10 — Practical Alerting</a> mostly describes Borgmon, Google’s traditional internal monitoring system. While Borgmon is proprietary to Google, the author points out that you can try out the described concepts at home (or at work, even) because many current open-source monitoring systems follow similar semantics, with <a href="https://prometheus.io">Prometheus</a> receiving a special shoutout. Conveniently, we use Prometheus for the large majority of our monitoring (which is not surprising, as most of Prometheus’ <a href="https://developers.soundcloud.com/blog/prometheus-monitoring-at-soundcloud">initial development took place here at SoundCloud</a>).</li>
</ul>
<p>Keep in mind, though, that <a href="https://blog.bradfieldcs.com/you-are-not-google-84912cf44afb">you are not Google</a>. We all can learn from the giants, but blindly copying them could go horribly wrong. On that note, how different organizations ventured into SRE territory is the main topic of yet another SRE-themed book: <a href="https://www.oreilly.com/library/view/seeking-sre/9781491978856/"><em>Seeking SRE</em></a>, which was curated and edited by David Blank-Edelman. We at SoundCloud contributed a chapter, too, in which we talked about our somewhat bumpy road to applying SRE principles in an organization that is totally not Google.</p>
<p>The Google SREs themselves also felt the pain of us “mere mortals” trying to figure out how to make use of the lessons from the SRE book in our day-to-day work, so they published a companion to the original SRE book, called [<em>The Site Reliability Workbook</em>], which they again made available online for free. There are a bunch of cookbook-style chapters that are directly applicable in practice. One of them, <a href="https://landing.google.com/sre/workbook/chapters/alerting-on-slos/">Chapter 5 — Alerting on SLOs</a>, describes perfectly what we had been doing already, albeit in a less sophisticated way. With the inspiration from that chapter, we were able to refine our SLO-based alerting with great success. In fact, the solution we ended up using is so similar to the one described in the chapter that you should just read it together with this blog post to get the most out of it.</p>
<h2>Symptoms vs. Causes</h2>
<p>But let’s first take a step back and think about alerting on symptoms vs. causes. This is an important topic in <a href="https://landing.google.com/sre/sre-book/chapters/monitoring-distributed-systems">Chapter 6</a> from the original SRE book (where you can read up all the details if you feel so inclined).</p>
<p>Back in the days when your entire site ran on one LAMP server, there wasn’t much of a need to think about the difference between symptoms and causes. If your one web server went down (cause), your site was down (symptom). Waking somebody up the moment your web server stopped replying to ping probes was the right thing to do.</p>
<p>Nowadays, even mid-size sites run on hundreds, if not thousands of servers, and one or two of them are almost guaranteed to be down at any time. And that’s fine, because the software is designed to tolerate some failures. As such, it wouldn’t make any sense to wake somebody up whenever a single machine goes down.</p>
<p>You can apply the same thought to many other — actual or potential — <em>causes</em> of an outage. Paging alerts, those that wake you up in the night, should be based on <em>symptoms</em>, i.e. something that actively or imminently impacts the user experience. In addition, the alerting condition should also be urgent and actionable. If any of these requirements are not met, the alert should not be a page.</p>
<p>However, there could still be a meaningful alert on a non-paging level. Let’s call these kinds of alerts <em>tickets</em>, as somebody will have to look at them eventually, e.g. a dead server has to be dealt with at a suitable time. A user-visible but not urgent problem can easily be addressed during work hours rather than in the middle of the night, but even during an actual outage, it is helpful to get informed (via some kind of dashboard rather than via your pager) about possible causes of an outage that are detected by the monitoring system.</p>
<p>Ideally, your SLO is a precise definition of an unimpacted user experience. In other words, symptom-based alerts are usually alerts on (active or imminent) violations of your SLO.</p>
<h2>Black-Box vs. White-Box Monitoring</h2>
<p>With the lessons of symptoms vs. causes in mind, you might think now that black-box monitoring is the obvious choice for detecting symptoms, while white-box monitoring is ideal for causes. That’s a reasonable thought, but in reality, our practice at SoundCloud is a perfect match of what Google tells us in <a href="https://landing.google.com/sre/sre-book/chapters/monitoring-distributed-systems">Chapter 6</a> of the SRE book: “We combine heavy use of white-box monitoring with modest but critical uses of black-box monitoring.”</p>
<p>We mostly use black-box probes as a workaround for legacy software that isn’t yet instrumented for white-box monitoring. And we run fairly complex Catchpoint probes to simulate typical usage of our site. Interestingly, these probes are what we use to determine if we have hit our monthly, quarterly, and yearly availability goals, but we don’t use them to page anybody. Why is that so? The lazy answer is: We need white-box monitoring anyway (to investigate causes). Why should we invest in additional black-box monitoring except for in the most critical cases? But there are more reasons (quotes are again from the SRE book):</p>
<ul>
<li>Ideally, we want to detect user-impacting outages <em>before</em> they do noticeable harm. However, “for not-yet-occurring but imminent problems, black-box monitoring is fairly useless.”</li>
<li>Probes are <em>not</em> real user traffic. For example, the test track that our Catchpoint probes attempt to play might be perfectly available, while, for some complex reasons, the rap battle that has just gone viral is not.</li>
<li>Measuring long-tail latency and detecting low error rates take a lot of probes and thus a long time. However, tail latency is quite crucial in distributed systems, and our availability goals are usually measured in many nines, and so even a small error rate might violate our SLO. This could be the main reason why we don’t page anybody based on our Catchpoint probe. The probes are too slow to detect low but significant error rates, despite already limiting them to probing a very specific business KPI. The latter is a problem on its own, as we want our paging alerts to cover all relevant features of the site. However, the long-term results of the Catchpoint probes are great as a general quality calibration for our other monitoring. If they detect something, we can revisit historical alerting records to see if the near-real-time monitoring caught the issue, too.</li>
<li>In a multilayered system, like a microservice architecture, “one person’s symptom is another person’s cause.” A frontend service sees a failing backend service as a cause, while for the backend service, its own failure is a symptom, as it is now violating the (internal service-to-service) SLO. Adopting this view, we can use the white-box monitoring of the instrumented frontend service to track requests issued to the backend service. This is essentially black-box probing of the backend with the complete and real user traffic.</li>
</ul>
<p>The most rewarding place to apply the pattern described in the last point above is our HAProxy-based terminator layer at the edge of our infrastructure. An <a href="https://developers.soundcloud.com/blog/how-soundcloud-uses-haproxy-with-kubernetes-for-user-facing-traffic">earlier Backstage Blog post</a> described it in detail.</p>
<p>Recently, HAProxy announced the introduction of a native <a href="https://www.haproxy.com/blog/haproxy-exposes-a-prometheus-metrics-endpoint/">Prometheus metrics endpoint</a>, but the versions we have been working with are not yet instrumented for Prometheus. This gap is bridged by the <a href="https://github.com/prometheus/haproxy_exporter">Prometheus HAProxy exporter</a>, a small glue binary that can be scraped by the Prometheus server in the usual way, whereupon it talks to HAProxy in its own fashion to retrieve metrics and convert them to the Prometheus format and then sends them back as the scrape result. By running such an HAProxy exporter next to each HAProxy instance, we get all the metrics we need. In particular, there is a counter for all HTTP requests HAProxy has sent to the various backends, partitioned not only by the backend, but also by the HTTP status code. With Prometheus, we can easily calculate relative rates of 5xx responses over arbitrary time spans.</p>
<h2>Multiwindow, Multi-Burn-Rate Alerts</h2>
<p>With the error rate as seen by the HAProxy terminator layer, we can now apply the alerting technique described in <a href="https://landing.google.com/sre/workbook/chapters/alerting-on-slos/">Chapter 5</a> of the SRE workbook. If you haven’t yet read that chapter, now is probably really a good time to do so. It progresses quite nicely, approaching the ultimate solution iteratively and solving one problem after another. Spoiler alert: Here, we will only implement the ultimate solution, the sixth iteration in the chapter, which is called <em>Multiwindow, Multi-Burn-Rate Alerts</em>. This features a set of alerts, each of which has a different alerting threshold that corresponds to different rates of burning through the monthly error budget. Depending on the burn rate, each alert uses a combination of two different time windows over which the error rate is calculated. Fast burning is detected quickly. Slow burning needs a longer time window.</p>
<p>First, we need to actually set the SLO. The chapter from the SRE workbook assumes an error rate of 0.1% (or a 99.9% success rate) as a typical value. That’s also generally true for SoundCloud, so the various numbers calculated in the chapter make sense for our scenario. However, some backends might have a stricter or more relaxed SLO. We still keep the various burn rates and window sizes the same for them, but we allow the target error rate to be configured by backend. This configuration takes the form of rules in PromQL (the Prometheus expression language, which is also used in the SRE workbook). They look like the following:</p>
<div class="gatsby-highlight" data-language="yaml"><pre class="language-yaml"><code class="language-yaml"><span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> slos_by_backend
<span class="token key atrule">rules</span><span class="token punctuation">:</span>
<span class="token punctuation">-</span> <span class="token key atrule">record</span><span class="token punctuation">:</span> backend<span class="token punctuation">:</span>error_slo<span class="token punctuation">:</span>percent
<span class="token key atrule">labels</span><span class="token punctuation">:</span>
<span class="token key atrule">backend</span><span class="token punctuation">:</span> <span class="token string">"api-v4"</span>
<span class="token key atrule">expr</span><span class="token punctuation">:</span> <span class="token number">0.1</span>
<span class="token punctuation">-</span> <span class="token key atrule">record</span><span class="token punctuation">:</span> backend<span class="token punctuation">:</span>error_slo<span class="token punctuation">:</span>percent
<span class="token key atrule">labels</span><span class="token punctuation">:</span>
<span class="token key atrule">backend</span><span class="token punctuation">:</span> <span class="token string">"api-v3"</span>
<span class="token key atrule">expr</span><span class="token punctuation">:</span> <span class="token number">0.2</span>
<span class="token comment"># ... Many more backends.</span></code></pre></div>
<p>Next, we need the actual error rate averaged over various time windows: 5m, 30m, 1h, 2h, 6h, 1d, 3d. These are also calculated as recording rules, but they are now based on actual live data instead of a constant number. This is the rule for the 5m case:</p>
<div class="gatsby-highlight" data-language="yaml"><pre class="language-yaml"><code class="language-yaml"><span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> multiwindow_recording_rules
<span class="token key atrule">rules</span><span class="token punctuation">:</span>
<span class="token punctuation">-</span> <span class="token key atrule">record</span><span class="token punctuation">:</span> backend<span class="token punctuation">:</span>haproxy_backend_http_errors_per_response<span class="token punctuation">:</span>ratio_rate5m
<span class="token key atrule">expr</span><span class="token punctuation">:</span> <span class="token punctuation">|</span>2
sum by (backend)(rate(haproxy_backend_http_responses_total<span class="token punctuation">{</span>job="haproxy"<span class="token punctuation">,</span> code="5xx"<span class="token punctuation">}</span><span class="token punctuation">[</span>5m<span class="token punctuation">]</span>))
/
sum by (backend)(rate(haproxy_backend_http_responses_total<span class="token punctuation">{</span>job="haproxy"<span class="token punctuation">}</span><span class="token punctuation">[</span>5m<span class="token punctuation">]</span>))
<span class="token comment"># Other rules accordingly with 5m replaced by 30m, 1h, 2h, 6h, 1d, 3d.</span></code></pre></div>
<p>The <code class="language-text">rate</code> with the <code class="language-text">[5m]</code> range selector calculates requests per second, averaged over the specified timeframe, from the ever-increasing counters. The <code class="language-text">sum</code> aggregator in both enumerator and denominator separately sums up the rate from all HAProxy instances for each <code class="language-text">backend</code>. Each <code class="language-text">backend</code> ends up as a label on the elements in the resulting vector. The denominator includes all requests, while the enumerator only selects those with <code class="language-text">code=&quot;5xx</code> — which results in an error rate.</p>
<p>With all these recording rules in place, we can now assemble the multiwindow, multi-burn-rate alerts. We have chosen to use four different alerts, each combining a <em>long window</em>, a <em>short window</em>, and a <em>burn rate factor</em>, as listed in the following table.</p>
<div class="blogPost__tableWrap"><table>
<thead>
<tr>
<th>Alert</th>
<th>Long Window</th>
<th>Short Window</th>
<th><code class="language-text">for</code> Duration</th>
<th>Burn Rate Factor</th>
<th>Error Budget Consumed</th>
</tr>
</thead>
<tbody>
<tr>
<td>Page</td>
<td>1h</td>
<td>5m</td>
<td>2m</td>
<td>14.4</td>
<td>2%</td>
</tr>
<tr>
<td>Page</td>
<td>6h</td>
<td>30m</td>
<td>15m</td>
<td>6</td>
<td>5%</td>
</tr>
<tr>
<td>Ticket</td>
<td>1d</td>
<td>2h</td>
<td>1h</td>
<td>3</td>
<td>10%</td>
</tr>
<tr>
<td>Ticket</td>
<td>3d</td>
<td>6h</td>
<td>1h</td>
<td>1</td>
<td>10%</td>
</tr>
</tbody>
</table></div>
<p>Each alert fires if the SLO error rate times the <em>burn rate factor</em> is exceeded when averaged over both the <em>long window</em> and the <em>short window</em>.</p>
<p><em>Example:</em> The SLO error rate for <code class="language-text">api-v4</code> is 0.1% (to achieve “three 9s” of availability). The owners of <code class="language-text">api-v4</code> get paged if their backend has returned more than 1.44% 5xx responses over the last 1h <em>and</em> over the last 5m. They also get paged if their backend has returned more than 0.6% 5xx responses over the last 6d <em>and</em> over the last 30m.</p>
<p>The short window ensures a short reset time of the alert, i.e. an alert should stop firing soon after the problem has been solved (which is not only convenient but also allows for paging to happen again if the problem comes back).</p>
<p>Why the different burn rate factors? They enable pages for fast burning of the monthly error budget and tickets for slow burning of the budget. (Slow burning of the error budget must not go undetected and must be addressed eventually, but it’s no reason to wake somebody up in the middle of the night.) The Error Budget Consumed column lists the percentage of the monthly error budget consumed at the time the alert triggers. The math is easiest to check in the last row, with a burn rate factor of 1: If the error rate happens to be exactly at the target of 0.1%, and that happens for 3 days, then 10% of the monthly error budget is burned because 3 days is 10% of a 30-day-long month.</p>
<p>Note that an alert can fire much more quickly than within the length of the long window. A backend that suddenly starts to return 100% 5xx will cross the 1.44% hourly error rate after only 52s (1.44% of 3,600s).</p>
<p>This leaves us with the <em><code class="language-text">for</code> duration</em> column, which is interestingly missing in the SRE workbook. We’ll talk about it in a minute. But let’s first see how the alerting rules actually appear:</p>
<div class="gatsby-highlight" data-language="yaml"><pre class="language-yaml"><code class="language-yaml"><span class="token punctuation">-</span> <span class="token key atrule">name</span><span class="token punctuation">:</span> multiwindow_alerts
<span class="token key atrule">rules</span><span class="token punctuation">:</span>
<span class="token punctuation">-</span> <span class="token key atrule">alert</span><span class="token punctuation">:</span> ErrorBudgetBurn
<span class="token key atrule">expr</span><span class="token punctuation">:</span> <span class="token punctuation">|</span>2
(
100 * backend<span class="token punctuation">:</span>haproxy_backend_http_errors_per_response<span class="token punctuation">:</span>ratio_rate1h
<span class="token punctuation">></span> on (backend)
14.4 * backend<span class="token punctuation">:</span>error_slo<span class="token punctuation">:</span>percent
)
and
(
100 * backend<span class="token punctuation">:</span>haproxy_backend_http_errors_per_response<span class="token punctuation">:</span>ratio_rate5m
<span class="token punctuation">></span> on (backend)
14.4 * backend<span class="token punctuation">:</span>error_slo<span class="token punctuation">:</span>percent
)
<span class="token key atrule">for</span><span class="token punctuation">:</span> 2m
<span class="token key atrule">labels</span><span class="token punctuation">:</span>
<span class="token key atrule">system</span><span class="token punctuation">:</span> <span class="token string">"{{$labels.backend}}"</span>
<span class="token key atrule">severity</span><span class="token punctuation">:</span> <span class="token string">"page"</span>
<span class="token key atrule">long_window</span><span class="token punctuation">:</span> <span class="token string">"1h"</span>
<span class="token key atrule">annotations</span><span class="token punctuation">:</span>
<span class="token key atrule">summary</span><span class="token punctuation">:</span> <span class="token string">"a backend burns its error budget very fast"</span>
<span class="token key atrule">description</span><span class="token punctuation">:</span> <span class="token string">"Backend {{$labels.backend}} has returned {{ $value | printf `%.2f` }}% 5xx over the last hour."</span>
<span class="token key atrule">runbook</span><span class="token punctuation">:</span> <span class="token string">"https://runbooks.soundcloud.com/terminator/#errorbudgetburn"</span>
<span class="token comment"># Followed by the other three alerting rules.</span></code></pre></div>
<p>For the sake of brevity, this only lists the first of the four alerting rules. By replacing the relevant recording rules, the factors, the <code class="language-text">for</code> duration, and the applicable label values with the values from the table above, you can easily deduce how the other three alerting rules appear.</p>
<p>The SRE workbook also lists the alerting rules in PromQL, but in a more compact form. Our example here is more verbose because we have added a few things we needed for our particular use case:</p>
<ul>
<li>As explained above, the SLO error rate might be different per backend and the threshold is configured as a recording rule named <code class="language-text">backend:error_slo:percent</code>. You can find it in the expression above. Note that the <code class="language-text">&gt;</code> comparison operator is used with a modifier, <code class="language-text">on (backend)</code>, to tell Prometheus which label to use for the label matching.</li>
<li><code class="language-text">annotations</code> which follow internal conventions, are added to facilitate handling the alerts. For example, each alert should contain a link to the relevant runbook in our runbook repository.</li>
<li>We also add a <code class="language-text">system</code> label for alert routing. The Prometheus Alertmanager allows us to configure alert routing based on labels in a very powerful way. Somewhere else in our codebase, we have a registry of which team owns which systems. From there, we autogenerate an Alertmanager config that routes alerts to their owners based on the <code class="language-text">system</code> label. Everything that can be labeled (for example, pods on Kubernetes) also gets a <code class="language-text">system</code> label, and that label is propagated into Prometheus. HAProxy only knows the backend, and thus the HAProxy exporter doesn’t expose a system label. However, by convention, we use the system name as the backend name, and thus we can simply set the <code class="language-text">system</code> label to the value of the <code class="language-text">backend</code> label. (In harsh reality, the <code class="language-text">backend</code> label is a multi-component string with other components, too. Prometheus has you covered there as well, with the <code class="language-text">label_replace</code> function used to cut out the system part. This little complication is left out above to keep things a bit shorter.)</li>
<li>Finally, there is the <code class="language-text">for</code> clause, with the duration from the table above. An alert with a <code class="language-text">for</code> clause only fires once the alerting condition has been fulfilled continuously for the configured duration. Intriguingly, the SRE workbook explicitly advises against <code class="language-text">for</code> clauses because they inevitably increase the detection time for the alerting condition. We still decided to use a <code class="language-text">for</code> clause with a duration that is short compared to expected response times (of the alert itself and of the engineers getting alerted). The <code class="language-text">for</code> clause is essentially a simple but effective defense against statistical outliers. For example, if a service has just started to receive traffic after a launch or an undrain operation, the error rate over any time window will be dominated by the very short time it has received traffic at all. Without the <code class="language-text">for</code> clause, even a very short and modest initial error spike (maybe caused by cold caches) will trigger all the alerts.</li>
</ul>
<h2>Conclusion</h2>
<p>With the alerts described above, we can reliably detect breaches of configurable SLOs with an appropriate detection time (the quicker the error budget is burned, the sooner the alert will fire) and a sufficiently short reset time. The resulting alerts are strictly symptom-based and therefore always relevant. To reach a person able to act on a particular alert, it is routed to the on-call rotation owning the affected backend.</p></div></article><aside class="blogPost__sidebar"><a class="blogPost__back" href="/blog/">← back</a><div class="blogPost__adjacent"><div class="blogPost__adjacentBlock"><p class="blogPost__adjacentLabel">Previous article</p><a rel="prev" class="blogPost__adjacentTitle" href="/blog/playback-on-web-at-soundcloud">Playback on Web at SoundCloud</a></div><div class="blogPost__adjacentBlock"><p class="blogPost__adjacentLabel">Next article</p><a rel="next" class="blogPost__adjacentTitle" href="/blog/soundcloud-is-playing-the-oboe">SoundCloud Is Playing the Oboe</a></div></div></aside></div></main><div class="sc-footer-wrap"><footer class="sc-footer"><div class="sc-footer__socials"><p class="sc-footer__follow">Follow us</p><div class="sc-footer__icons"><a href="https://x.com/soundclouddev" target="_blank" rel="noopener noreferrer" title="@SoundCloudDev on X"><img src="data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHZpZXdCb3g9IjIuMDQgMyAxOS45MiAxOCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTE3LjcyODEgM0gyMC43ODE2TDE0LjExMDUgMTAuNjI0NkwyMS45NTg1IDIxSDE1LjgxMzZMMTEuMDAwNyAxNC43MDc0TDUuNDkzNiAyMUgyLjQzODIxTDkuNTczNiAxMi44NDQ2TDIuMDQ0OTggM0g4LjM0NTkxTDEyLjY5NjQgOC43NTE2OUwxNy43MjgxIDNaTTE2LjY1NjQgMTkuMTcyM0gxOC4zNDg0TDcuNDI2NTIgNC43MzE2OUg1LjYxMDgzTDE2LjY1NjQgMTkuMTcyM1oiIGZpbGw9IiNGQUZBRkEiLz4KPC9zdmc+Cg==" alt="X" width="24" height="24"/></a><a href="https://github.com/soundcloud" target="_blank" rel="noopener noreferrer" title="SoundCloud on GitHub"><img src="data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZmlsbC1ydWxlPSJldmVub2RkIiBjbGlwLXJ1bGU9ImV2ZW5vZGQiIGQ9Ik0xMS45NjQyIDBDNS4zNDgzMyAwIDAgNS41IDAgMTIuMzA0MkMwIDE3Ljc0MzIgMy40MjY4NiAyMi4zNDcyIDguMTgwODIgMjMuOTc2N0M4Ljc3NTE4IDI0LjA5OTIgOC45OTI5IDIzLjcxMiA4Ljk5MjkgMjMuMzg2MkM4Ljk5MjkgMjMuMTAxIDguOTczMzEgMjIuMTIzMiA4Ljk3MzMxIDIxLjEwNDVDNS42NDUxNCAyMS44MzggNC45NTIwOCAxOS42Mzc4IDQuOTUyMDggMTkuNjM3OEM0LjQxNzIyIDE4LjIxMTggMy42MjQ3MyAxNy44NDUyIDMuNjI0NzMgMTcuODQ1MkMyLjUzNTQzIDE3LjA5MTUgMy43MDQwOCAxNy4wOTE1IDMuNzA0MDggMTcuMDkxNUM0LjkxMjQxIDE3LjE3MyA1LjU0NjQ1IDE4LjM1NDUgNS41NDY0NSAxOC4zNTQ1QzYuNjE1OTIgMjAuMjI4NSA4LjMzOTI2IDE5LjY5OSA5LjAzMjU3IDE5LjM3M0M5LjEzMTUxIDE4LjU3ODUgOS40NDg2NSAxOC4wMjg1IDkuNzg1MzkgMTcuNzIzQzcuMTMwOTQgMTcuNDM3NyA0LjMzODEyIDE2LjM3ODUgNC4zMzgxMiAxMS42NTIzQzQuMzM4MTIgMTAuMzA3OCA0LjgxMzIyIDkuMjA3NzUgNS41NjYwNCA4LjM1MjI1QzUuNDQ3MjcgOC4wNDY3NSA1LjAzMTE4IDYuNzgzNSA1LjY4NTA2IDUuMDkyNzVDNS42ODUwNiA1LjA5Mjc1IDYuNjk1MjcgNC43NjY3NSA4Ljk3MzA2IDYuMzU1NzVDOS45NDgyNyA2LjA4NjQyIDEwLjk1NCA1Ljk0OTQgMTEuOTY0MiA1Ljk0ODI1QzEyLjk3NDQgNS45NDgyNSAxNC4wMDQyIDYuMDkxIDE0Ljk1NTIgNi4zNTU3NUMxNy4yMzMyIDQuNzY2NzUgMTguMjQzNCA1LjA5Mjc1IDE4LjI0MzQgNS4wOTI3NUMxOC44OTczIDYuNzgzNSAxOC40ODEgOC4wNDY3NSAxOC4zNjIyIDguMzUyMjVDMTkuMTM0OSA5LjIwNzc1IDE5LjU5MDQgMTAuMzA3OCAxOS41OTA0IDExLjY1MjNDMTkuNTkwNCAxNi4zNzg1IDE2Ljc5NzYgMTcuNDE3MiAxNC4xMjMzIDE3LjcyM0MxNC41NTkyIDE4LjExIDE0LjkzNTMgMTguODQzMyAxNC45MzUzIDIwLjAwNDVDMTQuOTM1MyAyMS42NTQ1IDE0LjkxNTggMjIuOTc4OCAxNC45MTU4IDIzLjM4NkMxNC45MTU4IDIzLjcxMiAxNS4xMzM3IDI0LjA5OTIgMTUuNzI3OCAyMy45NzdDMjAuNDgxOCAyMi4zNDcgMjMuOTA4NyAxNy43NDMyIDIzLjkwODcgMTIuMzA0MkMyMy45MjgyIDUuNSAxOC41NjAzIDAgMTEuOTY0MiAwWiIgZmlsbD0iI0ZBRkFGQSIvPgo8L3N2Zz4K" alt="GitHub" width="24" height="24"/></a><a href="https://bsky.app/profile/soundcloud.dev" target="_blank" rel="noopener noreferrer" title="@soundcloud.dev on Bluesky"><img src="data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGNsaXBQYXRoIGlkPSJjbGlwIj4KICA8cmVjdCB3aWR0aD0iMjQiIGhlaWdodD0iMjQiLz4KPC9jbGlwUGF0aD4KPHBhdGggY2xpcC1wYXRoPSJ1cmwoI2NsaXApIiBkPSJNNS4yMDIzIDEuNjEyNjRDNy45NTM4NSAzLjk1NDYzIDEwLjkxMzMgOC43MDMyMyAxMiAxMS4yNTE0QzEzLjA4NjcgOC43MDMyOCAxNi4wNDYyIDMuOTU0NjMgMTguNzk3NyAxLjYxMjZDMjAuNzgzMSAtMC4wNzcxNzY2IDI0IC0xLjM4NDcyIDI0IDIuNzc1OUMyNCAzLjYwNjg1IDIzLjU3OTggOS43NTYxNyAyMy4zMzMzIDEwLjc1NDZDMjIuNDc2NiAxNC4yMjU0IDE5LjM1NDkgMTUuMTEwNyAxNi41NzggMTQuNTc0OEMyMS40MzE5IDE1LjUxMTQgMjIuNjY2NiAxOC42MTM3IDIwIDIxLjcxNTlDMTQuOTM1NSAyNy42MDc4IDEyLjcyMDkgMjAuMjM3NiAxMi4xNTM0IDE4LjM0OTJDMTIuMDQ5NCAxOC4wMDMgMTIuMDAwNyAxNy44NDEgMTIgMTcuOTc4N0MxMS45OTkzIDE3Ljg0MSAxMS45NTA2IDE4LjAwMyAxMS44NDY2IDE4LjM0OTJDMTEuMjc5MSAyMC4yMzc2IDkuMDY0NTIgMjcuNjA3OCA0LjAwMDAxIDIxLjcxNTlDMS4zMzMzNSAxOC42MTM3IDIuNTY4MDQgMTUuNTExNCA3LjQyMTk2IDE0LjU3NDhDNC42NDUwNiAxNS4xMTA2IDEuNTIzMzcgMTQuMjI1MyAwLjY2NjY3NiAxMC43NTQ1QzAuNDIwMjExIDkuNzU2MTIgMCAzLjYwNjc1IDAgMi43NzU4NUMwIC0xLjM4NDcyIDMuMjE2OTcgLTAuMDc3MjI0NSA1LjIwMjMgMS42MTI2NFoiIGZpbGw9IiNGQUZBRkEiLz4KPC9zdmc+Cg==" alt="Bluesky" width="24" height="24"/></a><a href="https://stackoverflow.com/questions/tagged/soundcloud" target="_blank" rel="noopener noreferrer" title="SoundCloud on Stack Overflow"><img src="data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjAiIGhlaWdodD0iMjQiIHZpZXdCb3g9IjAgMCAyMCAyNCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGNsaXBQYXRoIGlkPSJjbGlwIj4KICA8cmVjdCB3aWR0aD0iMjAiIGhlaWdodD0iMjQiLz4KPC9jbGlwUGF0aD4KPHBhdGggY2xpcC1wYXRoPSJ1cmwoI2NsaXApIiBkPSJNMTcuMjczMSAxMy4wNTE2TDE3LjMxMiAxMy4wNjc2TDE3LjI5NjUgMTMuMDc1NkwxNy4yNzMxIDEzLjA1MTZaTTE3LjI3MzEgMTMuMDUxNkMxNi41MDI3IDE0LjA2NzggMTUuODYxMyAxNS4xODA0IDE1LjM2NDIgMTYuMzYyNUwxNS4zNDA5IDE2LjQxODVDMTQuODUyNiAxNy41ODYxIDE0LjUxNzYgMTguODEyNCAxNC4zMzU4IDIwLjA5NzNWMjAuMTIxM0MxNC4xNTI0IDIxLjQwNjUgMTQuMTM5MyAyMi43MTEyIDE0LjI5NjggMjRIMFYyMC4wOTczSDEyLjQxOTJMMTIuNDk3MSAxOS42MTc1TDAuNTI5ODAxIDE2LjM3ODVMMS41MjcwNyAxMi42MTk4TDEzLjY0MjQgMTUuODk4N0wxMy44MTM4IDE1LjUzMDhMMi45MzcyOCA5LjE2NDk0TDQuODYxNzEgNS43OTAwN0wxNS44OTQgMTIuMjQzOUwxNi4xNTg5IDExLjkyNEw3LjEyMTE1IDIuNzc1MDdMOS44NDgwNyAwTDE5LjAyNjEgOS4zMDg5TDIwIDEwLjI5MjZDMTguOTc2OCAxMS4wOTIzIDE4LjA2NzggMTIuMDA5MyAxNy4yNzMxIDEzLjA0MzciIGZpbGw9IiNGQUZBRkEiLz4KPC9zdmc+Cg==" alt="Stack Overflow" width="20" height="24"/></a><a href="https://www.linkedin.com/company/soundcloud" target="_blank" rel="noopener noreferrer" title="SoundCloud on LinkedIn"><img src="data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjQiIGhlaWdodD0iMjQiIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTIwLjg5MjkgMjAuMjgxMkgxNy4yNTk2VjE0Ljc1NzlDMTcuMjU5NiAxMy40NDA4IDE3LjIzNTQgMTEuNzQ1MyAxNS4zNjk5IDExLjc0NTNDMTMuNDc3NSAxMS43NDUzIDEzLjE4OCAxMy4xODAzIDEzLjE4OCAxNC42NjJWMjAuMjgwOEg5LjU1NDY0VjguOTIyNjZIMTMuMDQyNlYxMC40NzQ5SDEzLjA5MTVDMTMuNDQwNSA5Ljg5NTUyIDEzLjk0NDkgOS40MTg5MiAxNC41NTEgOS4wOTU4MkMxNS4xNTcgOC43NzI3MyAxNS44NDIgOC42MTUyNSAxNi41MzI5IDguNjQwMTRDMjAuMjE1NCA4LjY0MDE0IDIwLjg5NDQgMTAuOTkxNCAyMC44OTQ0IDE0LjA1MDNMMjAuODkyOSAyMC4yODEyWk01LjQ1NTA1IDcuMzcwMDhDNS4wMzgwMyA3LjM3MDE1IDQuNjMwMzUgNy4yNTAxOSA0LjI4MzU3IDcuMDI1MzVDMy45MzY4IDYuODAwNTIgMy42NjY1MSA2LjQ4MDkxIDMuNTA2ODYgNi4xMDY5NUMzLjM0NzIgNS43MzI5OSAzLjMwNTM2IDUuMzIxNDcgMy4zODY2NCA0LjkyNDQzQzMuNDY3OTMgNC41MjczOSAzLjY2ODY4IDQuMTYyNjYgMy45NjM1IDMuODc2MzdDNC4yNTgzMiAzLjU5MDA4IDQuNjMzOTggMy4zOTUwOSA1LjA0Mjk3IDMuMzE2MDRDNS40NTE5NSAzLjIzNyA1Ljg3NTg5IDMuMjc3NDYgNi4yNjExOSAzLjQzMjNDNi42NDY0OSAzLjU4NzE0IDYuOTc1ODQgMy44NDk0MiA3LjIwNzU4IDQuMTg1OTZDNy40MzkzMiA0LjUyMjUgNy41NjMwNiA0LjkxODE5IDcuNTYzMTQgNS4zMjNDNy41NjMxOSA1LjU5MTc3IDcuNTA4NjkgNS44NTc5MyA3LjQwMjc4IDYuMTA2MjdDNy4yOTY4NiA2LjM1NDYxIDcuMTQxNjEgNi41ODAyNiA2Ljk0NTg2IDYuNzcwMzVDNi43NTAxIDYuOTYwNDQgNi41MTc2OCA3LjExMTI0IDYuMjYxODkgNy4yMTQxNEM2LjAwNjEgNy4zMTcwNCA1LjczMTk0IDcuMzcwMDMgNS40NTUwNSA3LjM3MDA4Wk03LjI3MTcxIDIwLjI4MTJIMy42MzQ2VjguOTIyNjZINy4yNzE3MVYyMC4yODEyWk0yMi43MDQzIDAuMDAxMzc3NTdIMS44MDg4NkMxLjMzNDYgLTAuMDAzODE3NzkgMC44Nzc1ODUgMC4xNzM5NzUgMC41MzgyOTEgMC40OTU2ODZDMC4xOTg5OTcgMC44MTczOTcgMC4wMDUxNzc4MSAxLjI1NjcxIC0wLjAwMDYxMDM1MiAxLjcxNzA4VjIyLjA4NDdDMC4wMDQ5Nzk3MSAyMi41NDUzIDAuMTk4Njg1IDIyLjk4NDkgMC41Mzc5NjggMjMuMzA3QzAuODc3MjUgMjMuNjI5IDEuMzM0MzYgMjMuODA3MiAxLjgwODg2IDIzLjgwMjNIMjIuNzA0M0MyMy4xNzk3IDIzLjgwOCAyMy42MzgxIDIzLjYzMDQgMjMuOTc4NyAyMy4zMDg0QzI0LjMxOTMgMjIuOTg2MyAyNC41MTQzIDIyLjU0NjIgMjQuNTIwOSAyMi4wODQ3VjEuNzE1NjFDMjQuNTE0MSAxLjI1NDMyIDI0LjMxOSAwLjgxNDUxNSAyMy45Nzg0IDAuNDkyODEyQzIzLjYzNzcgMC4xNzExMDggMjMuMTc5NSAtMC4wMDYxNzYzOCAyMi43MDQzIC05LjI1NzU2ZS0wNSIgZmlsbD0iI0ZBRkFGQSIvPgo8L3N2Zz4K" alt="LinkedIn" width="24" height="24"/></a></div></div><ul class="sc-footer__links"><li><a href="https://status.soundcloud.com" target="_blank" rel="noopener noreferrer">Status</a></li><li><a href="https://developers.soundcloud.com/docs/api/privacy">Privacy</a></li><li><a href="https://developers.soundcloud.com/docs/api/cookies">Cookies</a></li><li><button type="button" class="sc-footer__cookieManager">Cookie Manager</button></li><li><a href="https://soundcloud.com/imprint" target="_blank" rel="noopener noreferrer">Imprint</a></li></ul></footer></div></div></div><div id="gatsby-announcer" style="position:absolute;top:0;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0" aria-live="assertive" aria-atomic="true"></div></div><script id="gatsby-script-loader">/*<![CDATA[*/window.pagePath="/alerting-on-slos";/*]]>*/</script><script id="gatsby-chunk-mapping">/*<![CDATA[*/window.___chunkMapping={"polyfill":["/polyfill-9217de6ac40c524f2054.js"],"app":["/app-3fa20cd6fd93205429a7.js"],"component---src-pages-404-js":["/component---src-pages-404-js-1ebba9e0e8ef03bdd584.js"],"component---src-pagination-blog-posts-by-category-js":["/component---src-pagination-blog-posts-by-category-js-c510125cc77ad0d32f6e.js"],"component---src-pagination-blog-posts-js":["/component---src-pagination-blog-posts-js-f9d42bd5c2b11bcfff36.js"],"component---src-templates-blog-post-js":["/component---src-templates-blog-post-js-95a2dd758be9473349e5.js"]};/*]]>*/</script><script src="/blog/polyfill-9217de6ac40c524f2054.js" nomodule=""></script><script src="/blog/component---src-templates-blog-post-js-95a2dd758be9473349e5.js" async=""></script><script src="/blog/commons-05739180ee3b1321cf79.js" async=""></script><script src="/blog/styles-0ec71dd62c66cb95665c.js" async=""></script><script src="/blog/app-3fa20cd6fd93205429a7.js" async=""></script><script src="/blog/framework-ac576c16d7c8bd3ef403.js" async=""></script><script src="/blog/webpack-runtime-500dd8fe0fe45b062595.js" async=""></script></body></html>