14 lines
85 KiB
HTML
14 lines
85 KiB
HTML
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=yes"/><link rel="preload" as="image" href="/images/logo.png"/><link rel="preload" as="image" href="/images/linked-in.svg"/><link rel="preload" as="image" href="/images/x.svg"/><link rel="preload" as="image" href="/images/yt.svg"/><link rel="preload" as="image" href="/images/insta.svg"/><link rel="preload" as="image" href="/images/f.svg"/><link rel="preload" as="image" href="/images/feed.svg"/><link rel="preload" as="image" href="https://images.ctfassets.net/p762jor363g1/attachment_943e369d8982615e731b91feed9abde7/360a9dc4dcc43513c3d502072fd607bb/attachment_943e369d8982615e731b91feed9abde7.png"/><link rel="preload" as="image" href="/images/x-blue.svg"/><link rel="preload" as="image" href="/images/f-blue.svg"/><link rel="stylesheet" href="/_next/static/chunks/0sxc27dsa.k6-.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/chunks/0zaia6i55l01..js"/><script src="/_next/static/chunks/0pqt~8bl3ukh4.js" async=""></script><script src="/_next/static/chunks/0m_p1bxtorv5i.js" async=""></script><script src="/_next/static/chunks/02g3221oh~3le.js" async=""></script><script src="/_next/static/chunks/turbopack-0~dymj8._h2gc.js" async=""></script><script src="/_next/static/chunks/0a_f9~n73ra-c.js" async=""></script><script src="/_next/static/chunks/0d3shmwh5_nmn.js" async=""></script><script src="/_next/static/chunks/08c8uq~xsq31o.js" async=""></script><script src="/_next/static/chunks/098votwdj80t9.js" async=""></script><script src="/_next/static/chunks/0wni0lk_-haja.js" async=""></script><link rel="preload" as="image" href="/images/linked-in-blue.svg"/><link rel="preload" as="image" href="/images/copy.svg"/><link rel="preload" as="image" href="https://images.ctfassets.net/p762jor363g1/7MYEdTbczd11VcDP5GMLrp/755c9f5d9d2bc9362c1b3c5baf6f6df8/image3.png"/><link rel="preload" as="image" href="https://images.ctfassets.net/p762jor363g1/69pB35ICFY1sL97klnboyf/aed20373e05048d11cdd12de739b4524/image1.png"/><link rel="preload" as="image" href="https://images.ctfassets.net/p762jor363g1/7l2eUSuEW1R60JzrWTVCTa/c29b1e8011108d7c79a3b4d4d6f6237f/IMG_5977.png"/><link rel="preload" as="image" href="https://images.ctfassets.net/p762jor363g1/1liN9CCW7JU64GI5r03T6N/95c8d19c1287e6209831f31f8dd89158/3__1_.png"/><title>Incident Management at Spotify | Spotify Engineering</title><meta name="description" content="A few weeks ago Spotify had one of the biggest incidents in the last few years. It caused a major outage for a big chunk of our European users. For a few hours the music playback experience was damaged. Our users would see high latency when playing music and some of them were unable to log in."/><meta name="application-name" content="Spotify Engineering"/><meta name="author" content="David Poblador i Garcia"/><meta name="robots" content="index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1"/><meta name="twitter:data1" content="David Poblador i Garcia"/><meta name="twitter:data2" content="6 minutes"/><meta name="twitter:label1" content="Written by"/><meta name="twitter:label2" content="Est. reading time"/><meta property="og:title" content="Incident Management at Spotify | Spotify Engineering"/><meta property="og:description" content="A few weeks ago Spotify had one of the biggest incidents in the last few years. It caused a major outage for a big chunk of our European users. For a few hours the music playback experience was damaged. Our users would see high latency when playing music and some of them were unable to log in."/><meta property="og:site_name" content="Spotify Engineering"/><meta property="og:image" content="https://images.ctfassets.net/p762jor363g1/attachment_943e369d8982615e731b91feed9abde7/360a9dc4dcc43513c3d502072fd607bb/attachment_943e369d8982615e731b91feed9abde7.png"/><meta property="og:image:type" content="image/jpeg"/><meta property="og:image:width" content="1200"/><meta property="og:image:height" content="630"/><meta property="og:image:alt" content="featured image"/><meta property="og:type" content="article"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:creator" content="David Poblador i Garcia"/><meta name="twitter:title" content="Incident Management at Spotify | Spotify Engineering"/><meta name="twitter:description" content="Incident Management at Spotify | Spotify Engineering"/><meta name="twitter:image" content="https://images.ctfassets.net/p762jor363g1/attachment_943e369d8982615e731b91feed9abde7/360a9dc4dcc43513c3d502072fd607bb/attachment_943e369d8982615e731b91feed9abde7.png"/><meta name="twitter:image:alt" content="featured image"/><meta name="twitter:image:type" content="image/jpeg"/><meta name="twitter:image:width" content="1200"/><meta name="twitter:image:height" content="630"/><link rel="icon" href="/favicon.ico?favicon.0cd2g1ew_zd4v.ico" sizes="250x250" type="image/x-icon"/><script src="/_next/static/chunks/03~yq9q893hmn.js" noModule=""></script></head><body><div hidden=""><!--$--><!--/$--></div><div class="page-wrapper show-notification"><div class="content-wrapper "><!--$?--><template id="B:0"></template><!--/$--><div class="single-post"><div class="container"><h1 class="single-post__title">Incident Management at Spotify</h1></div><div class="container"><div class="single-post__content"><div class="single-post__featured-image"><img alt="Feature Image" width="740" height="404" decoding="async" data-nimg="1" style="color:transparent" src="https://images.ctfassets.net/p762jor363g1/attachment_943e369d8982615e731b91feed9abde7/360a9dc4dcc43513c3d502072fd607bb/attachment_943e369d8982615e731b91feed9abde7.png"/></div><div class="html-content"><p >A few weeks ago Spotify had one of the biggest incidents in the last few years. It caused a major outage for a big chunk of our European users. For a few hours the music playback experience was damaged. Our users would see high latency when playing music and some of them were unable to log in.</p><p >At Spotify we take these types of incidents seriously. Our mission when dealing with outages is to avoid having the same incident twice. We want to be able to learn something new with each incident. However, sometimes we fail.</p><h2>First symptoms: two months before the outage</h2><p >Two months before the big outage we had an incident connected with one of our smallest backend services: Popcount. Popcount (this is our internal name) is the service that takes care of storing the list of subscribers for each of our more than 1 billion playlists.</p><p >All our backend services and clients that communicate with Popcount are designed to fail fast in the event that the service is not available or too slow. Failing fast is key to scaling. We try to build our systems in such a way that partial failures are handled gracefully. For instance, it is better not to display the list of playlist subscribers for a while than to delay showing playlist data for a long time, completely damaging the user experience.</p><p >However, there was a piece of legacy code in our desktop client that did not honor that requirement. That particular code caused our clients to retry fetching Popcount information for every request that timed out. To make matters worse, the legacy implementation had no backoff logic at all. This put a lot of pressure on the Popcount service and made it reach a state where servers were overwhelmed in such a way that recovery was almost impossible: the slower the service became, the more requests it started receiving. Turning it off wouldn’t have helped either: the amount of pending requests coming from our clients was too big to be served without damaging the service again.</p><p >The Popcount developers quickly deployed a fix: in the event that the service became slow, we would fast fail and return an empty list of subscribers. With that fix in place, our Popcount system started making forward progress and very soon our clients went back to normal.</p><h2>Remediations: why are they important?</h2><p >For every incident we hold a post-mortem meeting. The goal of this meeting is producing a list of remediations that we need to put in place in order to prevent that incident from happening again and assign an urgency to each of them. We invite all the teams that are connected with a system involved in the given incident. If the severity of an incident is minor or it is connected with a very isolated part of our infrastructure, the post-mortem meetings are limited to a few key engineers. Post-mortem meetings are not about blame, they are about improving our systems and our processes!</p><p >One of the remediations for the Popcount incident was fixing the faulty retry behavior in our desktop client when fetching data from the Popcount system.</p><p >Distributed systems are hard, prioritizing bugs and assigning the right urgency is often non-obvious. There are many challenges we face when we release software to millions of users, and sometimes assessing the relative urgency of a fix is not easy. In this particular case, we focussed in the mitigations (fast fail behavior in Popcount) instead of in the root cause (the client bug).</p><p >We did not set the proper priority for the bugfix, and, guess what…</p><h2>The night of April 27th</h2><p >It was a rainy Saturday night in Stockholm. Some of our engineers were online on IRC, and discovered that Popcount was unhealthy again. We thought the fast fail logic we introduced in Popcount a few weeks before would prevent Popcount from dying again.</p>
|
||
<figure class="figure-image">
|
||
|
||
<img
|
||
src="//images.ctfassets.net/p762jor363g1/attachment_c8b02a41e754498534e445cbbadb9156/1b91c5dafa01f39b4f08837f387ad040/attachment_c8b02a41e754498534e445cbbadb9156.png"
|
||
style="width: 100%; max-width: 100%; height: auto; "
|
||
class="blog-image"
|
||
alt="attachment_c8b02a41e754498534e445cbbadb9156"
|
||
/>
|
||
|
||
<figcaption class="caption"><p><i>Request delay for Popcount in one of our Datacenters</i></p></figcaption>
|
||
</figure>
|
||
<p >What happened? Why was Popcount being hammered with requests again? A few days earlier we started rolling out our new cool Discovery feature to a percentage of our users. We had unknowingly added a dependency between our Discovery service (Bartender is our internal name) and Popcount. Some of the content we display in our discovery page display playlist counts, hence the Bartender service was fetching data directly from Popcount. The fast fail logic we introduced before did not help much: the service was getting more requests than it could handle. We realized too late that Popcount was underprovisioned.</p><p >Our Site Reliability and Infrastructure teams thought that removing that dependency via a hotfix would solve the problem. But we were wrong.</p><p >By the time we applied the fix, the Popcount request queues in our Accesspoints (the servers our clients connect to) had built up and some excessive logging in the Accesspoints making them die like flies. We intentionally had added more verbose logging of errors for debugging purposes, and we managed to shoot ourselves in the foot with it. Their I/O responsiveness was nonexistent and the wrong retry behavior in our desktop clients was making things worse and worse. Saturday night is usually a very busy night for our servers, and that did not help either.</p><p >By that time our service was notably degraded and we were in a situation where most of our Accesspoints were unreachable or responding extremely slowly. It was time for drastic measures.</p><p >We needed to do something to allow them to start breathing and the only possible solution was firewalling them off. Some of them were so unresponsive that we had to perform a hard reset before applying the necessary firewall rules. A few minutes later, things were responsive again. We removed the firewall blocks and the service started working smoothly as usual.</p><p >Why did that help? Well, our clients have a nice retry behavior when they are disconnected from the Accesspoints. They have an exponential backoff logic and offlining those servers for a while triggered the backoff that helped us make forward progress.</p><h2>Lessons learned</h2><ul><li><p >We should have prioritized fixing the backoff bug in the client after the first Popcount incident, we did not assign the right severity. We thought having the fast fail mechanism in place would be enough to keep the service up in case we experienced high latencies.</p></li><li><p >Logging is nice, but logging too much can be damaging when many error messages or warnings are produced.</p></li><li><p >We should spend more time testing extreme cases (high latency, low bandwidth) for our backend systems</p></li></ul><p >Sometimes, things you do to try to understand problems better (adding extra logging, to name one) can really come back to bite you.</p><h2>Remediations in place after the second incident</h2><p >After the outage (infamously known as OPS-6000 in our incident tracking system), we held a post-mortem meeting where we invited the entire Technology and Product departments. Many people had the opportunity to attend a post-mortem meeting for the first time.</p><p >We came up with a list of remediations, all of them are in place now. Some of them:</p><ul><li><p >All the data served by our discovery service should be statically cached with a TTL.</p></li><li><p >Fix the faulty retry behavior for Popcount in our desktop client.</p></li><li><p >Move valuable Accesspoint log messages into a separate log. In case we hit a similar IO problem in the future, we can easily discard non important messages. We also introduced rate limiting for those messages.</p></li><li><p >Implement some syslog fixes to prevent too frequent flushes to disk.</p></li></ul><p >Educating the company on why and how we failed is just as important as the fixes (it was even a remediation for this incident).</p></div><div>Tags:<!-- --> <a href="/tag/incident">incident</a>, <a href="/tag/incident-management">incident management</a>, <a href="/tag/outage">outage</a>, <a href="/tag/scalability">scalability</a>, <a href="/tag/sre">sre</a>, <a href="/tag/systems">systems</a></div><div class="social"><p>SHARE THIS ARTICLE</p><ul class="social__links"><li><a href="https://twitter.com/intent/tweet?url=https%3A%2F%2Fengineering.atspotify.com%2F2013%2F6%2Fincident-management-at-spotify&text=Incident%20Management%20at%20Spotify" target="_blank"><img alt="x" width="24" height="24" decoding="async" data-nimg="1" style="color:transparent" src="/images/x-blue.svg"/></a></li><li><a href="https://www.facebook.com/sharer/sharer.php?u=https%3A%2F%2Fengineering.atspotify.com%2F2013%2F6%2Fincident-management-at-spotify" target="_blank" rel="noopener noreferrer"><img alt="fb" width="24" height="24" decoding="async" data-nimg="1" style="color:transparent" src="/images/f-blue.svg"/></a></li><li><a href="https://www.linkedin.com/sharing/share-offsite/?url=https%3A%2F%2Fengineering.atspotify.com%2F2013%2F6%2Fincident-management-at-spotify" target="_blank" rel="noopener noreferrer"><img alt="linkedin" width="24" height="24" decoding="async" data-nimg="1" style="color:transparent" src="/images/linked-in-blue.svg"/></a></li><li><a><img alt="copy" width="24" height="24" decoding="async" data-nimg="1" style="color:transparent" src="/images/copy.svg"/></a></li></ul></div></div><aside class="sidebar"><div class="date">Jun 4, 2013</div><div class="published">Published by David Poblador i Garcia</div><div class="tag-wrapper"><a class="tag fill" href="/category/data">Data</a><a class="tag fill" href="/category/infrastructure">Infrastructure</a><a class="tag fill" href="/category/people">People</a></div><div class="social"><p>SHARE THIS ARTICLE</p><ul class="social__links"><li><a href="https://twitter.com/intent/tweet?url=https%3A%2F%2Fengineering.atspotify.com%2F2013%2F6%2Fincident-management-at-spotify&text=Incident%20Management%20at%20Spotify" target="_blank"><img alt="x" width="24" height="24" decoding="async" data-nimg="1" style="color:transparent" src="/images/x-blue.svg"/></a></li><li><a href="https://www.facebook.com/sharer/sharer.php?u=https%3A%2F%2Fengineering.atspotify.com%2F2013%2F6%2Fincident-management-at-spotify" target="_blank" rel="noopener noreferrer"><img alt="fb" width="24" height="24" decoding="async" data-nimg="1" style="color:transparent" src="/images/f-blue.svg"/></a></li><li><a href="https://www.linkedin.com/sharing/share-offsite/?url=https%3A%2F%2Fengineering.atspotify.com%2F2013%2F6%2Fincident-management-at-spotify" target="_blank" rel="noopener noreferrer"><img alt="linkedin" width="24" height="24" decoding="async" data-nimg="1" style="color:transparent" src="/images/linked-in-blue.svg"/></a></li><li><a><img alt="copy" width="24" height="24" decoding="async" data-nimg="1" style="color:transparent" src="/images/copy.svg"/></a></li></ul></div></aside></div></div><div class="related-posts"><div class="container"><h3>Related articles</h3></div><div class="container"><div class="cards"><div class="post-card related"><div class="post-card__image"><a href="/2026/9/why-spotify-is-not-using-bayesian-a-b-testing"><img alt="sticky" width="400" height="197" decoding="async" data-nimg="1" style="color:transparent" src="https://images.ctfassets.net/p762jor363g1/7MYEdTbczd11VcDP5GMLrp/755c9f5d9d2bc9362c1b3c5baf6f6df8/image3.png"/></a></div><div class="post-card__content"><div class="top-content"><div class="post-card__date">Sep 8, 2026</div><a href="/2026/9/why-spotify-is-not-using-bayesian-a-b-testing"><h4 class="post-card__title">Why Spotify Is Not Using Bayesian A/B Testing</h4></a><div class="post-card__description">Clearing the confusion about what Bayesian A/B testing is.</div><div class="post-card__published"></div></div><div class="post-card__tags"></div></div></div><div class="post-card related"><div class="post-card__image"><a href="/2026/7/indexing-the-data-lake-for-online-point-queries"><img alt="sticky" width="400" height="197" decoding="async" data-nimg="1" style="color:transparent" src="https://images.ctfassets.net/p762jor363g1/69pB35ICFY1sL97klnboyf/aed20373e05048d11cdd12de739b4524/image1.png"/></a></div><div class="post-card__content"><div class="top-content"><div class="post-card__date">Jul 27, 2026</div><a href="/2026/7/indexing-the-data-lake-for-online-point-queries"><h4 class="post-card__title">Indexing the Data Lake for Online Point Queries</h4></a><div class="post-card__description">Companies like Spotify need vast quantities of data accessible at low latency for online services and,...</div><div class="post-card__published"></div></div><div class="post-card__tags"></div></div></div><div class="post-card related"><div class="post-card__image"><a href="/2026/6/encoding-your-domain-expert-the-context-layer-behind-spotifys-data-assistant"><img alt="sticky" width="400" height="197" decoding="async" data-nimg="1" style="color:transparent" src="https://images.ctfassets.net/p762jor363g1/7l2eUSuEW1R60JzrWTVCTa/c29b1e8011108d7c79a3b4d4d6f6237f/IMG_5977.png"/></a></div><div class="post-card__content"><div class="top-content"><div class="post-card__date">Jun 10, 2026</div><a href="/2026/6/encoding-your-domain-expert-the-context-layer-behind-spotifys-data-assistant"><h4 class="post-card__title">Encoding Your Domain Expert: The Context Layer Behind Spotify's Data Assistant</h4></a><div class="post-card__description">At Spotify, data problems used to follow a specific pattern. You'd look for the relevant dashboard, there...</div><div class="post-card__published"></div></div><div class="post-card__tags"></div></div></div><div class="post-card related"><div class="post-card__image"><a href="/2026/5/better-experiments-with-llm-evals-a-funnel-not-a-fork"><img alt="sticky" width="400" height="197" decoding="async" data-nimg="1" style="color:transparent" src="https://images.ctfassets.net/p762jor363g1/1liN9CCW7JU64GI5r03T6N/95c8d19c1287e6209831f31f8dd89158/3__1_.png"/></a></div><div class="post-card__content"><div class="top-content"><div class="post-card__date">May 18, 2026</div><a href="/2026/5/better-experiments-with-llm-evals-a-funnel-not-a-fork"><h4 class="post-card__title">Better Experiments with LLM Evals — A funnel, not a fork</h4></a><div class="post-card__description">TL;DR LLM evals, automated judges that assess relevance, coherence, and quality at scale, are a powerful new...</div><div class="post-card__published"></div></div><div class="post-card__tags"></div></div></div></div></div></div><!--$--><!--/$--></div><footer class="footer"><div class="container"><section class="subscription"><div class="subscription__left"><h3>Want to stay in the loop?</h3><p>Subscribe to our newsletter to never miss an update!</p><small>By clicking sign up you’ll receive occasional emails from Spotify. You always have the choice to adjust your interest settings or unsubscribe.</small></div><div class="subscription__right"><div id="mc_embed_signup"><form id="mc-embedded-subscribe-form" name="mc-embedded-subscribe-form" class="validate" action="https://spotify.us20.list-manage.com/subscribe/post?u=99aa88d3fba458716d0cd1299&id=28cc680980" method="post" target="_blank"><div id="mc_embed_signup_scroll"><div class="mc-field-group"><input type="email" class="required email" id="mce-EMAIL" placeholder="Enter your best email" name="EMAIL"/><input type="submit" id="mc-embedded-subscribe" class="button button--submit" name="subscribe" value="Subscribe"/></div><div id="mce-responses" class="clear"><div class="response" id="mce-error-response" style="display:none"></div><div class="response" id="mce-success-response" style="display:none"></div></div><div style="left:-5000px;position:absolute" aria-hidden="true"><input type="text" name="b_99aa88d3fba458716d0cd1299_28cc680980"/></div></div></form></div></div></section><section class="links"><div class="links__left"><div class="logo"><a href="/"><img alt="feed" loading="lazy" width="167" height="50" decoding="async" data-nimg="1" style="color:transparent" src="/images/footer-logo.svg"/></a></div><div class="nav-wrapper"><nav class="nav"><ul><li><a target="_blank" rel="noopener noreferrer" href="https://spotify.com/">Spotify.com</a></li><li><a target="_blank" rel="noopener noreferrer" href="https://www.lifeatspotify.com/">Spotify Jobs</a></li><li><a target="_blank" rel="noopener noreferrer" href="https://newsroom.spotify.com/">Newsroom</a></li></ul><ul><li><a target="_blank" rel="noopener noreferrer" href="https://research.atspotify.com/">Spotify R&D Research</a></li><li><a target="_blank" rel="noopener noreferrer" href="https://backstage.spotify.com/">Spotify for Backstage</a></li></ul></nav></div></div><div class="links__right"><ul class="social"><li><a target="_blank" rel="noopener noreferrer" href="https://www.linkedin.com/showcase/spotify-r&d"><img alt="Linked In" width="24" height="24" decoding="async" data-nimg="1" style="color:transparent" src="/images/linked-in.svg"/></a></li><li><a target="_blank" rel="noopener noreferrer" href="https://twitter.com/spotifyeng"><img alt="X" width="24" height="24" decoding="async" data-nimg="1" style="color:transparent" src="/images/x.svg"/></a></li><li><a target="_blank" rel="noopener noreferrer" href="https://www.youtube.com/@SpotifyRnD"><img alt="Youtube" width="24" height="24" decoding="async" data-nimg="1" style="color:transparent" src="/images/yt.svg"/></a></li><li><a target="_blank" rel="noopener noreferrer" href="http://www.instagram.com/lifeatspotify"><img alt="Instagram" width="24" height="24" decoding="async" data-nimg="1" style="color:transparent" src="/images/insta.svg"/></a></li><li><a target="_blank" rel="noopener noreferrer" href="https://www.facebook.com/spotify"><img alt="facebook" width="24" height="24" decoding="async" data-nimg="1" style="color:transparent" src="/images/f.svg"/></a></li><li><a target="_blank" rel="noopener noreferrer" href="/feed"><img alt="feed" width="24" height="24" decoding="async" data-nimg="1" style="color:transparent" src="/images/feed.svg"/></a></li></ul></div></section><section class="copy"><nav class="copy__nav"><ul><li><a target="_blank" rel="noopener noreferrer" href="https://www.spotify.com/us/legal/end-user-agreement/">Legal</a></li><li><a target="_blank" rel="noopener noreferrer" href="https://www.spotify.com/us/legal/privacy-policy/">Privacy</a></li><li><a target="_blank" rel="noopener noreferrer" href="https://www.spotify.com/legal/cookies-policy/">Cookies</a></li><li><a target="_blank" rel="noopener noreferrer" href="https://www.spotify.com/us/legal/privacy-policy/">About Ads</a></li></ul></nav><span>© <!-- -->2026<!-- --> Spotify AB</span></section></div></footer></div><script>requestAnimationFrame(function(){$RT=performance.now()});</script><script src="/_next/static/chunks/0zaia6i55l01..js" id="_R_" async=""></script><div hidden id="S:0"><div class="notification"><div class="content"></div></div><header class="header "><section class="container"><div class="container-left"><a href="/"><img alt="Engineering logo" width="90" height="36" decoding="async" data-nimg="1" style="color:transparent" src="/images/logo.png"/></a><nav class="nav "><ul><li class=""><a href="/">Blog</a></li><li class=""><a href="/podcasts">Podcasts</a></li><li class=""><a href="/opensource">Open Source</a></li><li class=""><a href="/jobs">Jobs</a></li><li class=""><a href="/about">About</a></li></ul></nav></div><div class="container-right"><div class="search "><div class="search__input"><input type="text" id="search-input" placeholder="Search" value=""/></div><div class="search__icon"><img alt="Search" id="search-icon" loading="lazy" width="24" height="24" decoding="async" data-nimg="1" class="mag" style="color:transparent" src="/images/icon-search.svg"/><img alt="Search" loading="lazy" width="24" height="24" decoding="async" data-nimg="1" class="close" style="color:transparent" src="/images/icon-close.svg"/></div><div class="menu-button "><span></span><span></span><span></span></div></div></div></section></header></div><script>$RB=[];$RV=function(a){$RT=performance.now();for(var b=0;b<a.length;b+=2){var c=a[b],e=a[b+1];null!==e.parentNode&&e.parentNode.removeChild(e);var f=c.parentNode;if(f){var g=c.previousSibling,h=0;do{if(c&&8===c.nodeType){var d=c.data;if("/$"===d||"/&"===d)if(0===h)break;else h--;else"$"!==d&&"$?"!==d&&"$~"!==d&&"$!"!==d&&"&"!==d||h++}d=c.nextSibling;f.removeChild(c);c=d}while(c);for(;e.firstChild;)f.insertBefore(e.firstChild,c);g.data="$";g._reactRetry&&requestAnimationFrame(g._reactRetry)}}a.length=0};
|
||
$RC=function(a,b){if(b=document.getElementById(b))(a=document.getElementById(a))?(a.previousSibling.data="$~",$RB.push(a,b),2===$RB.length&&("number"!==typeof $RT?requestAnimationFrame($RV.bind(null,$RB)):(a=performance.now(),setTimeout($RV.bind(null,$RB),2300>a&&2E3<a?2300-a:$RT+300-a)))):b.parentNode.removeChild(b)};$RC("B:0","S:0")</script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n3:I[39756,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\"],\"default\"]\n4:I[37457,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\"],\"default\"]\n6:I[97367,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\"],\"OutletBoundary\"]\n7:\"$Sreact.suspense\"\na:I[97367,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\"],\"ViewportBoundary\"]\nc:I[97367,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\"],\"MetadataBoundary\"]\ne:I[68027,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\"],\"default\",1]\n:HL[\"/_next/static/chunks/0sxc27dsa.k6-.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"0:{\"P\":null,\"c\":[\"\",\"2013\",\"6\",\"incident-management-at-spotify\"],\"q\":\"\",\"i\":false,\"f\":[[[\"\",{\"children\":[[\"year\",\"2013\",\"d\",null],{\"children\":[[\"month\",\"6\",\"d\",null],{\"children\":[[\"slug\",\"incident-management-at-spotify\",\"d\",null],{\"children\":[\"__PAGE__\",{}]}]}]}]},\"$undefined\",\"$undefined\",16],[[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/chunks/0sxc27dsa.k6-.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-1\",{\"src\":\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-2\",{\"src\":\"/_next/static/chunks/08c8uq~xsq31o.js\",\"async\":true,\"nonce\":\"$undefined\"}]],\"$L2\"]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[\"$L5\",[[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/0wni0lk_-haja.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"$L6\",null,{\"children\":[\"$\",\"$7\",null,{\"name\":\"Next.MetadataOutlet\",\"children\":\"$@8\"}]}]]}],{},null,false,null]},null,false,\"$@9\"]},null,false,\"$@9\"]},null,false,\"$@9\"]},null,false,null],[\"$\",\"$1\",\"h\",{\"children\":[null,[\"$\",\"$La\",null,{\"children\":\"$Lb\"}],[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$Lc\",null,{\"children\":[\"$\",\"$7\",null,{\"name\":\"Next.Metadata\",\"children\":\"$Ld\"}]}]}],null]}],false]],\"m\":\"$undefined\",\"G\":[\"$e\",[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/chunks/0sxc27dsa.k6-.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}]]],\"S\":false,\"h\":null,\"s\":\"$undefined\",\"l\":\"$undefined\",\"p\":\"$undefined\",\"d\":\"$undefined\",\"b\":\"z1ONPGug6r-HUxPkOhj0S\"}\n"])</script><script>self.__next_f.push([1,"f:[]\n9:\"$Wf\"\nb:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1, maximum-scale=1, user-scalable=yes\"}]]\n"])</script><script>self.__next_f.push([1,"10:I[79520,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\"],\"\"]\n11:I[1013,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\"],\"HeaderContextProvider\"]\n12:I[32666,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\"],\"PostProvider\"]\n13:I[28044,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\"],\"UiTextProvider\"]\n14:I[70292,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\"],\"Initializer\"]\n15:I[97341,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\"],\"default\"]\n16:I[36676,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\"],\"default\"]\n17:I[58298,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\",\"/_next/static/chunks/098votwdj80t9.js\"],\"default\"]\n18:I[22016,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\",\"/_next/static/chunks/0wni0lk_-haja.js\"],\"\"]\n"])</script><script>self.__next_f.push([1,"2:[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[[\"$\",\"head\",null,{\"children\":[\"$\",\"$L10\",null,{\"id\":\"gtm\",\"dangerouslySetInnerHTML\":{\"__html\":\"(function(w, d, s, l, i) {\\n w[l] = w[l] || [];\\n w[l].push({\\n 'gtm.start': new Date().getTime(),\\n event: 'gtm.js'\\n });\\n var f = d.getElementsByTagName(s)[0],\\n j = d.createElement(s),\\n dl = l != 'dataLayer' ? '\u0026l=' + l : '';\\n j.async = true;\\n j.src =\\n 'https://www.googletagmanager.com/gtm.js?id=' + i + dl;\\n f.parentNode.insertBefore(j, f);\\n })(window, document, 'script', 'dataLayer', 'GTM-W5NPJP7');\"}}]}],[\"$\",\"body\",null,{\"children\":[\"$\",\"div\",null,{\"className\":\"page-wrapper show-notification\",\"children\":[[\"$\",\"$L11\",null,{\"children\":[\"$\",\"$L12\",null,{\"children\":[\"$\",\"$L13\",null,{\"defaultUiText\":{\"homepageBannerText\":{\"data\":{},\"content\":[{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Read about the magic behind the music \u0026 more. Welcome to our official technology blog.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"}],\"nodeType\":\"document\"},\"podcastTagline\":{\"data\":{},\"content\":[{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Two original shows from Spotify R\u0026D.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Listen and fill your brain with great tech stories.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"}],\"nodeType\":\"document\"},\"opensourceTagline\":{\"data\":{},\"content\":[{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"There’s never been a better time to take part in open source at Spotify. We don’t just want to contribute back to the community — we want to lead the way with projects we truly believe in. Jump into one of the projects below and join us.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"}],\"nodeType\":\"document\"},\"jobsTagline\":{\"nodeType\":\"document\",\"data\":{},\"content\":[{\"nodeType\":\"paragraph\",\"data\":{},\"content\":[{\"nodeType\":\"text\",\"value\":\"Complex problems, exceptional colleagues, and a direct impact on the joy of \",\"marks\":[],\"data\":{}},{\"nodeType\":\"text\",\"value\":\"713\",\"marks\":[],\"data\":{}},{\"nodeType\":\"text\",\"value\":\" million people. Our beliefs drive our success.\",\"marks\":[],\"data\":{}}]}]},\"aboutTagline\":{\"data\":{},\"content\":[{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Spotify Engineering is a diverse community of engineers, developers, researchers and more. Together, we build the infrastructure, features, and experiences you love – and help to shape the future of audio.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"}],\"nodeType\":\"document\"}},\"children\":[\"$\",\"$L14\",null,{\"children\":[[\"$\",\"$7\",null,{\"children\":[[\"$\",\"$L15\",null,{}],[\"$\",\"$L16\",null,{}]]}],[\"$\",\"$L3\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$17\",\"errorStyles\":[],\"errorScripts\":[[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/chunks/098votwdj80t9.js\",\"async\":true}]],\"template\":[\"$\",\"$L4\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"div\",null,{\"className\":\"not-found\",\"children\":[\"$\",\"div\",null,{\"className\":\"container\",\"children\":[[\"$\",\"h1\",null,{\"children\":\"404\"}],[\"$\",\"div\",null,{\"className\":\"content\",\"children\":[[\"$\",\"h3\",null,{\"children\":\"OOOOPS... Something went wrong\"}],[\"$\",\"p\",null,{\"children\":\"We couldn't find the page you're looking for.\"}]]}],[\"$\",\"$L18\",null,{\"href\":\"/\",\"className\":\"button\",\"children\":\"Go back home\"}]]}]}],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}]}]}]}],[\"$\",\"footer\",null,{\"className\":\"footer\",\"children\":[\"$\",\"div\",null,{\"className\":\"container\",\"children\":[[\"$\",\"section\",null,{\"className\":\"subscription\",\"children\":[[\"$\",\"div\",null,{\"className\":\"subscription__left\",\"children\":[[\"$\",\"h3\",null,{\"children\":\"Want to stay in the loop?\"}],[\"$\",\"p\",null,{\"children\":\"Subscribe to our newsletter to never miss an update!\"}],[\"$\",\"small\",null,{\"children\":\"By clicking sign up you’ll receive occasional emails from Spotify. You always have the choice to adjust your interest settings or unsubscribe.\"}]]}],[\"$\",\"div\",null,{\"className\":\"subscription__right\",\"children\":[\"$\",\"div\",null,{\"id\":\"mc_embed_signup\",\"children\":[\"$\",\"form\",null,{\"action\":\"https://spotify.us20.list-manage.com/subscribe/post?u=99aa88d3fba458716d0cd1299\u0026id=28cc680980\",\"method\":\"post\",\"id\":\"mc-embedded-subscribe-form\",\"name\":\"mc-embedded-subscribe-form\",\"className\":\"validate\",\"target\":\"_blank\",\"children\":\"$L19\"}]}]}]]}],\"$L1a\",\"$L1b\"]}]}]]}]}]]}]\n"])</script><script>self.__next_f.push([1,"1c:I[5500,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\",\"/_next/static/chunks/0wni0lk_-haja.js\"],\"Image\"]\n19:[\"$\",\"div\",null,{\"id\":\"mc_embed_signup_scroll\",\"children\":[[\"$\",\"div\",null,{\"className\":\"mc-field-group\",\"children\":[[\"$\",\"input\",null,{\"type\":\"email\",\"name\":\"EMAIL\",\"className\":\"required email\",\"id\":\"mce-EMAIL\",\"placeholder\":\"Enter your best email\"}],[\"$\",\"input\",null,{\"type\":\"submit\",\"name\":\"subscribe\",\"value\":\"Subscribe\",\"id\":\"mc-embedded-subscribe\",\"className\":\"button button--submit\"}]]}],[\"$\",\"div\",null,{\"id\":\"mce-responses\",\"className\":\"clear\",\"children\":[[\"$\",\"div\",null,{\"className\":\"response\",\"id\":\"mce-error-response\",\"style\":{\"display\":\"none\"}}],[\"$\",\"div\",null,{\"className\":\"response\",\"id\":\"mce-success-response\",\"style\":{\"display\":\"none\"}}]]}],[\"$\",\"div\",null,{\"style\":{\"left\":\"-5000px\",\"position\":\"absolute\"},\"aria-hidden\":\"true\",\"children\":[\"$\",\"input\",null,{\"type\":\"text\",\"name\":\"b_99aa88d3fba458716d0cd1299_28cc680980\"}]}]]}]\n"])</script><script>self.__next_f.push([1,"1a:[\"$\",\"section\",null,{\"className\":\"links\",\"children\":[[\"$\",\"div\",null,{\"className\":\"links__left\",\"children\":[[\"$\",\"div\",null,{\"className\":\"logo\",\"children\":[\"$\",\"$L18\",null,{\"href\":\"/\",\"children\":[\"$\",\"$L1c\",null,{\"src\":\"/images/footer-logo.svg\",\"alt\":\"feed\",\"width\":167,\"height\":50}]}]}],[\"$\",\"div\",null,{\"className\":\"nav-wrapper\",\"children\":[\"$\",\"nav\",null,{\"className\":\"nav\",\"children\":[[\"$\",\"ul\",null,{\"children\":[[\"$\",\"li\",null,{\"children\":[\"$\",\"$L18\",null,{\"href\":\"https://spotify.com/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Spotify.com\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L18\",null,{\"href\":\"https://www.lifeatspotify.com/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Spotify Jobs\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L18\",null,{\"href\":\"https://newsroom.spotify.com/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Newsroom\"}]}]]}],[\"$\",\"ul\",null,{\"children\":[[\"$\",\"li\",null,{\"children\":[\"$\",\"$L18\",null,{\"href\":\"https://research.atspotify.com/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Spotify R\u0026D Research\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L18\",null,{\"href\":\"https://backstage.spotify.com/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Spotify for Backstage\"}]}]]}]]}]}]]}],[\"$\",\"div\",null,{\"className\":\"links__right\",\"children\":[\"$\",\"ul\",null,{\"className\":\"social\",\"children\":[[\"$\",\"li\",null,{\"children\":[\"$\",\"$L18\",null,{\"href\":\"https://www.linkedin.com/showcase/spotify-r\u0026d\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"$\",\"$L1c\",null,{\"src\":\"/images/linked-in.svg\",\"alt\":\"Linked In\",\"width\":24,\"height\":24,\"priority\":true}]}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L18\",null,{\"href\":\"https://twitter.com/spotifyeng\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"$\",\"$L1c\",null,{\"src\":\"/images/x.svg\",\"alt\":\"X\",\"width\":24,\"height\":24,\"priority\":true}]}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L18\",null,{\"href\":\"https://www.youtube.com/@SpotifyRnD\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"$\",\"$L1c\",null,{\"src\":\"/images/yt.svg\",\"alt\":\"Youtube\",\"width\":24,\"height\":24,\"priority\":true}]}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L18\",null,{\"href\":\"http://www.instagram.com/lifeatspotify\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"$\",\"$L1c\",null,{\"src\":\"/images/insta.svg\",\"alt\":\"Instagram\",\"width\":24,\"height\":24,\"priority\":true}]}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L18\",null,{\"href\":\"https://www.facebook.com/spotify\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"$\",\"$L1c\",null,{\"src\":\"/images/f.svg\",\"alt\":\"facebook\",\"width\":24,\"height\":24,\"priority\":true}]}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L18\",null,{\"href\":\"/feed\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"$\",\"$L1c\",null,{\"src\":\"/images/feed.svg\",\"alt\":\"feed\",\"width\":24,\"height\":24,\"priority\":true}]}]}]]}]}]]}]\n"])</script><script>self.__next_f.push([1,"1b:[\"$\",\"section\",null,{\"className\":\"copy\",\"children\":[[\"$\",\"nav\",null,{\"className\":\"copy__nav\",\"children\":[\"$\",\"ul\",null,{\"children\":[[\"$\",\"li\",null,{\"children\":[\"$\",\"$L18\",null,{\"href\":\"https://www.spotify.com/us/legal/end-user-agreement/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Legal\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L18\",null,{\"href\":\"https://www.spotify.com/us/legal/privacy-policy/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Privacy\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L18\",null,{\"href\":\"https://www.spotify.com/legal/cookies-policy/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Cookies\"}]}],[\"$\",\"li\",null,{\"children\":[\"$\",\"$L18\",null,{\"href\":\"https://www.spotify.com/us/legal/privacy-policy/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"About Ads\"}]}]]}]}],[\"$\",\"span\",null,{\"children\":[\"© \",2026,\" Spotify AB\"]}]]}]\n"])</script><script>self.__next_f.push([1,"1d:I[27201,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\"],\"IconMark\"]\n8:null\n"])</script><script>self.__next_f.push([1,"d:[[\"$\",\"title\",\"0\",{\"children\":\"Incident Management at Spotify | Spotify Engineering\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"A few weeks ago Spotify had one of the biggest incidents in the last few years. It caused a major outage for a big chunk of our European users. For a few hours the music playback experience was damaged. Our users would see high latency when playing music and some of them were unable to log in.\"}],[\"$\",\"meta\",\"2\",{\"name\":\"application-name\",\"content\":\"Spotify Engineering\"}],[\"$\",\"meta\",\"3\",{\"name\":\"author\",\"content\":\"David Poblador i Garcia\"}],[\"$\",\"meta\",\"4\",{\"name\":\"robots\",\"content\":\"index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1\"}],[\"$\",\"meta\",\"5\",{\"name\":\"twitter:data1\",\"content\":\"David Poblador i Garcia\"}],[\"$\",\"meta\",\"6\",{\"name\":\"twitter:data2\",\"content\":\"6 minutes\"}],[\"$\",\"meta\",\"7\",{\"name\":\"twitter:label1\",\"content\":\"Written by\"}],[\"$\",\"meta\",\"8\",{\"name\":\"twitter:label2\",\"content\":\"Est. reading time\"}],[\"$\",\"meta\",\"9\",{\"property\":\"og:title\",\"content\":\"Incident Management at Spotify | Spotify Engineering\"}],[\"$\",\"meta\",\"10\",{\"property\":\"og:description\",\"content\":\"A few weeks ago Spotify had one of the biggest incidents in the last few years. It caused a major outage for a big chunk of our European users. For a few hours the music playback experience was damaged. Our users would see high latency when playing music and some of them were unable to log in.\"}],[\"$\",\"meta\",\"11\",{\"property\":\"og:site_name\",\"content\":\"Spotify Engineering\"}],[\"$\",\"meta\",\"12\",{\"property\":\"og:image\",\"content\":\"https://images.ctfassets.net/p762jor363g1/attachment_943e369d8982615e731b91feed9abde7/360a9dc4dcc43513c3d502072fd607bb/attachment_943e369d8982615e731b91feed9abde7.png\"}],[\"$\",\"meta\",\"13\",{\"property\":\"og:image:type\",\"content\":\"image/jpeg\"}],[\"$\",\"meta\",\"14\",{\"property\":\"og:image:width\",\"content\":\"1200\"}],[\"$\",\"meta\",\"15\",{\"property\":\"og:image:height\",\"content\":\"630\"}],[\"$\",\"meta\",\"16\",{\"property\":\"og:image:alt\",\"content\":\"featured image\"}],[\"$\",\"meta\",\"17\",{\"property\":\"og:type\",\"content\":\"article\"}],[\"$\",\"meta\",\"18\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"19\",{\"name\":\"twitter:creator\",\"content\":\"David Poblador i Garcia\"}],[\"$\",\"meta\",\"20\",{\"name\":\"twitter:title\",\"content\":\"Incident Management at Spotify | Spotify Engineering\"}],[\"$\",\"meta\",\"21\",{\"name\":\"twitter:description\",\"content\":\"Incident Management at Spotify | Spotify Engineering\"}],[\"$\",\"meta\",\"22\",{\"name\":\"twitter:image\",\"content\":\"https://images.ctfassets.net/p762jor363g1/attachment_943e369d8982615e731b91feed9abde7/360a9dc4dcc43513c3d502072fd607bb/attachment_943e369d8982615e731b91feed9abde7.png\"}],[\"$\",\"meta\",\"23\",{\"name\":\"twitter:image:alt\",\"content\":\"featured image\"}],[\"$\",\"meta\",\"24\",{\"name\":\"twitter:image:type\",\"content\":\"image/jpeg\"}],[\"$\",\"meta\",\"25\",{\"name\":\"twitter:image:width\",\"content\":\"1200\"}],[\"$\",\"meta\",\"26\",{\"name\":\"twitter:image:height\",\"content\":\"630\"}],[\"$\",\"link\",\"27\",{\"rel\":\"icon\",\"href\":\"/favicon.ico?favicon.0cd2g1ew_zd4v.ico\",\"sizes\":\"250x250\",\"type\":\"image/x-icon\"}],[\"$\",\"$L1d\",\"28\",{}]]\n"])</script><script>self.__next_f.push([1,"1e:T2075,"])</script><script>self.__next_f.push([1,"\u003cp \u003eA few weeks ago Spotify had one of the biggest incidents in the last few years. It caused a major outage for a big chunk of our European users. For a few hours the music playback experience was damaged. Our users would see high latency when playing music and some of them were unable to log in.\u003c/p\u003e\u003cp \u003eAt Spotify we take these types of incidents seriously. Our mission when dealing with outages is to avoid having the same incident twice. We want to be able to learn something new with each incident. However, sometimes we fail.\u003c/p\u003e\u003ch2\u003eFirst symptoms: two months before the outage\u003c/h2\u003e\u003cp \u003eTwo months before the big outage we had an incident connected with one of our smallest backend services: Popcount. Popcount (this is our internal name) is the service that takes care of storing the list of subscribers for each of our more than 1 billion playlists.\u003c/p\u003e\u003cp \u003eAll our backend services and clients that communicate with Popcount are designed to fail fast in the event that the service is not available or too slow. Failing fast is key to scaling. We try to build our systems in such a way that partial failures are handled gracefully. For instance, it is better not to display the list of playlist subscribers for a while than to delay showing playlist data for a long time, completely damaging the user experience.\u003c/p\u003e\u003cp \u003eHowever, there was a piece of legacy code in our desktop client that did not honor that requirement. That particular code caused our clients to retry fetching Popcount information for every request that timed out. To make matters worse, the legacy implementation had no backoff logic at all. This put a lot of pressure on the Popcount service and made it reach a state where servers were overwhelmed in such a way that recovery was almost impossible: the slower the service became, the more requests it started receiving. Turning it off wouldn’t have helped either: the amount of pending requests coming from our clients was too big to be served without damaging the service again.\u003c/p\u003e\u003cp \u003eThe Popcount developers quickly deployed a fix: in the event that the service became slow, we would fast fail and return an empty list of subscribers. With that fix in place, our Popcount system started making forward progress and very soon our clients went back to normal.\u003c/p\u003e\u003ch2\u003eRemediations: why are they important?\u003c/h2\u003e\u003cp \u003eFor every incident we hold a post-mortem meeting. The goal of this meeting is producing a list of remediations that we need to put in place in order to prevent that incident from happening again and assign an urgency to each of them. We invite all the teams that are connected with a system involved in the given incident. If the severity of an incident is minor or it is connected with a very isolated part of our infrastructure, the post-mortem meetings are limited to a few key engineers. Post-mortem meetings are not about blame, they are about improving our systems and our processes!\u003c/p\u003e\u003cp \u003eOne of the remediations for the Popcount incident was fixing the faulty retry behavior in our desktop client when fetching data from the Popcount system.\u003c/p\u003e\u003cp \u003eDistributed systems are hard, prioritizing bugs and assigning the right urgency is often non-obvious. There are many challenges we face when we release software to millions of users, and sometimes assessing the relative urgency of a fix is not easy. In this particular case, we focussed in the mitigations (fast fail behavior in Popcount) instead of in the root cause (the client bug).\u003c/p\u003e\u003cp \u003eWe did not set the proper priority for the bugfix, and, guess what…\u003c/p\u003e\u003ch2\u003eThe night of April 27th\u003c/h2\u003e\u003cp \u003eIt was a rainy Saturday night in Stockholm. Some of our engineers were online on IRC, and discovered that Popcount was unhealthy again. We thought the fast fail logic we introduced in Popcount a few weeks before would prevent Popcount from dying again.\u003c/p\u003e\n \u003cfigure class=\"figure-image\"\u003e\n \n \u003cimg \n src=\"//images.ctfassets.net/p762jor363g1/attachment_c8b02a41e754498534e445cbbadb9156/1b91c5dafa01f39b4f08837f387ad040/attachment_c8b02a41e754498534e445cbbadb9156.png\" \n style=\"width: 100%; max-width: 100%; height: auto; \"\n class=\"blog-image\"\n alt=\"attachment_c8b02a41e754498534e445cbbadb9156\"\n /\u003e\n \n \u003cfigcaption class=\"caption\"\u003e\u003cp\u003e\u003ci\u003eRequest delay for Popcount in one of our Datacenters\u003c/i\u003e\u003c/p\u003e\u003c/figcaption\u003e\n \u003c/figure\u003e\n \u003cp \u003eWhat happened? Why was Popcount being hammered with requests again? A few days earlier we started rolling out our new cool Discovery feature to a percentage of our users. We had unknowingly added a dependency between our Discovery service (Bartender is our internal name) and Popcount. Some of the content we display in our discovery page display playlist counts, hence the Bartender service was fetching data directly from Popcount. The fast fail logic we introduced before did not help much: the service was getting more requests than it could handle. We realized too late that Popcount was underprovisioned.\u003c/p\u003e\u003cp \u003eOur Site Reliability and Infrastructure teams thought that removing that dependency via a hotfix would solve the problem. But we were wrong.\u003c/p\u003e\u003cp \u003eBy the time we applied the fix, the Popcount request queues in our Accesspoints (the servers our clients connect to) had built up and some excessive logging in the Accesspoints making them die like flies. We intentionally had added more verbose logging of errors for debugging purposes, and we managed to shoot ourselves in the foot with it. Their I/O responsiveness was nonexistent and the wrong retry behavior in our desktop clients was making things worse and worse. Saturday night is usually a very busy night for our servers, and that did not help either.\u003c/p\u003e\u003cp \u003eBy that time our service was notably degraded and we were in a situation where most of our Accesspoints were unreachable or responding extremely slowly. It was time for drastic measures.\u003c/p\u003e\u003cp \u003eWe needed to do something to allow them to start breathing and the only possible solution was firewalling them off. Some of them were so unresponsive that we had to perform a hard reset before applying the necessary firewall rules. A few minutes later, things were responsive again. We removed the firewall blocks and the service started working smoothly as usual.\u003c/p\u003e\u003cp \u003eWhy did that help? Well, our clients have a nice retry behavior when they are disconnected from the Accesspoints. They have an exponential backoff logic and offlining those servers for a while triggered the backoff that helped us make forward progress.\u003c/p\u003e\u003ch2\u003eLessons learned\u003c/h2\u003e\u003cul\u003e\u003cli\u003e\u003cp \u003eWe should have prioritized fixing the backoff bug in the client after the first Popcount incident, we did not assign the right severity. We thought having the fast fail mechanism in place would be enough to keep the service up in case we experienced high latencies.\u003c/p\u003e\u003c/li\u003e\u003cli\u003e\u003cp \u003eLogging is nice, but logging too much can be damaging when many error messages or warnings are produced.\u003c/p\u003e\u003c/li\u003e\u003cli\u003e\u003cp \u003eWe should spend more time testing extreme cases (high latency, low bandwidth) for our backend systems\u003c/p\u003e\u003c/li\u003e\u003c/ul\u003e\u003cp \u003eSometimes, things you do to try to understand problems better (adding extra logging, to name one) can really come back to bite you.\u003c/p\u003e\u003ch2\u003eRemediations in place after the second incident\u003c/h2\u003e\u003cp \u003eAfter the outage (infamously known as OPS-6000 in our incident tracking system), we held a post-mortem meeting where we invited the entire Technology and Product departments. Many people had the opportunity to attend a post-mortem meeting for the first time.\u003c/p\u003e\u003cp \u003eWe came up with a list of remediations, all of them are in place now. Some of them:\u003c/p\u003e\u003cul\u003e\u003cli\u003e\u003cp \u003eAll the data served by our discovery service should be statically cached with a TTL.\u003c/p\u003e\u003c/li\u003e\u003cli\u003e\u003cp \u003eFix the faulty retry behavior for Popcount in our desktop client.\u003c/p\u003e\u003c/li\u003e\u003cli\u003e\u003cp \u003eMove valuable Accesspoint log messages into a separate log. In case we hit a similar IO problem in the future, we can easily discard non important messages. We also introduced rate limiting for those messages.\u003c/p\u003e\u003c/li\u003e\u003cli\u003e\u003cp \u003eImplement some syslog fixes to prevent too frequent flushes to disk.\u003c/p\u003e\u003c/li\u003e\u003c/ul\u003e\u003cp \u003eEducating the company on why and how we failed is just as important as the fixes (it was even a remediation for this incident).\u003c/p\u003e"])</script><script>self.__next_f.push([1,"5:[[\"$\",\"div\",null,{\"className\":\"single-post\",\"children\":[[\"$\",\"div\",null,{\"className\":\"container\",\"children\":[\"$\",\"h1\",null,{\"className\":\"single-post__title\",\"children\":\"Incident Management at Spotify\"}]}],[\"$\",\"div\",null,{\"className\":\"container\",\"children\":[[\"$\",\"div\",null,{\"className\":\"single-post__content\",\"children\":[[\"$\",\"div\",null,{\"className\":\"single-post__featured-image\",\"children\":[\"$\",\"$L1c\",null,{\"src\":\"https://images.ctfassets.net/p762jor363g1/attachment_943e369d8982615e731b91feed9abde7/360a9dc4dcc43513c3d502072fd607bb/attachment_943e369d8982615e731b91feed9abde7.png\",\"alt\":\"Feature Image\",\"width\":740,\"height\":404,\"priority\":true}]}],[\"$\",\"div\",null,{\"className\":\"html-content\",\"dangerouslySetInnerHTML\":{\"__html\":\"$1e\"}}],\"$L1f\",\"$L20\"]}],\"$L21\"]}]]}],\"$L22\"]\n"])</script><script>self.__next_f.push([1,"23:I[57581,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\",\"/_next/static/chunks/0wni0lk_-haja.js\"],\"PostTags\"]\n24:I[58931,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\",\"/_next/static/chunks/0wni0lk_-haja.js\"],\"SocialShare\"]\n25:I[40367,[\"/_next/static/chunks/0a_f9~n73ra-c.js\",\"/_next/static/chunks/0d3shmwh5_nmn.js\",\"/_next/static/chunks/08c8uq~xsq31o.js\",\"/_next/static/chunks/0wni0lk_-haja.js\"],\"default\"]\n"])</script><script>self.__next_f.push([1,"1f:[\"$\",\"$L23\",null,{\"tags\":[{\"metadata\":{\"tags\":[],\"concepts\":[]},\"sys\":{\"space\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"Space\",\"id\":\"p762jor363g1\"}},\"id\":\"blogtag_incident\",\"type\":\"Entry\",\"createdAt\":\"2025-03-13T13:25:40.421Z\",\"updatedAt\":\"2025-03-13T13:25:40.421Z\",\"environment\":{\"sys\":{\"id\":\"spotify-engineering\",\"type\":\"Link\",\"linkType\":\"Environment\"}},\"publishedVersion\":2,\"revision\":1,\"contentType\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"ContentType\",\"id\":\"blogTag\"}},\"locale\":\"en-US\"},\"fields\":{\"name\":\"incident\",\"slug\":\"incident\"}},{\"metadata\":{\"tags\":[],\"concepts\":[]},\"sys\":{\"space\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"Space\",\"id\":\"p762jor363g1\"}},\"id\":\"blogtag_incident-management\",\"type\":\"Entry\",\"createdAt\":\"2025-03-13T13:25:42.911Z\",\"updatedAt\":\"2025-03-13T13:25:42.911Z\",\"environment\":{\"sys\":{\"id\":\"spotify-engineering\",\"type\":\"Link\",\"linkType\":\"Environment\"}},\"publishedVersion\":2,\"revision\":1,\"contentType\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"ContentType\",\"id\":\"blogTag\"}},\"locale\":\"en-US\"},\"fields\":{\"name\":\"incident management\",\"slug\":\"incident-management\"}},{\"metadata\":{\"tags\":[],\"concepts\":[]},\"sys\":{\"space\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"Space\",\"id\":\"p762jor363g1\"}},\"id\":\"blogtag_outage\",\"type\":\"Entry\",\"createdAt\":\"2025-03-13T13:26:50.070Z\",\"updatedAt\":\"2025-03-13T13:26:50.070Z\",\"environment\":{\"sys\":{\"id\":\"spotify-engineering\",\"type\":\"Link\",\"linkType\":\"Environment\"}},\"publishedVersion\":2,\"revision\":1,\"contentType\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"ContentType\",\"id\":\"blogTag\"}},\"locale\":\"en-US\"},\"fields\":{\"name\":\"outage\",\"slug\":\"outage\"}},{\"metadata\":{\"tags\":[],\"concepts\":[]},\"sys\":{\"space\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"Space\",\"id\":\"p762jor363g1\"}},\"id\":\"blogtag_scalability\",\"type\":\"Entry\",\"createdAt\":\"2025-03-13T13:27:52.113Z\",\"updatedAt\":\"2025-03-13T13:27:52.113Z\",\"environment\":{\"sys\":{\"id\":\"spotify-engineering\",\"type\":\"Link\",\"linkType\":\"Environment\"}},\"publishedVersion\":2,\"revision\":1,\"contentType\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"ContentType\",\"id\":\"blogTag\"}},\"locale\":\"en-US\"},\"fields\":{\"name\":\"scalability\",\"slug\":\"scalability\"}},{\"metadata\":{\"tags\":[],\"concepts\":[]},\"sys\":{\"space\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"Space\",\"id\":\"p762jor363g1\"}},\"id\":\"blogtag_sre\",\"type\":\"Entry\",\"createdAt\":\"2025-03-13T13:28:13.232Z\",\"updatedAt\":\"2025-03-13T13:28:13.232Z\",\"environment\":{\"sys\":{\"id\":\"spotify-engineering\",\"type\":\"Link\",\"linkType\":\"Environment\"}},\"publishedVersion\":2,\"revision\":1,\"contentType\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"ContentType\",\"id\":\"blogTag\"}},\"locale\":\"en-US\"},\"fields\":{\"name\":\"sre\",\"slug\":\"sre\"}},{\"metadata\":{\"tags\":[],\"concepts\":[]},\"sys\":{\"space\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"Space\",\"id\":\"p762jor363g1\"}},\"id\":\"blogtag_systems\",\"type\":\"Entry\",\"createdAt\":\"2025-03-13T13:28:20.761Z\",\"updatedAt\":\"2025-03-13T13:28:20.761Z\",\"environment\":{\"sys\":{\"id\":\"spotify-engineering\",\"type\":\"Link\",\"linkType\":\"Environment\"}},\"publishedVersion\":2,\"revision\":1,\"contentType\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"ContentType\",\"id\":\"blogTag\"}},\"locale\":\"en-US\"},\"fields\":{\"name\":\"systems\",\"slug\":\"systems\"}}]}]\n"])</script><script>self.__next_f.push([1,"20:[\"$\",\"$L24\",null,{\"heading\":\"SHARE THIS ARTICLE\",\"title\":\"Incident Management at Spotify\"}]\n"])</script><script>self.__next_f.push([1,"21:[\"$\",\"$L25\",null,{\"blogPost\":{\"metadata\":{\"tags\":[],\"concepts\":[]},\"sys\":{\"space\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"Space\",\"id\":\"p762jor363g1\"}},\"id\":\"blogpost_2dd58151ed36d6f68692055d1e5d18c5\",\"type\":\"Entry\",\"createdAt\":\"2025-03-13T22:25:37.810Z\",\"updatedAt\":\"2025-04-24T15:30:30.819Z\",\"environment\":{\"sys\":{\"id\":\"spotify-engineering\",\"type\":\"Link\",\"linkType\":\"Environment\"}},\"publishedVersion\":6,\"revision\":3,\"contentType\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"ContentType\",\"id\":\"blogPost\"}},\"locale\":\"en-US\"},\"fields\":{\"title\":\"Incident Management at Spotify\",\"slug\":\"incident-management-at-spotify\",\"content\":{\"data\":{},\"content\":[{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"A few weeks ago Spotify had one of the biggest incidents in the last few years. It caused a major outage for a big chunk of our European users. For a few hours the music playback experience was damaged. Our users would see high latency when playing music and some of them were unable to log in.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"At Spotify we take these types of incidents seriously. Our mission when dealing with outages is to avoid having the same incident twice. We want to be able to learn something new with each incident. However, sometimes we fail.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"First symptoms: two months before the outage\",\"nodeType\":\"text\"}],\"nodeType\":\"heading-2\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Two months before the big outage we had an incident connected with one of our smallest backend services: Popcount. Popcount (this is our internal name) is the service that takes care of storing the list of subscribers for each of our more than 1 billion playlists.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"All our backend services and clients that communicate with Popcount are designed to fail fast in the event that the service is not available or too slow. Failing fast is key to scaling. We try to build our systems in such a way that partial failures are handled gracefully. For instance, it is better not to display the list of playlist subscribers for a while than to delay showing playlist data for a long time, completely damaging the user experience.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"However, there was a piece of legacy code in our desktop client that did not honor that requirement. That particular code caused our clients to retry fetching Popcount information for every request that timed out. To make matters worse, the legacy implementation had no backoff logic at all. This put a lot of pressure on the Popcount service and made it reach a state where servers were overwhelmed in such a way that recovery was almost impossible: the slower the service became, the more requests it started receiving. Turning it off wouldn’t have helped either: the amount of pending requests coming from our clients was too big to be served without damaging the service again.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"The Popcount developers quickly deployed a fix: in the event that the service became slow, we would fast fail and return an empty list of subscribers. With that fix in place, our Popcount system started making forward progress and very soon our clients went back to normal.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Remediations: why are they important?\",\"nodeType\":\"text\"}],\"nodeType\":\"heading-2\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"For every incident we hold a post-mortem meeting. The goal of this meeting is producing a list of remediations that we need to put in place in order to prevent that incident from happening again and assign an urgency to each of them. We invite all the teams that are connected with a system involved in the given incident. If the severity of an incident is minor or it is connected with a very isolated part of our infrastructure, the post-mortem meetings are limited to a few key engineers. Post-mortem meetings are not about blame, they are about improving our systems and our processes!\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"One of the remediations for the Popcount incident was fixing the faulty retry behavior in our desktop client when fetching data from the Popcount system.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Distributed systems are hard, prioritizing bugs and assigning the right urgency is often non-obvious. There are many challenges we face when we release software to millions of users, and sometimes assessing the relative urgency of a fix is not easy. In this particular case, we focussed in the mitigations (fast fail behavior in Popcount) instead of in the root cause (the client bug).\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"We did not set the proper priority for the bugfix, and, guess what…\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"The night of April 27th\",\"nodeType\":\"text\"}],\"nodeType\":\"heading-2\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"It was a rainy Saturday night in Stockholm. Some of our engineers were online on IRC, and discovered that Popcount was unhealthy again. We thought the fast fail logic we introduced in Popcount a few weeks before would prevent Popcount from dying again.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{\"target\":{\"metadata\":{\"tags\":[],\"concepts\":[]},\"sys\":{\"space\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"Space\",\"id\":\"p762jor363g1\"}},\"id\":\"assetwrapper_69ff8be069ed4f3ee288997be0ce7768\",\"type\":\"Entry\",\"createdAt\":\"2025-03-13T14:43:42.010Z\",\"updatedAt\":\"2025-03-13T14:43:42.010Z\",\"environment\":{\"sys\":{\"id\":\"spotify-engineering\",\"type\":\"Link\",\"linkType\":\"Environment\"}},\"publishedVersion\":2,\"revision\":1,\"contentType\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"ContentType\",\"id\":\"assetWrapper\"}},\"locale\":\"en-US\"},\"fields\":{\"title\":\"attachment_c8b02a41e754498534e445cbbadb9156\",\"asset\":{\"metadata\":{\"tags\":[],\"concepts\":[]},\"sys\":{\"space\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"Space\",\"id\":\"p762jor363g1\"}},\"id\":\"attachment_c8b02a41e754498534e445cbbadb9156\",\"type\":\"Asset\",\"createdAt\":\"2025-03-13T13:54:22.984Z\",\"updatedAt\":\"2025-03-13T13:54:22.984Z\",\"environment\":{\"sys\":{\"id\":\"spotify-engineering\",\"type\":\"Link\",\"linkType\":\"Environment\"}},\"publishedVersion\":3,\"revision\":1,\"locale\":\"en-US\"},\"fields\":{\"title\":\"attachment_c8b02a41e754498534e445cbbadb9156\",\"file\":{\"url\":\"//images.ctfassets.net/p762jor363g1/attachment_c8b02a41e754498534e445cbbadb9156/1b91c5dafa01f39b4f08837f387ad040/attachment_c8b02a41e754498534e445cbbadb9156.png\",\"details\":{\"size\":35946,\"image\":{\"width\":697,\"height\":491}},\"fileName\":\"attachment_c8b02a41e754498534e445cbbadb9156.png\",\"contentType\":\"image/png\"}}},\"caption\":{\"nodeType\":\"document\",\"data\":{},\"content\":[{\"nodeType\":\"paragraph\",\"content\":[{\"nodeType\":\"text\",\"value\":\"Request delay for Popcount in one of our Datacenters\",\"marks\":[{\"type\":\"italic\"}],\"data\":{}}],\"data\":{}}]},\"alt\":\"Request delay for Popcount in one of our Datacenters\",\"width\":\"None\",\"height\":\"None\",\"type\":\"image\",\"alignment\":\"left\",\"raw_html\":\"\u003cimg alt=\\\"Request delay for Popcount in one of our Datacenters\\\" class=\\\"wp-image-347\\\" decoding=\\\"async\\\" src=\\\"https://storage.googleapis.com/production-eng/1/2013/06/lon-1.png?w=584\\\"/\u003e\",\"html_attributes\":{\"decoding\":\"async\",\"src\":\"https://storage.googleapis.com/production-eng/1/2013/06/lon-1.png?w=584\",\"alt\":\"Request delay for Popcount in one of our Datacenters\",\"class\":[\"wp-image-347\"]}}}},\"content\":[],\"nodeType\":\"embedded-entry-block\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"What happened? Why was Popcount being hammered with requests again? A few days earlier we started rolling out our new cool Discovery feature to a percentage of our users. We had unknowingly added a dependency between our Discovery service (Bartender is our internal name) and Popcount. Some of the content we display in our discovery page display playlist counts, hence the Bartender service was fetching data directly from Popcount. The fast fail logic we introduced before did not help much: the service was getting more requests than it could handle. We realized too late that Popcount was underprovisioned.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Our Site Reliability and Infrastructure teams thought that removing that dependency via a hotfix would solve the problem. But we were wrong.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"By the time we applied the fix, the Popcount request queues in our Accesspoints (the servers our clients connect to) had built up and some excessive logging in the Accesspoints making them die like flies. We intentionally had added more verbose logging of errors for debugging purposes, and we managed to shoot ourselves in the foot with it. Their I/O responsiveness was nonexistent and the wrong retry behavior in our desktop clients was making things worse and worse. Saturday night is usually a very busy night for our servers, and that did not help either.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"By that time our service was notably degraded and we were in a situation where most of our Accesspoints were unreachable or responding extremely slowly. It was time for drastic measures.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"We needed to do something to allow them to start breathing and the only possible solution was firewalling them off. Some of them were so unresponsive that we had to perform a hard reset before applying the necessary firewall rules. A few minutes later, things were responsive again. We removed the firewall blocks and the service started working smoothly as usual.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Why did that help? Well, our clients have a nice retry behavior when they are disconnected from the Accesspoints. They have an exponential backoff logic and offlining those servers for a while triggered the backoff that helped us make forward progress.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Lessons learned\",\"nodeType\":\"text\"}],\"nodeType\":\"heading-2\"},{\"data\":{},\"content\":[{\"data\":{},\"content\":[{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"We should have prioritized fixing the backoff bug in the client after the first Popcount incident, we did not assign the right severity. We thought having the fast fail mechanism in place would be enough to keep the service up in case we experienced high latencies.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"}],\"nodeType\":\"list-item\"},{\"data\":{},\"content\":[{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Logging is nice, but logging too much can be damaging when many error messages or warnings are produced.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"}],\"nodeType\":\"list-item\"},{\"data\":{},\"content\":[{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"We should spend more time testing extreme cases (high latency, low bandwidth) for our backend systems\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"}],\"nodeType\":\"list-item\"}],\"nodeType\":\"unordered-list\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Sometimes, things you do to try to understand problems better (adding extra logging, to name one) can really come back to bite you.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Remediations in place after the second incident\",\"nodeType\":\"text\"}],\"nodeType\":\"heading-2\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"After the outage (infamously known as OPS-6000 in our incident tracking system), we held a post-mortem meeting where we invited the entire Technology and Product departments. Many people had the opportunity to attend a post-mortem meeting for the first time.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"We came up with a list of remediations, all of them are in place now. Some of them:\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"},{\"data\":{},\"content\":[{\"data\":{},\"content\":[{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"All the data served by our discovery service should be statically cached with a TTL.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"}],\"nodeType\":\"list-item\"},{\"data\":{},\"content\":[{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Fix the faulty retry behavior for Popcount in our desktop client.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"}],\"nodeType\":\"list-item\"},{\"data\":{},\"content\":[{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Move valuable Accesspoint log messages into a separate log. In case we hit a similar IO problem in the future, we can easily discard non important messages. We also introduced rate limiting for those messages.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"}],\"nodeType\":\"list-item\"},{\"data\":{},\"content\":[{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Implement some syslog fixes to prevent too frequent flushes to disk.\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"}],\"nodeType\":\"list-item\"}],\"nodeType\":\"unordered-list\"},{\"data\":{},\"content\":[{\"data\":{},\"marks\":[],\"value\":\"Educating the company on why and how we failed is just as important as the fixes (it was even a remediation for this incident).\",\"nodeType\":\"text\"}],\"nodeType\":\"paragraph\"}],\"nodeType\":\"document\"},\"categories\":[{\"metadata\":{\"tags\":[],\"concepts\":[]},\"sys\":{\"space\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"Space\",\"id\":\"p762jor363g1\"}},\"id\":\"blogcategory_data\",\"type\":\"Entry\",\"createdAt\":\"2025-03-13T13:22:21.529Z\",\"updatedAt\":\"2025-05-26T09:52:46.244Z\",\"environment\":{\"sys\":{\"id\":\"spotify-engineering\",\"type\":\"Link\",\"linkType\":\"Environment\"}},\"publishedVersion\":4,\"revision\":2,\"contentType\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"ContentType\",\"id\":\"blogCategory\"}},\"locale\":\"en-US\"},\"fields\":{\"name\":\"Data\",\"slug\":\"data\",\"has_blogs\":true}},{\"metadata\":{\"tags\":[],\"concepts\":[]},\"sys\":{\"space\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"Space\",\"id\":\"p762jor363g1\"}},\"id\":\"blogcategory_infrastructure\",\"type\":\"Entry\",\"createdAt\":\"2025-03-13T13:22:31.760Z\",\"updatedAt\":\"2025-05-26T09:53:14.772Z\",\"environment\":{\"sys\":{\"id\":\"spotify-engineering\",\"type\":\"Link\",\"linkType\":\"Environment\"}},\"publishedVersion\":4,\"revision\":2,\"contentType\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"ContentType\",\"id\":\"blogCategory\"}},\"locale\":\"en-US\"},\"fields\":{\"name\":\"Infrastructure\",\"slug\":\"infrastructure\",\"has_blogs\":true}},{\"metadata\":{\"tags\":[],\"concepts\":[]},\"sys\":{\"space\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"Space\",\"id\":\"p762jor363g1\"}},\"id\":\"blogcategory_people\",\"type\":\"Entry\",\"createdAt\":\"2025-03-13T13:22:47.242Z\",\"updatedAt\":\"2025-05-26T09:54:46.114Z\",\"environment\":{\"sys\":{\"id\":\"spotify-engineering\",\"type\":\"Link\",\"linkType\":\"Environment\"}},\"publishedVersion\":4,\"revision\":2,\"contentType\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"ContentType\",\"id\":\"blogCategory\"}},\"locale\":\"en-US\"},\"fields\":{\"name\":\"People\",\"slug\":\"people\",\"has_blogs\":true}}],\"tags\":\"$1f:props:tags\",\"featured_media\":{\"metadata\":{\"tags\":[],\"concepts\":[]},\"sys\":{\"space\":{\"sys\":{\"type\":\"Link\",\"linkType\":\"Space\",\"id\":\"p762jor363g1\"}},\"id\":\"attachment_943e369d8982615e731b91feed9abde7\",\"type\":\"Asset\",\"createdAt\":\"2025-03-13T13:09:39.199Z\",\"updatedAt\":\"2025-03-13T13:09:39.199Z\",\"environment\":{\"sys\":{\"id\":\"spotify-engineering\",\"type\":\"Link\",\"linkType\":\"Environment\"}},\"publishedVersion\":3,\"revision\":1,\"locale\":\"en-US\"},\"fields\":{\"title\":\"lon-1\",\"file\":{\"url\":\"//images.ctfassets.net/p762jor363g1/attachment_943e369d8982615e731b91feed9abde7/360a9dc4dcc43513c3d502072fd607bb/attachment_943e369d8982615e731b91feed9abde7.png\",\"details\":{\"size\":38100,\"image\":{\"width\":697,\"height\":491}},\"fileName\":\"attachment_943e369d8982615e731b91feed9abde7.png\",\"contentType\":\"image/png\"}}},\"is_sticky\":false,\"is_old\":true,\"created_at\":\"2013-06-04T16:53:34+00:00\",\"updated_at\":\"2013-06-04T16:53:34+00:00\",\"author\":\"David Poblador i Garcia\",\"user_info\":{\"data\":{},\"content\":[],\"nodeType\":\"document\"}}}}]\n"])</script><script>self.__next_f.push([1,"22:[\"$\",\"div\",null,{\"className\":\"related-posts\",\"children\":[[\"$\",\"div\",null,{\"className\":\"container\",\"children\":[\"$\",\"h3\",null,{\"children\":\"Related articles\"}]}],[\"$\",\"div\",null,{\"className\":\"container\",\"children\":[\"$\",\"div\",null,{\"className\":\"cards\",\"children\":[[\"$\",\"div\",\"4ode0KMR0ZNP2v8r180XjX\",{\"className\":\"post-card related\",\"children\":[[\"$\",\"div\",null,{\"className\":\"post-card__image\",\"children\":[\"$\",\"$L18\",null,{\"href\":\"/2026/9/why-spotify-is-not-using-bayesian-a-b-testing\",\"onClick\":\"$undefined\",\"children\":[\"$\",\"$L1c\",null,{\"src\":\"https://images.ctfassets.net/p762jor363g1/7MYEdTbczd11VcDP5GMLrp/755c9f5d9d2bc9362c1b3c5baf6f6df8/image3.png\",\"alt\":\"sticky\",\"width\":400,\"height\":197,\"quality\":100,\"priority\":true}]}]}],[\"$\",\"div\",null,{\"className\":\"post-card__content\",\"children\":[[\"$\",\"div\",null,{\"className\":\"top-content\",\"children\":[[\"$\",\"div\",null,{\"className\":\"post-card__date\",\"children\":\"Sep 8, 2026\"}],[\"$\",\"$L18\",null,{\"href\":\"/2026/9/why-spotify-is-not-using-bayesian-a-b-testing\",\"onClick\":\"$undefined\",\"children\":[\"$\",\"h4\",null,{\"className\":\"post-card__title\",\"children\":\"Why Spotify Is Not Using Bayesian A/B Testing\"}]}],[\"$\",\"div\",null,{\"className\":\"post-card__description\",\"children\":\"Clearing the confusion about what Bayesian A/B testing is.\"}],[\"$\",\"div\",null,{\"className\":\"post-card__published\",\"children\":\"$undefined\"}]]}],[\"$\",\"div\",null,{\"className\":\"post-card__tags\",\"children\":\"$undefined\"}]]}]]}],[\"$\",\"div\",\"2BHiaulFaaun4MVkEBakNn\",{\"className\":\"post-card related\",\"children\":[[\"$\",\"div\",null,{\"className\":\"post-card__image\",\"children\":[\"$\",\"$L18\",null,{\"href\":\"/2026/7/indexing-the-data-lake-for-online-point-queries\",\"onClick\":\"$undefined\",\"children\":[\"$\",\"$L1c\",null,{\"src\":\"https://images.ctfassets.net/p762jor363g1/69pB35ICFY1sL97klnboyf/aed20373e05048d11cdd12de739b4524/image1.png\",\"alt\":\"sticky\",\"width\":400,\"height\":197,\"quality\":100,\"priority\":true}]}]}],[\"$\",\"div\",null,{\"className\":\"post-card__content\",\"children\":[[\"$\",\"div\",null,{\"className\":\"top-content\",\"children\":[[\"$\",\"div\",null,{\"className\":\"post-card__date\",\"children\":\"Jul 27, 2026\"}],[\"$\",\"$L18\",null,{\"href\":\"/2026/7/indexing-the-data-lake-for-online-point-queries\",\"onClick\":\"$undefined\",\"children\":[\"$\",\"h4\",null,{\"className\":\"post-card__title\",\"children\":\"Indexing the Data Lake for Online Point Queries\"}]}],[\"$\",\"div\",null,{\"className\":\"post-card__description\",\"children\":\"Companies like Spotify need vast quantities of data accessible at low latency for online services and,...\"}],[\"$\",\"div\",null,{\"className\":\"post-card__published\",\"children\":\"$undefined\"}]]}],[\"$\",\"div\",null,{\"className\":\"post-card__tags\",\"children\":\"$undefined\"}]]}]]}],[\"$\",\"div\",\"6OwIaRIOSFOCZSHMjmLIxv\",{\"className\":\"post-card related\",\"children\":[[\"$\",\"div\",null,{\"className\":\"post-card__image\",\"children\":[\"$\",\"$L18\",null,{\"href\":\"/2026/6/encoding-your-domain-expert-the-context-layer-behind-spotifys-data-assistant\",\"onClick\":\"$undefined\",\"children\":[\"$\",\"$L1c\",null,{\"src\":\"https://images.ctfassets.net/p762jor363g1/7l2eUSuEW1R60JzrWTVCTa/c29b1e8011108d7c79a3b4d4d6f6237f/IMG_5977.png\",\"alt\":\"sticky\",\"width\":400,\"height\":197,\"quality\":100,\"priority\":true}]}]}],[\"$\",\"div\",null,{\"className\":\"post-card__content\",\"children\":[[\"$\",\"div\",null,{\"className\":\"top-content\",\"children\":[[\"$\",\"div\",null,{\"className\":\"post-card__date\",\"children\":\"Jun 10, 2026\"}],[\"$\",\"$L18\",null,{\"href\":\"/2026/6/encoding-your-domain-expert-the-context-layer-behind-spotifys-data-assistant\",\"onClick\":\"$undefined\",\"children\":[\"$\",\"h4\",null,{\"className\":\"post-card__title\",\"children\":\"Encoding Your Domain Expert: The Context Layer Behind Spotify's Data Assistant\"}]}],[\"$\",\"div\",null,{\"className\":\"post-card__description\",\"children\":\"At Spotify, data problems used to follow a specific pattern. You'd look for the relevant dashboard, there...\"}],[\"$\",\"div\",null,{\"className\":\"post-card__published\",\"children\":\"$undefined\"}]]}],[\"$\",\"div\",null,{\"className\":\"post-card__tags\",\"children\":\"$undefined\"}]]}]]}],[\"$\",\"div\",\"1XeG8OFPaFVQT83ShkXhoz\",{\"className\":\"post-card related\",\"children\":[[\"$\",\"div\",null,{\"className\":\"post-card__image\",\"children\":[\"$\",\"$L18\",null,{\"href\":\"/2026/5/better-experiments-with-llm-evals-a-funnel-not-a-fork\",\"onClick\":\"$undefined\",\"children\":[\"$\",\"$L1c\",null,{\"src\":\"https://images.ctfassets.net/p762jor363g1/1liN9CCW7JU64GI5r03T6N/95c8d19c1287e6209831f31f8dd89158/3__1_.png\",\"alt\":\"sticky\",\"width\":400,\"height\":197,\"quality\":100,\"priority\":true}]}]}],[\"$\",\"div\",null,{\"className\":\"post-card__content\",\"children\":[[\"$\",\"div\",null,{\"className\":\"top-content\",\"children\":[\"$L26\",\"$L27\",\"$L28\",\"$L29\"]}],\"$L2a\"]}]]}]]}]}]]}]\n"])</script><script>self.__next_f.push([1,"26:[\"$\",\"div\",null,{\"className\":\"post-card__date\",\"children\":\"May 18, 2026\"}]\n27:[\"$\",\"$L18\",null,{\"href\":\"/2026/5/better-experiments-with-llm-evals-a-funnel-not-a-fork\",\"onClick\":\"$undefined\",\"children\":[\"$\",\"h4\",null,{\"className\":\"post-card__title\",\"children\":\"Better Experiments with LLM Evals — A funnel, not a fork\"}]}]\n28:[\"$\",\"div\",null,{\"className\":\"post-card__description\",\"children\":\"TL;DR LLM evals, automated judges that assess relevance, coherence, and quality at scale, are a powerful new...\"}]\n29:[\"$\",\"div\",null,{\"className\":\"post-card__published\",\"children\":\"$undefined\"}]\n2a:[\"$\",\"div\",null,{\"className\":\"post-card__tags\",\"children\":\"$undefined\"}]\n"])</script></body></html> |