Files
nexus/sreweekly/articles/265/08-learning-from-incidents-getting-sidekiq-ready-to-serve-a-billion-jobs.html
2026-09-12 17:23:01 +08:00

582 lines
48 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html class="" lang="en"><head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1"><!-- Begin Jekyll SEO tag v2.8.0 -->
<title>Learning from incidents: getting Sidekiq ready to serve a billion jobs | Scribd Technology</title>
<meta name="generator" content="Jekyll v3.10.0" />
<meta property="og:title" content="Learning from incidents: getting Sidekiq ready to serve a billion jobs" />
<meta name="author" content="Nakul Pathak" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Scribd currently serves hundreds of Sidekiq jobs per second and has served 25 billion jobs since its adoption 2 years ago. Getting to this scale wasn’t easy. In this post, I’ll walk you through one of our first ever Sidekiq incidents and how we improved our Sidekiq implementation as a result of this incident." />
<meta property="og:description" content="Scribd currently serves hundreds of Sidekiq jobs per second and has served 25 billion jobs since its adoption 2 years ago. Getting to this scale wasn’t easy. In this post, I’ll walk you through one of our first ever Sidekiq incidents and how we improved our Sidekiq implementation as a result of this incident." />
<link rel="canonical" href="https://tech.scribd.com/blog/2020/sidekiq-incident-learnings.html" />
<meta property="og:url" content="https://tech.scribd.com/blog/2020/sidekiq-incident-learnings.html" />
<meta property="og:site_name" content="Scribd Technology" />
<meta property="og:type" content="article" />
<meta property="article:published_time" content="2020-12-21T00:00:00+00:00" />
<meta name="twitter:card" content="summary" />
<meta property="twitter:title" content="Learning from incidents: getting Sidekiq ready to serve a billion jobs" />
<script type="application/ld+json">
{"@context":"https://schema.org","@type":"BlogPosting","author":{"@type":"Person","name":"Nakul Pathak"},"dateModified":"2020-12-21T00:00:00+00:00","datePublished":"2020-12-21T00:00:00+00:00","description":"Scribd currently serves hundreds of Sidekiq jobs per second and has served 25 billion jobs since its adoption 2 years ago. Getting to this scale wasn’t easy. In this post, I’ll walk you through one of our first ever Sidekiq incidents and how we improved our Sidekiq implementation as a result of this incident.","headline":"Learning from incidents: getting Sidekiq ready to serve a billion jobs","mainEntityOfPage":{"@type":"WebPage","@id":"https://tech.scribd.com/blog/2020/sidekiq-incident-learnings.html"},"url":"https://tech.scribd.com/blog/2020/sidekiq-incident-learnings.html"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/assets/css/main.css"><link type="application/atom+xml" rel="alternate" href="https://tech.scribd.com/feed.xml" title="Scribd Technology" /><script>
if(!(window.doNotTrack === "1" || navigator.doNotTrack === "1" || navigator.doNotTrack === "yes" || navigator.msDoNotTrack === "1")) {
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
ga('create', 'UA-443684-30', 'auto');
ga('send', 'pageview');
}
</script>
</head>
<body><header class="nav" role="banner">
<!-- Skip to button -->
<a class="skip-link" href="#main-content">Skip to content</a>
<nav class="nav__container container"><!-- Logo -->
<a rel="author" href="/">
<h1 class="visually-hidden">Scribd Technology</h1>
<svg class="logo " role="img" aria-hidden="true" aria-label="Scribd Technology Logo" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 133 42">
<g fill="none" fill-rule="nonzero">
<!-- Icon -->
<path class="logo__icon" fill="#1E7B85" d="M18.203 13.653l.238.006a4.882 4.882 0 013.93 1.776c.996 1.185 1.494 2.832 1.494 4.941 0 2.91-.916 5.19-2.747 6.842a11.176 11.176 0 01-6.87 2.676c-.466.065-.942.065-1.436.065A17.724 17.724 0 01.959 25.47L0 24.61l3.77-4.464 2.212 1.706L3.888 24.3a14.924 14.924 0 008.953 2.941c3.847 0 6.512-1.459 7.547-4a3.53 3.53 0 00.088-.788c0-.753-.311-1.13-.935-1.13a4.412 4.412 0 00-.841.16c-.346.079-.698.124-1.053.135a3.594 3.594 0 01-2.447-1.03 3.53 3.53 0 01-1.141-2.753 4.118 4.118 0 011.176-2.94 4.206 4.206 0 013.206-1.236zM11.483.588a16.218 16.218 0 0110.44 3.594l1.012.794-3.14 4.483-2.319-1.53 1.671-2.4A13.53 13.53 0 0011.5 3.306a9.459 9.459 0 00-3.912.776 8.935 8.935 0 00-1.135.694c-1.263.91-1.894 1.999-1.894 3.265 0 .753.312 1.13.935 1.13.285-.026.567-.08.841-.16.346-.079.699-.124 1.053-.135.904.02 1.767.379 2.418 1.006a3.53 3.53 0 011.135 2.77 4.118 4.118 0 01-1.176 2.942 4.206 4.206 0 01-3.206 1.235 4.882 4.882 0 01-3.93-1.764c-1-1.183-1.535-2.853-1.54-4.965v-.382c0-5.377 4.276-9.13 10.393-9.13z"/>
<!-- Wordmark -->
<path class="logo__wordmark" fill="#00293F" d="M60.941 7.224a7.994 7.994 0 015.988 2.435l-1.176 1.27a6.676 6.676 0 00-4.824-2.105c-3.4 0-5.947 2.747-5.947 6.317 0 3.57 2.57 6.359 5.947 6.359 2.106 0 3.489-.812 4.953-2.212l1.177 1.118a7.994 7.994 0 01-6.17 2.694c-4.418 0-7.713-3.524-7.713-7.912s3.248-7.964 7.765-7.964zm-18.217.04a8.012 8.012 0 015.394 1.889l-1.012 1.335a6.747 6.747 0 00-4.43-1.664c-2 0-3.264 1.029-3.264 2.5 0 1.47.782 2.282 4.164 3.005 3.418.747 5 1.995 5 4.365 0 2.653-2.194 4.365-5.24 4.365a8.982 8.982 0 01-6.277-2.47l1.047-1.254a7.312 7.312 0 005.294 2.17c2.065 0 3.424-1.076 3.424-2.634 0-1.447-.771-2.26-3.995-2.942-3.529-.764-5.153-2.005-5.153-4.453 0-2.447 2.106-4.211 5.048-4.211zm36.023.224c3.382 0 5.553 1.818 5.553 4.583 0 2.564-1.753 4.058-4.153 4.488l4.706 6.276h-2.13l-4.452-5.964h-4.389v5.97h-1.735V7.488zm14.282-.006v15.353h-1.735V7.482h1.735zm12.983 0c3.023 0 5 1.489 5 3.842a3.718 3.718 0 01-2.436 3.576c1.818.547 3.289 1.57 3.289 3.724 0 2.652-2.212 4.211-5.57 4.211H99.47V7.482zm17.97 0c4.842 0 8.16 3.312 8.16 7.653 0 4.341-3.336 7.7-8.16 7.7h-5.33V7.482zm-17.958 8.4h-4.848v5.4h5.165c2.324 0 3.765-1.058 3.765-2.747 0-1.688-1.406-2.653-4.082-2.653zm17.958-6.8h-3.6v12.153h3.6c3.883 0 6.36-2.63 6.36-6.053s-2.477-6.1-6.36-6.1zm-45.358 0h-4.742v6.212h4.712c2.306 0 3.959-1.259 3.959-3.165 0-1.905-1.453-3.047-3.93-3.047zm27.258-.04h-4.706v5.287h4.577c2.082 0 3.53-.94 3.53-2.764 0-1.536-1.207-2.524-3.4-2.524z"/>
<!-- Subhead -->
<path class="logo__subhead" fill="#57617A" d="M39.884 32.134h-1.918v-.868l1.96-.07.154-1.904h.952v1.904h3.346v.938h-3.346v3.234c0 .308.028.576.084.805.056.229.15.422.28.581.13.159.308.278.532.357.224.08.5.119.826.119.327 0 .62-.028.882-.084a5.05 5.05 0 00.756-.224l.252.84a7.646 7.646 0 01-1.022.287c-.373.08-.747.119-1.12.119-.495 0-.91-.068-1.246-.203a1.949 1.949 0 01-.805-.574 2.296 2.296 0 01-.434-.889 4.616 4.616 0 01-.133-1.148v-3.22zm6.482 2.478c0-.56.098-1.062.294-1.505.196-.443.455-.819.777-1.127a3.471 3.471 0 012.387-.952c.485 0 .92.077 1.302.231.383.154.707.371.973.651s.469.616.609 1.008.21.826.21 1.302c0 .121-.002.24-.007.357a1.725 1.725 0 01-.035.301h-5.32c.01.364.082.69.217.98.135.29.317.537.546.742.229.205.5.364.812.476.313.112.656.168 1.029.168.401 0 .768-.056 1.099-.168.331-.112.656-.266.973-.462l.406.756a5.97 5.97 0 01-1.148.56c-.43.159-.92.238-1.47.238-.504 0-.978-.082-1.421-.245a3.484 3.484 0 01-1.162-.7 3.246 3.246 0 01-.784-1.113c-.191-.439-.287-.938-.287-1.498zm5.488-.56c0-.681-.177-1.204-.532-1.568-.355-.364-.85-.546-1.484-.546-.28 0-.548.047-.805.14a2.137 2.137 0 00-.693.413 2.38 2.38 0 00-.518.665c-.14.261-.233.56-.28.896h4.312zm3.066.56c0-.57.1-1.076.301-1.519.2-.443.474-.819.819-1.127a3.51 3.51 0 011.211-.7c.462-.159.95-.238 1.463-.238.56 0 1.043.098 1.449.294.406.196.749.43 1.029.7l-.574.742a3.716 3.716 0 00-.875-.574 2.326 2.326 0 00-.987-.21c-.392 0-.751.063-1.078.189a2.315 2.315 0 00-.833.539 2.54 2.54 0 00-.539.833c-.13.322-.196.679-.196 1.071 0 .392.063.747.189 1.064.126.317.303.59.532.819.229.229.504.406.826.532.322.126.679.189 1.071.189.42 0 .805-.082 1.155-.245.35-.163.67-.366.959-.609l.504.742a4.227 4.227 0 01-1.253.784c-.462.187-.945.28-1.449.28-.532 0-1.024-.08-1.477-.238a3.443 3.443 0 01-1.183-.693 3.112 3.112 0 01-.784-1.12 3.846 3.846 0 01-.28-1.505zm8.582-6.58h1.148v2.716l-.056 1.61c.364-.383.751-.7 1.162-.952.41-.252.882-.378 1.414-.378.812 0 1.407.231 1.785.693.378.462.567 1.146.567 2.051V38h-1.148v-4.074c0-.644-.121-1.122-.364-1.435-.243-.313-.644-.469-1.204-.469-.205 0-.394.026-.567.077a2.16 2.16 0 00-.511.231c-.168.103-.34.231-.518.385a8.362 8.362 0 00-.56.539V38h-1.148v-9.968zm8.4 3.164h.952l.098 1.162h.056c.355-.383.737-.7 1.148-.952.41-.252.882-.378 1.414-.378.812 0 1.407.231 1.785.693.378.462.567 1.146.567 2.051V38h-1.148v-4.074c0-.644-.121-1.122-.364-1.435-.243-.313-.644-.469-1.204-.469-.205 0-.394.026-.567.077a2.16 2.16 0 00-.511.231c-.168.103-.34.231-.518.385a8.362 8.362 0 00-.56.539V38h-1.148v-6.804zm7.938 3.416c0-.57.091-1.076.273-1.519.182-.443.427-.819.735-1.127a3.088 3.088 0 011.071-.7 3.488 3.488 0 011.281-.238c.448 0 .875.08 1.281.238.406.159.763.392 1.071.7.308.308.553.684.735 1.127.182.443.273.95.273 1.519 0 .56-.091 1.062-.273 1.505a3.316 3.316 0 01-.735 1.12 3.138 3.138 0 01-1.071.693 3.488 3.488 0 01-1.281.238c-.448 0-.875-.08-1.281-.238a3.138 3.138 0 01-1.071-.693 3.316 3.316 0 01-.735-1.12 3.932 3.932 0 01-.273-1.505zm1.19 0c0 .392.051.747.154 1.064.103.317.25.59.441.819.191.229.42.406.686.532.266.126.562.189.889.189.327 0 .623-.063.889-.189s.495-.303.686-.532c.191-.229.338-.502.441-.819.103-.317.154-.672.154-1.064s-.051-.749-.154-1.071c-.103-.322-.25-.6-.441-.833a1.968 1.968 0 00-.686-.539 2.05 2.05 0 00-.889-.189 2.05 2.05 0 00-.889.189 1.968 1.968 0 00-.686.539 2.491 2.491 0 00-.441.833 3.513 3.513 0 00-.154 1.071zm7.504-6.58h3.472v7.77c0 .504.119.866.357 1.085.238.22.562.329.973.329.373 0 .803-.107 1.288-.322l.294.868c-.308.112-.597.208-.868.287-.27.08-.593.119-.966.119-.728 0-1.281-.205-1.659-.616-.378-.41-.567-1.022-.567-1.834V28.97h-2.324v-.938zm8.106 6.58c0-.57.091-1.076.273-1.519.182-.443.427-.819.735-1.127a3.088 3.088 0 011.071-.7 3.488 3.488 0 011.281-.238c.448 0 .875.08 1.281.238.406.159.763.392 1.071.7.308.308.553.684.735 1.127.182.443.273.95.273 1.519 0 .56-.091 1.062-.273 1.505a3.316 3.316 0 01-.735 1.12 3.138 3.138 0 01-1.071.693 3.488 3.488 0 01-1.281.238c-.448 0-.875-.08-1.281-.238a3.138 3.138 0 01-1.071-.693 3.316 3.316 0 01-.735-1.12 3.932 3.932 0 01-.273-1.505zm1.19 0c0 .392.051.747.154 1.064.103.317.25.59.441.819.191.229.42.406.686.532.266.126.562.189.889.189.327 0 .623-.063.889-.189s.495-.303.686-.532c.191-.229.338-.502.441-.819.103-.317.154-.672.154-1.064s-.051-.749-.154-1.071c-.103-.322-.25-.6-.441-.833a1.968 1.968 0 00-.686-.539 2.05 2.05 0 00-.889-.189 2.05 2.05 0 00-.889.189 1.968 1.968 0 00-.686.539 2.491 2.491 0 00-.441.833 3.513 3.513 0 00-.154 1.071zm8.372 4.522c0 .355.189.644.567.868.378.224.931.336 1.659.336.392 0 .747-.04 1.064-.119.317-.08.588-.184.812-.315.224-.13.394-.28.511-.448a.898.898 0 00.175-.518c0-.327-.128-.546-.385-.658-.257-.112-.646-.168-1.169-.168h-1.33c-.196 0-.38-.007-.553-.021a2.216 2.216 0 01-.483-.091c-.327.177-.553.362-.679.553a1.047 1.047 0 00-.189.581zm-.994.112c0-.27.096-.537.287-.798.191-.261.469-.5.833-.714v-.056a1.61 1.61 0 01-.49-.42c-.14-.177-.21-.41-.21-.7 0-.215.07-.436.21-.665.14-.229.34-.432.602-.609v-.056a2.233 2.233 0 01-.595-.707c-.163-.294-.245-.646-.245-1.057a2.283 2.283 0 01.798-1.771 2.67 2.67 0 01.847-.49 2.997 2.997 0 011.029-.175c.373 0 .71.056 1.008.168h2.814v.882h-1.82c.159.168.303.373.434.616.13.243.196.513.196.812 0 .364-.068.69-.203.98-.135.29-.322.534-.56.735-.238.2-.518.355-.84.462a3.233 3.233 0 01-1.029.161c-.177 0-.366-.021-.567-.063a2.819 2.819 0 01-.581-.189c-.355.224-.532.485-.532.784 0 .27.124.469.371.595s.595.189 1.043.189h1.526c.877 0 1.528.121 1.953.364.425.243.637.653.637 1.232 0 .317-.089.62-.266.91-.177.29-.43.541-.756.756a4.237 4.237 0 01-1.19.518c-.467.13-.99.196-1.568.196-.999 0-1.771-.163-2.317-.49-.546-.327-.819-.793-.819-1.4zm3.066-4.172c.439 0 .81-.145 1.113-.434.303-.29.455-.681.455-1.176a1.618 1.618 0 00-.462-1.148 1.574 1.574 0 00-1.106-.448 1.574 1.574 0 00-1.106.448 1.618 1.618 0 00-.462 1.148c0 .495.154.887.462 1.176.308.29.677.434 1.106.434zm5.502 4.802c.093.028.194.051.301.07.107.019.212.028.315.028.476 0 .863-.138 1.162-.413.299-.275.532-.623.7-1.043l.21-.504-3.178-6.818h1.162l1.666 3.766.413.966.427 1.008h.056l.728-1.974 1.484-3.766h1.092l-2.996 7.364c-.13.336-.28.649-.448.938-.168.29-.369.539-.602.749a2.68 2.68 0 01-.805.497 2.751 2.751 0 01-1.029.182c-.336 0-.635-.047-.896-.14l.238-.91z"/>
</g>
</svg>
</a>
<!-- Mobile menu button -->
<button class="nav__menu-trigger hidden-md-up" type="button" aria-label="Navigation Menu" aria-expanded="false">
<span class="visually-hidden">Menu</span>
<span class="hamburger">
<span class="hamburger__inner"></span>
</span>
</button>
<!-- Site links -->
<ul class="nav-list list--plain monospace">
<li class="nav-list__item">
<a class="nav-list__link active" href="/blog/">Blog</a>
</li>
<li class="nav-list__item">
<a class="nav-list__link " href="/feed.xml">RSS</a>
</li>
<li class="nav-list__item">
<a class="nav-list__link " href="/projects/">Projects</a>
</li>
<li class="nav-list__item">
<a class="nav-list__link " href="/careers/">Careers</a>
</li>
<!-- Social icons (mobile only) -->
<li class="nav-social-list hidden-md-up">
<a class="nav-social-list__item text-white no-underline" href="https://www.facebook.com/scribd">
<svg class="svg-icon"><use xlink:href="/assets/images/icons/icon-sprite.svg#facebook"></use></svg>
<span class="visually-hidden">Scribd on Facebook</span>
</a>
<a class="nav-social-list__item text-white no-underline" href="https://www.linkedin.com/company/scribd">
<svg class="svg-icon"><use xlink:href="/assets/images/icons/icon-sprite.svg#linkedin"></use></svg>
<span class="visually-hidden">Scribd on LinkedIn</span>
</a>
<a class="nav-social-list__item text-white no-underline" href="https://github.com/scribd">
<svg class="svg-icon"><use xlink:href="/assets/images/icons/icon-sprite.svg#github"></use></svg>
<span class="visually-hidden">Scribd on Github</span>
</a>
</li>
<!-- Github links (mobile only) -->
<li class="hidden-md-up monospace fs-md">
<a class="nav-github-link text-white" href="https://github.com/scribd/scribd.github.io/edit/main/_posts/2020-12-21-sidekiq-incident-learnings.md">Improve this Page</a><a class="nav-github-link text-white" href="https://github.com/scribd/scribd.github.io/commits/main/_posts/2020-12-21-sidekiq-incident-learnings.md">Page History</a>
</li>
</ul>
</nav>
</header>
<main id="main-content" aria-label="Content">
<article itemscope itemtype="http://schema.org/BlogPosting">
<!-- Post hero -->
<!-- Assign theme colors based of the project team -->
<!-- See include below for logic and color.scss for available themes -->
<section class="hero theme-midnight">
<div class="hero__container container container-offset">
<header class="hero__header">
<!-- Post Title -->
<h1 class="hero__title" itemprop="name headline">Learning from incidents: getting Sidekiq ready to serve a billion jobs</h1>
<!-- Post Metadata -->
<ul class="hero__meta">
<!-- Post Author --><li class="hero__meta-item author media">
<!-- Author Avatar --><img class="hero__meta-avatar"
srcset="https://github.com/nakulpathak3.png?size=48,
https://github.com/nakulpathak3.png?size=96 2x"
src="https://github.com/nakulpathak3.png?size=48" role="presentation" alt=""><div class="media-body">
<strong>Author</strong><br>
<span itemprop="author" itemscope itemtype="http://schema.org/Person">
<span itemprop="name">Nakul Pathak</span>
</span>
</div>
</li><!-- Post Date --><li class="hero__meta-item published">
<strong>Published</strong><br>
<time datetime="2020-12-21T00:00:00+00:00" itemprop="datePublished">December 21, 2020
</time>
</li><!-- Team Info -->
<li class="hero__meta-item category">
<strong>Team</strong><br>
<span>Developer Platform, Internal Tools</span>
</li></ul>
</header>
</div>
<svg class="logo-texture" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1800 500" aria-hidden="true" preserveAspectRatio="xMidYMin slice"><path fill="none" stroke="#000" stroke-width="2" d="M1799.827-962.497l56.146 319.467c70.96 2.313 137.884 34.876 183.516 89.942 50.973 60.248 78.258 145.376 78.558 252.984v19.484c0 250.737-182.584 432.178-452.593 461.145l-.007-.014a258.945 258.945 0 0 1-62.73 85.702 300.651 300.651 0 0 1-40.556 36.266 972.911 972.911 0 0 1 354.746 82.623l70.974 30.313-112.45 308.575-158.081-52.65 59.554-165.045a811.83 811.83 0 0 0-477.751-10.112 567.576 567.576 0 0 0-214.666 105.754 536.156 536.156 0 0 0-55.022 57.861c-10.683 13.087-20.079 26.191-28.186 39.313a249.28 249.28 0 0 1-18.614 35.15c-21.128 47.824-25.22 95.881-12.28 144.171 11.692 43.64 35.609 60.618 71.748 50.934a265.08 265.08 0 0 0 46.288-22.269 326.85 326.85 0 0 1 58.927-24.194 215.664 215.664 0 0 1 155.747 20.753 211.775 211.775 0 0 1 108.828 142.949 247.07 247.07 0 0 1-22.511 188.737 252.372 252.372 0 0 1-166.628 121.384c-86.848 26.538-181.093 11.3-255.152-41.255-76.322-52.997-133.292-141.51-166.428-263.813l-4.842-18.07-1.096-4.09c-15.172-56.623-21.328-112.332-19.115-166.159l.01.004a289.76 289.76 0 0 0 109.065-40.588 249.713 249.713 0 0 0 97.44-88.495c5.177-11.731 11.383-23.447 18.613-35.15a249.33 249.33 0 0 0 12.126-35.051 244.485 244.485 0 0 0-26.821-186.165 209.883 209.883 0 0 0-3.232-6.047c73.907-58.51 165.753-104.279 272.573-132.901.848-.242 1.696-.483 2.545-.722-62.993-23.849-125.712-66.011-186.743-126.703l-10.63-10.631-6.009-6.008C970.75 110.544 919.783 11.195 901.561-93.858a803.248 803.248 0 0 1 166.025-35.237l6.572-9.376c-7.258-34.446-7.443-72.85-.553-115.21a550.116 550.116 0 0 1 7.87-39.117 962.532 962.532 0 0 1 218.365 9.082l75.602 10.797-99.758 78.794 7.013 4.627-85.154 122.296a689.742 689.742 0 0 0 41.56 25.27 271.683 271.683 0 0 1-16.021 21.208c-27.144 27.144-57.099 24.333-89.864-8.432-28.369-28.369-47.4-61.96-57.095-100.776a802.95 802.95 0 0 1 120.938-2.089l78.623-62.104-111.123-73.307-90.403 128.96a236.27 236.27 0 0 0 1.965 8.54c-2.847.262-5.693.542-8.537.837l-63.14 90.069 51.573 40.465c150.513 121.405 338.84 186.212 532.213 183.144 26.44 0 52.206-1.375 77.222-4.058l-.007-.014a258.876 258.876 0 0 0 23.075-107.614 253.435 253.435 0 0 0-2.82-35.286 482.21 482.21 0 0 0 101.023-31.076 455.48 455.48 0 0 0 57.87-35.37c64.365-46.36 96.547-101.813 96.547-166.358 0-38.368-15.891-57.551-47.674-57.551a225.241 225.241 0 0 0-42.877 8.093 277.733 277.733 0 0 1-53.671 6.894 183.23 183.23 0 0 1-123.234-51.256 179.819 179.819 0 0 1-57.869-141.18 209.787 209.787 0 0 1 59.968-149.872 214.413 214.413 0 0 1 19.284-17.192l163.071-33.44-2.167-12.333a249.247 249.247 0 0 0-16.776.018 214.436 214.436 0 0 0-144.128 45.755l-4.377.899-31.714-177.165a833.152 833.152 0 0 0-429.657 236.136 582.511 582.511 0 0 0-136.5 204.078 550.155 550.155 0 0 0-11.346 40.511A962.793 962.793 0 0 0 899.75-261.98c-8.292 56.718-7.716 113.2 1.81 168.124A803.1 803.1 0 0 0 737.827-19.33a561.638 561.638 0 0 0-178.097 156.06 530.547 530.547 0 0 0-37.772 69.397c-37.922 84.271-38.081 158.973-.476 224.106 22.353 38.717 49.56 48.82 81.623 30.308a262.308 262.308 0 0 0 38.54-33.14 323.431 323.431 0 0 1 50.127-38.216 213.407 213.407 0 0 1 154.181-20.053 209.56 209.56 0 0 1 137.4 102.714C856.489 572.252 782.445 710.18 776.15 863.339l.01.004a289.823 289.823 0 0 1-145.38-14.673c-86.524-31.11-163.645-101.12-226.641-209.534l-9.256-16.031-2.096-3.63c-77.694-134.57-91.866-279.442-49.666-413.649l3.153-1.861c132.515-76.508 211.57-188.894 227.26-308.12l-.026-.01c25.744-19.237 53.336-37.414 82.729-54.383A962.784 962.784 0 0 1 899.75-261.981c19.823-135.728 90.414-272.775 210.54-392.9 162.528-167.769 378.712-273.264 610.935-298.128l78.602-9.488zm-537.33 1299.594l-4.026 1.135c32.123 12.165 64.317 19.57 96.395 22.244 74.5 6.978 148.228-14.115 207.295-57.997a972.809 972.809 0 0 0-299.663 34.618zM189.63-467.325a198.314 198.314 0 0 1 174.061-17.728c59.189 21.281 111.946 69.174 155.04 143.338l6.332 10.967 1.434 2.483c41.952 72.663 56.83 149.718 47.04 224.11l-.027-.01C457.842-17.736 379.517 90.078 343.122 205.825l.016.005A658.738 658.738 0 0 1-94 289.59l-51.718-7.385 19.465-221.465 112.587 6.729-10.025 118.35a549.546 549.546 0 0 0 314.153-77.09A384.205 384.205 0 0 0 412.295 1.97a362.936 362.936 0 0 0 25.839-47.473c25.942-57.648 26.05-108.75.326-153.307-15.291-26.485-33.904-33.396-55.837-20.733a179.44 179.44 0 0 0-26.364 22.67 221.253 221.253 0 0 1-34.291 26.144 145.988 145.988 0 0 1-105.473 13.717 143.355 143.355 0 0 1-96.203-74.4 167.248 167.248 0 0 1-18.347-127.352 170.837 170.837 0 0 1 87.684-108.561zm1252.128 289.873a217.166 217.166 0 0 1 169.99 71.144 253.35 253.35 0 0 1 73.955 143.92 482.25 482.25 0 0 1-98.37 8.49 689.803 689.803 0 0 1-348.23-88.035 271.695 271.695 0 0 0 13.687-22.32 335.461 335.461 0 0 1 39.95-51.725 221.31 221.31 0 0 1 149.018-61.474z"/></svg>
</section>
<div class="post-grid section-container container-offset">
<!-- Post sidebar -->
<div class="post-sidebar fs-md monospace">
<!-- Tags --><ul class="post-sidebar__tag-list">
<li class="post-sidebar__tag-label fw-bold">Tags: </li>
<li class="post-sidebar__tags"><a href="/tag/incident response/">incident response</a></li>
<li class="post-sidebar__tags"><a href="/tag/sidekiq/">sidekiq</a></li>
<li class="post-sidebar__tags"><a href="/tag/monitoring/">monitoring</a></li>
<li class="post-sidebar__tags"><a href="/tag/featured/">featured</a></li>
</ul><!-- Sharing / RSS -->
<ul class="post-sidebar__action-list list--inline">
<!-- LinkedIn -->
<li>
<a class="post-sidebar__action-link" href="https://www.linkedin.com/shareArticle?mini=true&url=https%3A%2F%2Ftech.scribd.com%2Fblog%2F2020%2Fsidekiq-incident-learnings.html&title=Learning+from+incidents%3A+getting+Sidekiq+ready+to+serve+a+billion+jobs&source=https://tech.scribd.com" rel="noopener" target="_blank">
<svg class="svg-icon"><use xlink:href="/assets/images/icons/icon-sprite.svg#linkedin"></use></svg>
<span class="visually-hidden">Share on LinkedIn</span>
</a>
</li>
<!-- RSS -->
<li>
<a class="post-sidebar__action-link" href="/feed.xml">
<svg class="svg-icon"><use xlink:href="/assets/images/icons/icon-sprite.svg#rss"></use></svg>
<span class="visually-hidden">RSS Feed</span>
</a>
</li>
</ul>
</div>
<div class="post-grid__body" itemprop="articleBody">
<!-- Post content -->
<div class="post-content">
<p>Scribd currently serves hundreds of Sidekiq jobs per second and has served 25 billion jobs since its adoption 2 years ago. Getting to this scale wasn’t easy. In this post, I’ll walk you through one of our first ever <a href="https://sidekiq.org/">Sidekiq</a> incidents and how we improved our Sidekiq implementation as a result of this incident.</p>
<h3 id="the-incident">The Incident</h3>
<p>A large number of jobs for importing podcasts into Scribd were enqueued via Sidekiq. They took many hours to run and since they were added to our “default” queue, all our servers picked them up unlike if they were in the “bulk” queue. These jobs quickly starved all other jobs including the highest priority ones.</p>
<p><strong>Detection:</strong> The incident was detected by an internal user noticing the queue build-up in Sidekiq’s web UI and a corresponding customer complaint that we linked back to this issue. Our systems were negatively affected for around <strong>7 hours</strong> and the incident was noticed at the 6 hour mark.</p>
<p><strong>Resolution:</strong> We ran a script on production to delete all existing jobs of this problematic worker from Sidekiq’s Redis instance and removed the batch job that was enqueuing them. We let the currently running jobs finish since killing them would require ssh-ing and running risky sudo commands on production servers.</p>
<h3 id="what-we-learned">What we learned</h3>
<p>As pretty much our first ever major Sidekiq incident, we wrote an in-depth incident review that focused on 4 problem areas -</p>
<h4 id="quicker-detection">Quicker Detection</h4>
<p>Our mean-time-to-detect this incident was way too high. To address this, we needed metrics and alerting. Since we have a Sidekiq Enterprise license, we simply integrated the <a href="https://github.com/mperham/sidekiq/wiki/Pro-Metrics#enabling-metrics">Pro</a> and <a href="https://github.com/mperham/sidekiq/wiki/Ent-Historical-Metrics">Enterprise metrics</a> into our existing Ruby Dogstatsd client.</p>
<p>We added the following Datadog monitors -</p>
<ul>
<li>X queue latency &gt; Y value over past Z minutes</li>
<li>% of job failures / total jobs &gt; X% over last Y minutes</li>
</ul>
<h4 id="quicker-debugging">Quicker Debugging</h4>
<p>To help add some debugging power to the monitors above, we also created some useful dashboards.
<img src="/post-images/2020-12-sidekiq-incident-learnings/overall-sidekiq-dashboard.png" alt="Overall Sidekiq dashboard" />
<img src="/post-images/2020-12-sidekiq-incident-learnings/overall-worker-metrics.png" alt="Overall worker metrics" /></p>
<p>We added Sidekiq system-level, queue-level, and worker-level graphs that allow us to quickly go from system health to queue health to erroneous worker. From there, we can go over to the worker dashboard to find out whether the issue is around processing time or job failures and debug further in Sentry if needed.</p>
<p><img src="/post-images/2020-12-sidekiq-incident-learnings/single-worker-dashboard.png" alt="Dashboard for each worker" /></p>
<p>Later, as Scribd adopted Datadog further, we added <a href="https://docs.datadoghq.com/tracing/setup_overview/setup/ruby/#sidekiq">APM for Sidekiq</a> which covered a lot of the functionality we had but also added tracing of worker performance to further debug issues.</p>
<h4 id="quicker-resolution">Quicker Resolution</h4>
<p>Now that we’re able to quickly identify incidents and debug them, the next step is to resolve the issue.</p>
<p>Something we learned from the incident was that editing Sidekiq Redis while it is already overloaded is a slow and highly error-prone process. To overcome this, we utilized Sidekiq’s ability to <a href="https://github.com/mperham/sidekiq/wiki/Middleware">inject custom middlewares</a>.</p>
<p><strong>Job Dropping Middleware:</strong> We created a client middleware that would check a worker’s name against a live feature flag sidekiq_dropped_workers to decide if that worker should execute or be dropped pre-execution. This allowed us to “drain” a specific worker without having to manually edit Sidekiq Redis.</p>
<p><img src="/post-images/2020-12-sidekiq-incident-learnings/sidekiq-dropped-workers.jpeg" alt="Flow diagram for dropping Sidekiq worker jobs" /></p>
<p><strong>Job Disabling Middleware:</strong> In some cases, the worker’s issues may be easily resolvable in an upcoming deploy or re-enqueuing the workers may be extremely difficult. To address such a case, we introduced sidekiq_disabled_workers feature flag which utilized Sidekiq’s <a href="https://github.com/mperham/sidekiq/wiki/Scheduled-Jobs">ScheduledSet</a> to return those jobs to Redis to be run 24 hours later.</p>
<p><img src="/post-images/2020-12-sidekiq-incident-learnings/sidekiq-disabled-workers.jpeg" alt="Flow diagram for temporarily disabling Sidekiq worker jobs" /></p>
<p><strong>Job Termination Incidents Page:</strong> Finally, it was essential to find a way to quickly terminate existing problematic workers that have overtaken the queue. Sidekiq’s web UI is also <a href="https://github.com/mperham/sidekiq/issues/3335">quite extensible</a> so we added a new web page called the “Incidents” tab which allows us to pause queues and terminate existing processes.</p>
<p><img src="/post-images/2020-12-sidekiq-incident-learnings/sidekiq-web-ui.jpg" alt="Sidekiq incidents tab in web UI" /></p>
<h4 id="future-prevention">Future prevention</h4>
<p>The team that added the problematic worker was not aware of Sidekiq’s shared model of usage and their worker’s ability to affect the system. They didn’t know when they should be using the default queue or the bulk queue.</p>
<p><strong>Documentation:</strong> We created processing time and worker importance expectations for each queue. We listed best practices such as using timeouts, preferring multiple smaller jobs, idempotency, etc. and we linked to the <a href="https://github.com/mperham/sidekiq/wiki/Best-Practices">Sidekiq documentation</a> where we felt people may want more information.</p>
<p><strong>Runbook:</strong> We also created an Incident Handling runbook that walks people through finding a problematic worker, debugging, and resolving the incident.</p>
<p><img src="/post-images/2020-12-sidekiq-incident-learnings/sidekiq-incident-handling.png" alt="Sidekiq runbook for incident handling" /></p>
<p><strong>Guardrails:</strong> We also seriously considered adding timeouts which would forcefully terminate workers that go significantly over their queue’s expected processing time. However, we settled for a Sentry exception for workers that missed our guidelines auto-assigned to the team that owns the worker (via CODEOWNERS file). This approach has been sufficient for us so far.</p>
<h3 id="where-we-are-now">Where we are now</h3>
<p>Our systems are far from perfect but Sidekiq issues are now recognized within 5-10 minutes of their occurrence and usually resolved with no significant production impact.</p>
<p>When we addressed these incidents, we were running on data center servers but since then we’ve moved our workloads to AWS Fargate tasks. We’d like to add queue-based auto-scaling and the ability for degradation in database performances caused by Sidekiq workers to be recognizable and auto-resolve.</p>
<!-- Post separator line -->
<hr>
</div>
<!-- Related jobs -->
<section>
<h5 class="mb-3">Related Jobs
<a href="/careers/#open-positions" class="float-right monospace fs-md fw-normal no-underline">
View All <span class="visually-hidden">Jobs</span>
<svg class="svg-icon"><use xlink:href="/assets/images/icons/icon-sprite.svg#arrow-right"></use></svg>
</a>
</h5>
<ul class="card-grid card-grid-sm" id="jobs"></ul>
<script type="text/javascript">
<!--
window.onload = () =>{
// Find jobs for Developer Platform
renderJobs(document.getElementById('jobs'), "", 4);
// Find jobs for Internal Tools
renderJobs(document.getElementById('jobs'), "Internal Tools", 4);
// Always include some Engineering jobs if we have any general ones
renderJobs(document.getElementById('jobs'), "", 2);
};
-->
</script>
</section>
</div>
</div>
<!-- Related posts -->
<!-- Maximum number of related posts to output -->
<!-- Minimum number of common tags that have to match -->
<!-- Render component if there are one or more related posts -->
<div class="related-posts bg-slate-100">
<div class="related-posts__wrapper section-container">
<h1 class="related-posts__title">Keep<br>Reading</h1>
<!-- Show related posts -->
<ul class="post-list">
<!-- Post item template -->
<li class="post-list__item">
<!-- Post title -->
<h4 class="post-list__heading">
<a href="/blog/2026/fast-llm-human-in-the-loop-classification.html" class="link-text-color">
Stop Building Models. Start Building Systems.
</a>
</h4>
<!-- Post metadata -->
<span class="post-list__author">
<span class="visually-hidden">Author:</span>Anish Kumar</span><!--
--><time class="post-list__date" datetime="2026-07-11T00:00:00+00:00">July 11, 2026</time>
</li>
<!-- Post item template -->
<li class="post-list__item">
<!-- Post title -->
<h4 class="post-list__heading">
<a href="/blog/2026/content-trust-score.html" class="link-text-color">
Dual-Embedding Trust Scoring
</a>
</h4>
<!-- Post metadata -->
<span class="post-list__author">
<span class="visually-hidden">Author:</span>Eric Chang</span><!--
--><time class="post-list__date" datetime="2026-02-25T00:00:00+00:00">February 25, 2026</time>
</li>
<!-- Post item template -->
<li class="post-list__item">
<!-- Post title -->
<h4 class="post-list__heading">
<a href="/blog/2026/screaming-in-the-cloud.html" class="link-text-color">
Screaming in the Cloud
</a>
</h4>
<!-- Post metadata -->
<span class="post-list__author">
<span class="visually-hidden">Author:</span>R Tyler Croy</span><!--
--><time class="post-list__date" datetime="2026-02-10T00:00:00+00:00">February 10, 2026</time>
</li>
<!-- Post item template -->
<li class="post-list__item">
<!-- Post title -->
<h4 class="post-list__heading">
<a href="/blog/2026/photodna-csam-detection.html" class="link-text-color">
Deploying a Cost-Effective, Scalable PhotoDNA System for CSAM Detection
</a>
</h4>
<!-- Post metadata -->
<span class="post-list__author">
<span class="visually-hidden">Author:</span>Anish Kumar</span><!--
--><time class="post-list__date" datetime="2026-01-20T00:00:00+00:00">January 20, 2026</time>
</li>
</ul>
<!-- Decorative arrows -->
<svg class="related-posts__arrow svg-icon"><use xlink:href="/assets/images/icons/icon-sprite.svg#arrow-right"></use></svg>
<svg class="related-posts__arrow svg-icon"><use xlink:href="/assets/images/icons/icon-sprite.svg#arrow-right"></use></svg>
<svg class="related-posts__arrow svg-icon"><use xlink:href="/assets/images/icons/icon-sprite.svg#arrow-right"></use></svg>
</div>
</div>
</article>
</main><footer class="footer">
<div class="footer__container container">
<!-- Logo -->
<a rel="author" href="/">
<h2 class="visually-hidden">Scribd Technology></h2>
<svg class="logo dark" role="img" aria-hidden="true" aria-label="Scribd Technology Logo" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 133 42">
<g fill="none" fill-rule="nonzero">
<!-- Icon -->
<path class="logo__icon" fill="#1E7B85" d="M18.203 13.653l.238.006a4.882 4.882 0 013.93 1.776c.996 1.185 1.494 2.832 1.494 4.941 0 2.91-.916 5.19-2.747 6.842a11.176 11.176 0 01-6.87 2.676c-.466.065-.942.065-1.436.065A17.724 17.724 0 01.959 25.47L0 24.61l3.77-4.464 2.212 1.706L3.888 24.3a14.924 14.924 0 008.953 2.941c3.847 0 6.512-1.459 7.547-4a3.53 3.53 0 00.088-.788c0-.753-.311-1.13-.935-1.13a4.412 4.412 0 00-.841.16c-.346.079-.698.124-1.053.135a3.594 3.594 0 01-2.447-1.03 3.53 3.53 0 01-1.141-2.753 4.118 4.118 0 011.176-2.94 4.206 4.206 0 013.206-1.236zM11.483.588a16.218 16.218 0 0110.44 3.594l1.012.794-3.14 4.483-2.319-1.53 1.671-2.4A13.53 13.53 0 0011.5 3.306a9.459 9.459 0 00-3.912.776 8.935 8.935 0 00-1.135.694c-1.263.91-1.894 1.999-1.894 3.265 0 .753.312 1.13.935 1.13.285-.026.567-.08.841-.16.346-.079.699-.124 1.053-.135.904.02 1.767.379 2.418 1.006a3.53 3.53 0 011.135 2.77 4.118 4.118 0 01-1.176 2.942 4.206 4.206 0 01-3.206 1.235 4.882 4.882 0 01-3.93-1.764c-1-1.183-1.535-2.853-1.54-4.965v-.382c0-5.377 4.276-9.13 10.393-9.13z"/>
<!-- Wordmark -->
<path class="logo__wordmark" fill="#00293F" d="M60.941 7.224a7.994 7.994 0 015.988 2.435l-1.176 1.27a6.676 6.676 0 00-4.824-2.105c-3.4 0-5.947 2.747-5.947 6.317 0 3.57 2.57 6.359 5.947 6.359 2.106 0 3.489-.812 4.953-2.212l1.177 1.118a7.994 7.994 0 01-6.17 2.694c-4.418 0-7.713-3.524-7.713-7.912s3.248-7.964 7.765-7.964zm-18.217.04a8.012 8.012 0 015.394 1.889l-1.012 1.335a6.747 6.747 0 00-4.43-1.664c-2 0-3.264 1.029-3.264 2.5 0 1.47.782 2.282 4.164 3.005 3.418.747 5 1.995 5 4.365 0 2.653-2.194 4.365-5.24 4.365a8.982 8.982 0 01-6.277-2.47l1.047-1.254a7.312 7.312 0 005.294 2.17c2.065 0 3.424-1.076 3.424-2.634 0-1.447-.771-2.26-3.995-2.942-3.529-.764-5.153-2.005-5.153-4.453 0-2.447 2.106-4.211 5.048-4.211zm36.023.224c3.382 0 5.553 1.818 5.553 4.583 0 2.564-1.753 4.058-4.153 4.488l4.706 6.276h-2.13l-4.452-5.964h-4.389v5.97h-1.735V7.488zm14.282-.006v15.353h-1.735V7.482h1.735zm12.983 0c3.023 0 5 1.489 5 3.842a3.718 3.718 0 01-2.436 3.576c1.818.547 3.289 1.57 3.289 3.724 0 2.652-2.212 4.211-5.57 4.211H99.47V7.482zm17.97 0c4.842 0 8.16 3.312 8.16 7.653 0 4.341-3.336 7.7-8.16 7.7h-5.33V7.482zm-17.958 8.4h-4.848v5.4h5.165c2.324 0 3.765-1.058 3.765-2.747 0-1.688-1.406-2.653-4.082-2.653zm17.958-6.8h-3.6v12.153h3.6c3.883 0 6.36-2.63 6.36-6.053s-2.477-6.1-6.36-6.1zm-45.358 0h-4.742v6.212h4.712c2.306 0 3.959-1.259 3.959-3.165 0-1.905-1.453-3.047-3.93-3.047zm27.258-.04h-4.706v5.287h4.577c2.082 0 3.53-.94 3.53-2.764 0-1.536-1.207-2.524-3.4-2.524z"/>
<!-- Subhead -->
<path class="logo__subhead" fill="#57617A" d="M39.884 32.134h-1.918v-.868l1.96-.07.154-1.904h.952v1.904h3.346v.938h-3.346v3.234c0 .308.028.576.084.805.056.229.15.422.28.581.13.159.308.278.532.357.224.08.5.119.826.119.327 0 .62-.028.882-.084a5.05 5.05 0 00.756-.224l.252.84a7.646 7.646 0 01-1.022.287c-.373.08-.747.119-1.12.119-.495 0-.91-.068-1.246-.203a1.949 1.949 0 01-.805-.574 2.296 2.296 0 01-.434-.889 4.616 4.616 0 01-.133-1.148v-3.22zm6.482 2.478c0-.56.098-1.062.294-1.505.196-.443.455-.819.777-1.127a3.471 3.471 0 012.387-.952c.485 0 .92.077 1.302.231.383.154.707.371.973.651s.469.616.609 1.008.21.826.21 1.302c0 .121-.002.24-.007.357a1.725 1.725 0 01-.035.301h-5.32c.01.364.082.69.217.98.135.29.317.537.546.742.229.205.5.364.812.476.313.112.656.168 1.029.168.401 0 .768-.056 1.099-.168.331-.112.656-.266.973-.462l.406.756a5.97 5.97 0 01-1.148.56c-.43.159-.92.238-1.47.238-.504 0-.978-.082-1.421-.245a3.484 3.484 0 01-1.162-.7 3.246 3.246 0 01-.784-1.113c-.191-.439-.287-.938-.287-1.498zm5.488-.56c0-.681-.177-1.204-.532-1.568-.355-.364-.85-.546-1.484-.546-.28 0-.548.047-.805.14a2.137 2.137 0 00-.693.413 2.38 2.38 0 00-.518.665c-.14.261-.233.56-.28.896h4.312zm3.066.56c0-.57.1-1.076.301-1.519.2-.443.474-.819.819-1.127a3.51 3.51 0 011.211-.7c.462-.159.95-.238 1.463-.238.56 0 1.043.098 1.449.294.406.196.749.43 1.029.7l-.574.742a3.716 3.716 0 00-.875-.574 2.326 2.326 0 00-.987-.21c-.392 0-.751.063-1.078.189a2.315 2.315 0 00-.833.539 2.54 2.54 0 00-.539.833c-.13.322-.196.679-.196 1.071 0 .392.063.747.189 1.064.126.317.303.59.532.819.229.229.504.406.826.532.322.126.679.189 1.071.189.42 0 .805-.082 1.155-.245.35-.163.67-.366.959-.609l.504.742a4.227 4.227 0 01-1.253.784c-.462.187-.945.28-1.449.28-.532 0-1.024-.08-1.477-.238a3.443 3.443 0 01-1.183-.693 3.112 3.112 0 01-.784-1.12 3.846 3.846 0 01-.28-1.505zm8.582-6.58h1.148v2.716l-.056 1.61c.364-.383.751-.7 1.162-.952.41-.252.882-.378 1.414-.378.812 0 1.407.231 1.785.693.378.462.567 1.146.567 2.051V38h-1.148v-4.074c0-.644-.121-1.122-.364-1.435-.243-.313-.644-.469-1.204-.469-.205 0-.394.026-.567.077a2.16 2.16 0 00-.511.231c-.168.103-.34.231-.518.385a8.362 8.362 0 00-.56.539V38h-1.148v-9.968zm8.4 3.164h.952l.098 1.162h.056c.355-.383.737-.7 1.148-.952.41-.252.882-.378 1.414-.378.812 0 1.407.231 1.785.693.378.462.567 1.146.567 2.051V38h-1.148v-4.074c0-.644-.121-1.122-.364-1.435-.243-.313-.644-.469-1.204-.469-.205 0-.394.026-.567.077a2.16 2.16 0 00-.511.231c-.168.103-.34.231-.518.385a8.362 8.362 0 00-.56.539V38h-1.148v-6.804zm7.938 3.416c0-.57.091-1.076.273-1.519.182-.443.427-.819.735-1.127a3.088 3.088 0 011.071-.7 3.488 3.488 0 011.281-.238c.448 0 .875.08 1.281.238.406.159.763.392 1.071.7.308.308.553.684.735 1.127.182.443.273.95.273 1.519 0 .56-.091 1.062-.273 1.505a3.316 3.316 0 01-.735 1.12 3.138 3.138 0 01-1.071.693 3.488 3.488 0 01-1.281.238c-.448 0-.875-.08-1.281-.238a3.138 3.138 0 01-1.071-.693 3.316 3.316 0 01-.735-1.12 3.932 3.932 0 01-.273-1.505zm1.19 0c0 .392.051.747.154 1.064.103.317.25.59.441.819.191.229.42.406.686.532.266.126.562.189.889.189.327 0 .623-.063.889-.189s.495-.303.686-.532c.191-.229.338-.502.441-.819.103-.317.154-.672.154-1.064s-.051-.749-.154-1.071c-.103-.322-.25-.6-.441-.833a1.968 1.968 0 00-.686-.539 2.05 2.05 0 00-.889-.189 2.05 2.05 0 00-.889.189 1.968 1.968 0 00-.686.539 2.491 2.491 0 00-.441.833 3.513 3.513 0 00-.154 1.071zm7.504-6.58h3.472v7.77c0 .504.119.866.357 1.085.238.22.562.329.973.329.373 0 .803-.107 1.288-.322l.294.868c-.308.112-.597.208-.868.287-.27.08-.593.119-.966.119-.728 0-1.281-.205-1.659-.616-.378-.41-.567-1.022-.567-1.834V28.97h-2.324v-.938zm8.106 6.58c0-.57.091-1.076.273-1.519.182-.443.427-.819.735-1.127a3.088 3.088 0 011.071-.7 3.488 3.488 0 011.281-.238c.448 0 .875.08 1.281.238.406.159.763.392 1.071.7.308.308.553.684.735 1.127.182.443.273.95.273 1.519 0 .56-.091 1.062-.273 1.505a3.316 3.316 0 01-.735 1.12 3.138 3.138 0 01-1.071.693 3.488 3.488 0 01-1.281.238c-.448 0-.875-.08-1.281-.238a3.138 3.138 0 01-1.071-.693 3.316 3.316 0 01-.735-1.12 3.932 3.932 0 01-.273-1.505zm1.19 0c0 .392.051.747.154 1.064.103.317.25.59.441.819.191.229.42.406.686.532.266.126.562.189.889.189.327 0 .623-.063.889-.189s.495-.303.686-.532c.191-.229.338-.502.441-.819.103-.317.154-.672.154-1.064s-.051-.749-.154-1.071c-.103-.322-.25-.6-.441-.833a1.968 1.968 0 00-.686-.539 2.05 2.05 0 00-.889-.189 2.05 2.05 0 00-.889.189 1.968 1.968 0 00-.686.539 2.491 2.491 0 00-.441.833 3.513 3.513 0 00-.154 1.071zm8.372 4.522c0 .355.189.644.567.868.378.224.931.336 1.659.336.392 0 .747-.04 1.064-.119.317-.08.588-.184.812-.315.224-.13.394-.28.511-.448a.898.898 0 00.175-.518c0-.327-.128-.546-.385-.658-.257-.112-.646-.168-1.169-.168h-1.33c-.196 0-.38-.007-.553-.021a2.216 2.216 0 01-.483-.091c-.327.177-.553.362-.679.553a1.047 1.047 0 00-.189.581zm-.994.112c0-.27.096-.537.287-.798.191-.261.469-.5.833-.714v-.056a1.61 1.61 0 01-.49-.42c-.14-.177-.21-.41-.21-.7 0-.215.07-.436.21-.665.14-.229.34-.432.602-.609v-.056a2.233 2.233 0 01-.595-.707c-.163-.294-.245-.646-.245-1.057a2.283 2.283 0 01.798-1.771 2.67 2.67 0 01.847-.49 2.997 2.997 0 011.029-.175c.373 0 .71.056 1.008.168h2.814v.882h-1.82c.159.168.303.373.434.616.13.243.196.513.196.812 0 .364-.068.69-.203.98-.135.29-.322.534-.56.735-.238.2-.518.355-.84.462a3.233 3.233 0 01-1.029.161c-.177 0-.366-.021-.567-.063a2.819 2.819 0 01-.581-.189c-.355.224-.532.485-.532.784 0 .27.124.469.371.595s.595.189 1.043.189h1.526c.877 0 1.528.121 1.953.364.425.243.637.653.637 1.232 0 .317-.089.62-.266.91-.177.29-.43.541-.756.756a4.237 4.237 0 01-1.19.518c-.467.13-.99.196-1.568.196-.999 0-1.771-.163-2.317-.49-.546-.327-.819-.793-.819-1.4zm3.066-4.172c.439 0 .81-.145 1.113-.434.303-.29.455-.681.455-1.176a1.618 1.618 0 00-.462-1.148 1.574 1.574 0 00-1.106-.448 1.574 1.574 0 00-1.106.448 1.618 1.618 0 00-.462 1.148c0 .495.154.887.462 1.176.308.29.677.434 1.106.434zm5.502 4.802c.093.028.194.051.301.07.107.019.212.028.315.028.476 0 .863-.138 1.162-.413.299-.275.532-.623.7-1.043l.21-.504-3.178-6.818h1.162l1.666 3.766.413.966.427 1.008h.056l.728-1.974 1.484-3.766h1.092l-2.996 7.364c-.13.336-.28.649-.448.938-.168.29-.369.539-.602.749a2.68 2.68 0 01-.805.497 2.751 2.751 0 01-1.029.182c-.336 0-.635-.047-.896-.14l.238-.91z"/>
</g>
</svg>
</a>
<!-- Site links -->
<ul class="footer-nav list--plain monospace">
<li class="footer-nav__item">
<a class="footer-link active" href="/blog/">Blog</a>
</li>
<li class="footer-nav__item">
<a class="footer-link " href="/feed.xml">RSS</a>
</li>
<li class="footer-nav__item">
<a class="footer-link " href="/projects/">Projects</a>
</li>
<li class="footer-nav__item">
<a class="footer-link " href="/careers/">Careers</a>
</li>
</ul>
<!-- Social icons-->
<ul class="footer-social-list list--inline">
<li class="footer-social-list__item">
<a class="footer-link no-underline" href="https://www.facebook.com/scribd">
<svg class="svg-icon"><use xlink:href="/assets/images/icons/icon-sprite.svg#facebook"></use></svg>
<span class="visually-hidden">Scribd on Facebook</span>
</a>
</li>
<li class="footer-social-list__item">
<a class="footer-link no-underline" href="https://www.linkedin.com/company/scribd">
<svg class="svg-icon"><use xlink:href="/assets/images/icons/icon-sprite.svg#linkedin"></use></svg>
<span class="visually-hidden">Scribd on LinkedIn</span>
</a>
</li>
<li class="footer-social-list__item">
<a class="footer-link no-underline" href="https://github.com/scribd">
<svg class="svg-icon"><use xlink:href="/assets/images/icons/icon-sprite.svg#github"></use></svg>
<span class="visually-hidden">Scribd on Github</span>
</a>
</li>
</ul>
</div>
<!-- Secondary Footer -->
<section class="footer-secondary">
<div class="footer-secondary__container container">
<!-- Open Source Links -->
<ul class="open-source-nav list--inline fs-md monospace">
<li class="open-source-nav__item">
<a class="footer-link" href="https://github.com/scribd/scribd.github.io/edit/main/_posts/2020-12-21-sidekiq-incident-learnings.md">Improve this Page</a>
</li>
<li class="open-source-nav__item">
<a class="footer-link" href="https://github.com/scribd/scribd.github.io/commits/main/_posts/2020-12-21-sidekiq-incident-learnings.md">Page History</a>
</li>
</ul>
<!-- License Info -->
<p class="footer-license">
<a href="https://creativecommons.org/licenses/by-sa/4.0/">
<img class="footer-license__img" alt="Creative Commons Attribution-ShareAlike license" src="https://licensebuttons.net/l/by-sa/4.0/88x31.png">
</a>
This work is licensed under a Creative Commons Attribution-Share Alike 4.0 International License
</p>
</div>
</section>
</footer>
<script type="text/javascript" src="/assets/js/main.js"></script>
<script type="text/javascript" src="/assets/js/jobs.js"></script>
<script>
if(!(window.doNotTrack === "1" || navigator.doNotTrack === "1" || navigator.doNotTrack === "yes" || navigator.msDoNotTrack === "1")) {
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
ga('create', 'UA-443684-30', 'auto');
ga('send', 'pageview');
}
</script>
</body>
</html>