Files
nexus/sreweekly/articles/68/03-who-owns-on-call.html
2026-09-12 17:23:01 +08:00

19 lines
31 KiB
HTML
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!doctype html><html><head><meta charset=utf-8><title>Who owns on-call? – Increment: On-Call</title><meta name=description content='Industry leaders like Google, Amazon, Dropbox, Spotify, and Netflix are putting developers on-call for their software. We spoke to these companies to find out why.'><link rel=canonical href=http://localhost:3000/on-call/who-owns-on-call/ ><link rel=apple-touch-icon-precomposed href=/img/icon-571805a1.png><meta property=og:title content='Who owns on-call? – Increment: On-Call'><meta property=og:url content=http://localhost:3000/on-call/who-owns-on-call/ ><meta property=og:description content='Industry leaders like Google, Amazon, Dropbox, Spotify, and Netflix are putting developers on-call for their software. We spoke to these companies to find out why.'><meta property=og:image content='https://images.ctfassets.net/3njn2qm7rrbs/7F3e4wzisAhWIob74WcGP3/8f3990eab66d19965f11dbc9fe966c19/rotation.png?w=1000'><meta name=twitter:card content=summary_large_image><meta name=twitter:image content='https://images.ctfassets.net/3njn2qm7rrbs/7F3e4wzisAhWIob74WcGP3/8f3990eab66d19965f11dbc9fe966c19/rotation.png?w=1000'><meta name=twitter:site content=@IncrementMag><meta name=twitter:title content='Who owns on-call? – Increment: On-Call'><meta name=twitter:description content='Industry leaders like Google, Amazon, Dropbox, Spotify, and Netflix are putting developers on-call for their software. We spoke to these companies to find out why.'><link rel=alternate type=application/rss+xml title=Increment href=/feed.xml><meta name=viewport content='width=device-width,initial-scale=1'><link rel=preload href=/fonts/baton-turbo/400-30a55d66.woff2 as=font type=font/woff2 crossorigin=anonymous><link rel=preload href=/fonts/baton-turbo/500-1603c0e8.woff2 as=font type=font/woff2 crossorigin=anonymous><link rel=preload href=/fonts/tiempos-text/400-c4810745.woff2 as=font type=font/woff2 crossorigin=anonymous><link rel=preload href=/fonts/tiempos-head/700-383ede62.woff2 as=font type=font/woff2 crossorigin=anonymous><link rel=stylesheet type=text/css href=/css/bundle-289d885f.css><link rel=stylesheet type=text/css href=/css/issues/1-a2738f50.css><script>// Don't fade art if it loads ~instantly
setTimeout(()=>{document.documentElement.classList.add('fadeArt')},250);</script><script defer src=/js/defer-737fbd90.js></script><script>const INCREMENT_META={issueNumber:1,issueSlug:'on-call',articleSlug:'who-owns-on-call'};</script></head><body class='Issue_on-call Article_who-owns-on-call'><nav class=PageNav><div class=u-Container><div class=column><h1 class=logo><a href=/ ><img src=/img/logo-ae2c55d5.svg alt=Increment></a></h1><a class=out-now href=https://store.increment.com/ style=color:#595959><div class='IssueTitle tiny'><div class='t-Caps meta tiny'><span>NEW</span></div><h3 class='t-IssueTitle title'>Buy the print edition</h3></div></a><ul class=nav><li><a href=/issues/ ><span>Issues</span></a></li><li><a href=/topics/ ><span>Topics</span></a></li><li><a href=https://store.increment.com/ ><span>Store</span></a></li><li><a href=/about/ ><span>About</span></a></li></ul></div></div></nav><div class=ArticlePage itemscope itemtype=http://schema.org/Article><script type=application/ld+json>{
"@context": "http://schema.org",
"@type": "Article",
"headline": "Who owns on-call?",
"image": " https://images.ctfassets.net/3njn2qm7rrbs/7F3e4wzisAhWIob74WcGP3/8f3990eab66d19965f11dbc9fe966c19/rotation.png?w&#x3D;1000",
"datePublished": "Thu, 13 Apr 2017 09:00:00 GMT",
"dateModified": "Thu, 13 Apr 2017 09:00:00 GMT",
"publisher": {
"@type": "Organization",
"name": "Increment",
"logo": {
"@type": "ImageObject",
"url": "https://increment.com/img/logo.png"
}
},
"description": "Industry leaders like Google, Amazon, Dropbox, Spotify, and Netflix are putting developers on-call for their software. We spoke to these companies to find out why.",
"mainEntityOfPage": "http://localhost:3000/on-call/who-owns-on-call/"
}</script><header class='u-Container ArticleHeader'><div class='u-Container art'><div class=u-Art><div class=placeholder style=background:#f2f2f2></div><picture><source srcset='https://images.ctfassets.net/3njn2qm7rrbs/7F3e4wzisAhWIob74WcGP3/8f3990eab66d19965f11dbc9fe966c19/rotation.png?w=2000 2000w, https://images.ctfassets.net/3njn2qm7rrbs/7F3e4wzisAhWIob74WcGP3/8f3990eab66d19965f11dbc9fe966c19/rotation.png?w=1000 1000w, https://images.ctfassets.net/3njn2qm7rrbs/7F3e4wzisAhWIob74WcGP3/8f3990eab66d19965f11dbc9fe966c19/rotation.png?w=500 500w, https://images.ctfassets.net/3njn2qm7rrbs/7F3e4wzisAhWIob74WcGP3/8f3990eab66d19965f11dbc9fe966c19/rotation.png?w=100 100w' sizes=100vw><img class=j-SmoothLoad src='https://images.ctfassets.net/3njn2qm7rrbs/7F3e4wzisAhWIob74WcGP3/8f3990eab66d19965f11dbc9fe966c19/rotation.png?w=1000' sizes=100vw alt='Who owns on-call?'></picture></div></div><div class=column><div class=u-Grid><div class=main><h4 class='t-Byline byline large'><a href=#authors class=j-SmoothScroll itemprop=author><span>Increment Staff</span></a></h4><h1 class='t-TitleSerif large title' itemprop=name>Who owns on-call?</h1><div class='t-BodySans large intro' itemprop=description>Industry leaders like Google, Amazon, Dropbox, Spotify, and Netflix are putting developers on-call for their software. We spoke to these companies to find out&nbsp;why.</div></div><a class=issue href=/on-call/ ><span class='t-Caps tiny part-of'>Part of</span><div class='IssueTitle small' style=color:#ef766e><div class='t-Caps meta'><span>Issue 1</span> <span>April 2017</span></div><h2 class='t-IssueTitle title'>On-Call</h2></div></a></div></div></header><div class='u-Container ArticleContent'><article class='ContentBody column' itemprop=articleBody><div class=ArticleLayout><p>When something goes wrong with a software application, service, or system, someone needs to be responsible for figuring out what went wrong and fixing it. In the tech industry, this set of tasks is usually referred to as “being on-call” for that software. Similarly to the practice of doctors being on-call at a hospital, a set of engineers is placed on an <i>on-call rotation</i> (meaning that they share the on-call responsibility with a team, and everyone on the team takes turns being on the rotation), during their on-call shifts they are <i>paged</i> any time something breaks (usually via an automated push notification on their smartphone, a text, or a call), and they are responsible for quickly responding to the page, fixing what broke, and making sure that the same problem never happens again. On-call engineers are the “first responders” of software engineering.</p><p>Historically, the responsibilities required to run large software applications and systems have been divvied up between two kinds of teams: so-called “development” teams, who are responsible for all tasks associated with building and adding new features to applications and systems, and so-called “operational” teams, who are responsible for running and maintaining them. On-call responsibilities have been viewed for a long time as being part of the operational workload, and developers have rarely been on-call for the software they build.</p><p>In the past several years, everything in the industry changed. It’s difficult to pinpoint exactly when the industry changed its mind about on-call responsibilities, but the “who”, the “where”, and the “why” are relatively straightforward to uncover and understand. To determine the state of the industry, <i>Increment</i> spoke with over thirty industry leaders about the “who” and the “why”, and what we learned from our conversations about the industry-wide movement to put developers on-call for their software.</p><p>The majority of the companies we surveyed used to divide engineering tasks between their technical teams in the old way: their development teams wrote the code (and sometimes did the testing), and then threw the debugging, the testing, the running, and the maintenance of the code over to an operational team. Over the past few years, most of these companies discovered that this approach to running software simply didn’t scale, and that developers felt a lack of ownership when they weren’t on-call for the code they wrote—most importantly, this lack of ownership translated into unreliable systems being built and run. To fix these scalability and reliability problems, they moved the operational workload onto the development teams, who quickly (though not painlessly) learned to build better, more resilient systems.</p><p><strong>Google</strong> notoriously was one of the first companies in the tech world to realize that the old way of doing operational work wouldn’t and couldn’t scale at the level their systems required, so they created a new role for “Site Reliability Engineers” (SREs). These new SREs approached the operational tasks with a software engineering mindset: they automated away all of the operational grunt-work, and made the systems run more reliably. Nowadays, SREs at Google run, maintain, and are on-call only for the most important and stable services (like Ads, Gmail, and Search), while development teams carry the operational workload for other non-stable, non-critical services (which aren’t staffed by SREs). The SRE approach to operations is now credited with the success of Google’s systems—success that much of the industry has tried to emulate by adopting the SRE role and practices. However, many industry adopters have taken the SRE title without also adopting the SRE mindset or Google’s requirement that SREs only run and maintain stable systems: Google requires development teams to run their own services if those systems <i>aren’t</i> stable.</p><p><strong>Spotify</strong> was one of the companies that adopted the SRE role early on, and treated SREs as typical operations engineers. In Spotify’s early days, their small SRE team was responsible for all operational work, including being on-call for all Spotify systems. As the company grew, and the operational workload grew alongside it, Spotify’s leadership discovered that they couldn’t hire SREs quickly enough to meet the operational demands. The only scalable solution they found was moving the on-call responsibilities to the development teams.</p><p><strong>Airbnb</strong> discovered that having a separate operations team “creates a divide and simply doesn’t scale,” says Airbnb SRE manager Joey Parsons, and “it puts the onus of responsibility for fixing an issue on the wrong team.” Airbnb decided to put developers on-call for their systems, taking the stance that if developers can deploy to production whenever they want, then they should be the ones fixing problems caused by their services and deployments. Though Airbnb has SREs that work closely with development teams, their SREs focus only on improving reliability across systems, and they are the only team that is <i>not</i> on-call for any of Airbnb’s services. Many other companies, like Pinterest and New Relic, have followed a similar approach to that of Airbnb: developers are on-call for their services, but have SREs working alongside them (usually “embedded” within the team) to make sure that the development teams are following industry best practices for on-call and general service reliability.</p><div class='Sidebar large'><blockquote><p>Airbnb discovered that having a separate operations team “creates a divide and simply doesn’t scale.”</p></blockquote></div><p>Some companies—like <strong>Datadog</strong>, <strong>Digital Ocean</strong>, and <strong>Dropbox</strong>—have focused on taking a shared, holistic approach to on-call responsibilities, and have put both development and operations teams on-call for services together. At Datadog, engineering leadership was determined to avoid an ops/dev split from the very start, and so they ensured that operational tasks were distributed between ops and dev teams. Importantly, SREs and developers at Datadog share the on-call rotations, ensuring that every on-call shift is staffed by both experts in the code (developers) and experts in reliability (the SREs). Dropbox takes a similar approach, viewing on-call responsibility as something that both development and SRE teams need to own. DigitalOcean has both development teams <i>and</i> operational teams on-call, but with a twist: development teams are on-call for their services, while operations teams are on-call for the interactions <i>between</i> the services.</p><p><strong>PagerDuty</strong>, on the other hand, has what engineering manager Sweta Ackerman refers to as a “you build it you own it” and “end-to-end ownership” model: SREs are on-call and responsible for low-level infrastructure (like hardware, middleware, communication, databases, etc), while developers are on-call and responsible for everything on top of that infrastructure (including development, deployment, monitoring, and the hardware they run their services on). Ackerman says that PagerDuty had to switch to the shared-responsibility model two years ago, in an effort to ship features more quickly, encourage teams to “control their own destinies,” and to “reduce [inter-team] dependencies”—a model that the company has found wildly successful.</p><p><strong>Amazon</strong> is famous (or, rather, infamous) for practically doing away with the operational role altogether, and were one of the first industry leaders to do so. Throughout all engineering organizations at Amazon (including AWS), developers are responsible for all development and operational tasks associated with their services. Putting the onus on developers to run, maintain, and be on-call for their services is part of Amazon’s cultural emphasis on “ownership”: you don’t “own” the code you write, Amazon says, unless you run and maintain it, too.</p><div class='Sidebar large'><blockquote><p>Putting the onus on developers to run, maintain, and be on-call for their services is part of Amazon’s cultural emphasis on “ownership”: you don’t “own” the code you write, Amazon says, unless you run and maintain it, too.</p></blockquote></div><p><strong>Netflix</strong> takes an approach similar to Amazon’s, with the motto “You own it, you run it.” Development teams at Netflix are on-call for their services 24/7, and there’s a Core SRE team that monitors services at a very high level and engages development teams only when large-scale outages occur. According to Netflix SRE Manager Blake Scrivener, “When something goes wrong [at Netflix], which our automation doesn’t handle correctly, we want the experts in the service to be immediately available to make the repair and [bring] stability to the customer experience…when things are broken, we want people with the best context trying to fix things.” In an engineering environment where services are being deployed multiple times a day, the people with the best context are almost always the development teams.</p><p>Out of all of the companies we surveyed, only <strong>Slack</strong> still had anything resembling an old-school operations team. Slack’s operations team, which is on-call for all of Slack’s services, is spread across the globe and uses a follow-the-sun rotation, with operations engineers located in Melbourne, Dublin, and San Francisco. “The decision to put operators on-call as the first responders is as old as the company itself,” says Richard Crowley, Director of Operations at Slack, because “historically, the things that broke tended to ultimately have contributing factors like hardware failures or network partitions.” Crowley says that they’ve recently started to see scalability problems with the old way of operations, however, which led Slack to create a secondary on-call rotation full of developers; software and performance bugs, he says, are becoming much more common than low-level infrastructure problems—bugs that only the development teams know how to fix. Given the industry trend, we don’t think it’ll be long before Slack joins the rest of the industry and puts their development teams on-call for all of their services.</p></div></article></div><div class='u-Container ArticleFooter'><div class=column><div class='u-Grid ContentBody small content'><div class=authors id=authors><div><div class=photo><figure style='background-image:url(https://images.ctfassets.net/3njn2qm7rrbs/3buorQfLVwQiD58sXNILJE/d59d2cbaf87ce0d40e499f26f586ad77/pager.png?w=500)'></figure></div><div class=text><h4>Artwork by</h4><p><b>Mark Conlan</b></p><p><a href=https://markconlan.com/ target=_blank>markconlan.com</a></p></div></div></div><div class=topics><div class=text><h4>Topics</h4></div></div></div></div></div></div><div class=SubscribeBox><a id=newsletter class=anchor href=#newsletter></a><div class='ContentBody inverted store'><div class=u-Container><div class='u-Grid column'><div class=box style=background:#4c70b1><div class=text><h2>Buy the print edition</h2><p class=j-TextBalance>Visit the Increment Store to purchase print issues.</p><p><a class='t-Caps u-Arrow' href=https://store.increment.com/ >Store</a></p></div><a href=https://store.increment.com/ class=magazine><figure class=j-MaskedImage data-fill=/art/19/19-cutout-1000-7ffb5dba.png data-mask=/art/19/fill-cms-1000-83522f38.png></figure></a></div></div></div></div><div class='ContentBody email'><div class=u-Container><div class='u-Grid column'><div class='j-EmailForm box' style='box-shadow:0 -5px 0 #4c70b1'></div></div></div></div></div><div class=ContinueReading><div class=u-Container><div class=column><h2 class='t-Caps xlarge'>Continue Reading</h2><ul class='u-Grid articles'><li class='ArticleBlock footer' style=color:#ef766e><a class=j-Preload href=/on-call/when-the-pager-goes-off/ ><div class='IssueTitle tiny' style=color:#ef766e><div class='t-Caps meta'><span>1</span></div><h3 class='t-IssueTitle title'>On-Call</h3></div><div class=text><div class=head><h4 class='t-Byline byline'><span>Increment Staff</span></h4><h3 class='t-TitleSans title'>What happens when the pager goes off?</h3></div><div class='t-BodySerif small intro'>To discover the state of incident response across the tech industry, we surveyed over thirty industry leaders (including Amazon, Dropbox, Facebook, Google, and Netflix) about their incident response processes.</div></div></a></li><li class='ArticleBlock footer' style=color:#ef766e><a class=j-Preload href=/on-call/the-benefits-of-transparency/ ><div class='IssueTitle tiny' style=color:#ef766e><div class='t-Caps meta'><span>1</span></div><h3 class='t-IssueTitle title'>On-Call</h3></div><div class=text><div class=head><h4 class='t-Byline byline'><span>Increment Staff</span></h4><h3 class='t-TitleSans title'>The benefits of transparency: Interview with Sytse “Sid” Sijbrandij, CEO of GitLab</h3></div><div class='t-BodySerif small intro'>We spoke with Sid about GitLab’s recent outage, its aftermath, and the impressive level of transparency GitLab offered the public during the&nbsp;outage.</div></div></a></li><li class='ArticleBlock footer' style=color:#ef766e><a class=j-Preload href=/on-call/ask-an-expert/ ><div class='IssueTitle tiny' style=color:#ef766e><div class='t-Caps meta'><span>1</span></div><h3 class='t-IssueTitle title'>On-Call</h3></div><div class=text><div class=head><h4 class='t-Byline byline'><span>Increment Staff</span></h4><h3 class='t-TitleSans title'>Ask an expert: How should startups approach on-call and incident response?</h3></div><div class='t-BodySerif small intro'>We asked several industry experts if they had any advice for small companies who are just starting to set up their on-call and incident response processes, and here’s what they&nbsp;said.</div></div></a></li><li class='ArticleBlock footer' style=color:#ef766e><a class=j-Preload href=/on-call/on-call-at-any-size/ ><div class='IssueTitle tiny' style=color:#ef766e><div class='t-Caps meta'><span>1</span></div><h3 class='t-IssueTitle title'>On-Call</h3></div><div class=text><div class=head><h4 class='t-Byline byline'><span>Increment Staff</span></h4><h3 class='t-TitleSans title'>On-call at any size</h3></div><div class='t-BodySerif small intro'>We take a close look at how to make on-call work at any scale, sharing industry best practices that apply to companies at any size, from tiny startups in garages to companies the size of Amazon, Facebook, and&nbsp;Google.</div></div></a></li><li class='ArticleBlock footer' style=color:#53a88e><a class=j-Preload href=/development/what-its-like-to-be-a-developer-at/ ><div class='IssueTitle tiny' style=color:#53a88e><div class='t-Caps meta'><span>3</span></div><h3 class='t-IssueTitle title'>Development</h3></div><div class=text><div class=head><h4 class='t-Byline byline'><span>Increment Staff</span></h4><h3 class='t-TitleSans title'>What it’s like to be a developer at …</h3></div><div class='t-BodySerif small intro'>From popular tools to code review, deployment to daily life, here’s a look at the developer experience at Slack, Lyft, DigitalOcean, and&nbsp;more.</div></div></a></li><li class='ArticleBlock footer' style=color:#707aed><a class=j-Preload href=/cloud/interview-with-ben-uretsky-julia-austin-digitalocean/ ><div class='IssueTitle tiny' style=color:#707aed><div class='t-Caps meta'><span>2</span></div><h3 class='t-IssueTitle title'>Cloud</h3></div><div class=text><div class=head><h4 class='t-Byline byline'><span>Increment Staff</span></h4><h3 class='t-TitleSans title'>An interview with Ben Uretsky and Julia Austin, CEO and CTO of DigitalOcean</h3></div><div class='t-BodySerif small intro'>How DigitalOcean got started, how they managed to position it in the market, how they overcame the technical and economic challenges they faced along the way, and what they’re planning for the&nbsp;future.</div></div></a></li><li class='ArticleBlock footer' style=color:#d69336><a class=j-Preload href=/programming-languages/six-questions-on-programming-languages/ ><div class='IssueTitle tiny' style=color:#d69336><div class='t-Caps meta'><span>5</span></div><h3 class='t-IssueTitle title'>Programming Languages</h3></div><div class=text><div class=head><h4 class='t-Byline byline'><span>Increment Staff</span></h4><h3 class='t-TitleSans title'>Six questions on programming languages</h3></div><div class='t-BodySerif small intro'>Engineers at Fastly, Glossier, Optimizely, and more share which languages they use (and love!), how language knowledge factors into hiring, and where they see their current languages&nbsp;heading.</div></div></a></li><li class='ArticleBlock footer' style=color:#4a5ad3><a class=j-Preload href=/open-source/open-source-at-scale/ ><div class='IssueTitle tiny' style=color:#4a5ad3><div class='t-Caps meta'><span>9</span></div><h3 class='t-IssueTitle title'>Open Source</h3></div><div class=text><div class=head><h4 class='t-Byline byline'><span>Increment Staff</span></h4><h3 class='t-TitleSans title'>Open source at scale</h3></div><div class='t-BodySerif small intro'>Technical leaders at Microsoft, Kickstarter, DigitalOcean, and Red Hat answer questions about when and why to opt for OSS, how open source has influenced their organizations, and its future role in corporations.</div></div></a></li><li class='ArticleBlock footer' style=color:#e89e00><a class=j-Preload href=/testing/testing-at-scale/ ><div class='IssueTitle tiny' style=color:#e89e00><div class='t-Caps meta'><span>10</span></div><h3 class='t-IssueTitle title'>Testing</h3></div><div class=text><div class=head><h4 class='t-Byline byline'><span>Increment Staff</span></h4><h3 class='t-TitleSans title'>Testing at scale</h3></div><div class='t-BodySerif small intro'>Rob Zuber (CircleCI), Greg Bell and Claudiu Coman (Hootsuite), and Scott Triglia (Yelp) talk test suite times, manual versus automated testing, and how to build testing infrastructure.</div></div></a></li></ul><h3 class='t-Caps large'>Explore Topics</h3><ul class='t-BodySerif large topics'><li><a href=/topics/learn/ >Learn Something New</a></li><li><a href=/topics/scaling/ >Scaling &amp; Growth</a></li><li><a href=/topics/ask-an-expert/ >Ask an Expert</a></li><li><a href=/topics/interviews/ >Interviews &amp; Surveys</a></li><li><a href=/topics/guides/ >Guides &amp; Best Practices</a></li><li><a href=/topics/opinion/ >Essays &amp; Opinion</a></li><li><a href=/topics/culture/ >Workplace &amp; Culture</a></li></ul><h3 class='t-Caps large'>All Issues</h3><ul class=issues><li><a href=/planning/ ><div class='IssueTitle large' style=color:#4c70b1><div class='t-Caps meta'><span>Issue 19</span> <span>November 2021</span></div><h1 class='t-IssueTitle title'>Planning</h1></div></a></li><li><a href=/mobile/ ><div class='IssueTitle large' style=color:#439eab><div class='t-Caps meta'><span>Issue 18</span> <span>August 2021</span></div><h1 class='t-IssueTitle title'>Mobile</h1></div></a></li><li><a href=/containers/ ><div class='IssueTitle large' style=color:#443d79><div class='t-Caps meta'><span>Issue 17</span> <span>May 2021</span></div><h1 class='t-IssueTitle title'>Containers</h1></div></a></li><li><a href=/reliability/ ><div class='IssueTitle large' style=color:#863051><div class='t-Caps meta'><span>Issue 16</span> <span>February 2021</span></div><h1 class='t-IssueTitle title'>Reliability</h1></div></a></li><li><a href=/remote/ ><div class='IssueTitle large' style=color:#29386a><div class='t-Caps meta'><span>Issue 15</span> <span>November 2020</span></div><h1 class='t-IssueTitle title'>Remote</h1></div></a></li><li><a href=/apis/ ><div class='IssueTitle large' style=color:#00afbe><div class='t-Caps meta'><span>Issue 14</span> <span>August 2020</span></div><h1 class='t-IssueTitle title'>APIs</h1></div></a></li><li><a href=/frontend/ ><div class='IssueTitle large' style=color:#5ebe92><div class='t-Caps meta'><span>Issue 13</span> <span>May 2020</span></div><h1 class='t-IssueTitle title'>Frontend</h1></div></a></li><li><a href=/software-architecture/ ><div class='IssueTitle large' style=color:#40af9e><div class='t-Caps meta'><span>Issue 12</span> <span>February 2020</span></div><h1 class='t-IssueTitle title'>Software Architecture</h1></div></a></li><li><a href=/teams/ ><div class='IssueTitle large' style=color:#8e65bf><div class='t-Caps meta'><span>Issue 11</span> <span>November 2019</span></div><h1 class='t-IssueTitle title'>Teams</h1></div></a></li><li><a href=/testing/ ><div class='IssueTitle large' style=color:#e89e00><div class='t-Caps meta'><span>Issue 10</span> <span>August 2019</span></div><h1 class='t-IssueTitle title'>Testing</h1></div></a></li><li><a href=/open-source/ ><div class='IssueTitle large' style=color:#4a5ad3><div class='t-Caps meta'><span>Issue 9</span> <span>May 2019</span></div><h1 class='t-IssueTitle title'>Open Source</h1></div></a></li><li><a href=/internationalization/ ><div class='IssueTitle large' style=color:#c096ca><div class='t-Caps meta'><span>Issue 8</span> <span>February 2019</span></div><h1 class='t-IssueTitle title'>Internationalization</h1></div></a></li><li><a href=/security/ ><div class='IssueTitle large' style=color:#4dbac5><div class='t-Caps meta'><span>Issue 7</span> <span>October 2018</span></div><h1 class='t-IssueTitle title'>Security</h1></div></a></li><li><a href=/documentation/ ><div class='IssueTitle large' style=color:#f5684d><div class='t-Caps meta'><span>Issue 6</span> <span>August 2018</span></div><h1 class='t-IssueTitle title'>Documentation</h1></div></a></li><li><a href=/programming-languages/ ><div class='IssueTitle large' style=color:#d69336><div class='t-Caps meta'><span>Issue 5</span> <span>April 2018</span></div><h1 class='t-IssueTitle title'>Programming Languages</h1></div></a></li><li><a href=/energy-environment/ ><div class='IssueTitle large' style=color:#d6658e><div class='t-Caps meta'><span>Issue 4</span> <span>February 2018</span></div><h1 class='t-IssueTitle title'>Energy & Environment</h1></div></a></li><li><a href=/development/ ><div class='IssueTitle large' style=color:#53a88e><div class='t-Caps meta'><span>Issue 3</span> <span>October 2017</span></div><h1 class='t-IssueTitle title'>Development</h1></div></a></li><li><a href=/cloud/ ><div class='IssueTitle large' style=color:#707aed><div class='t-Caps meta'><span>Issue 2</span> <span>July 2017</span></div><h1 class='t-IssueTitle title'>Cloud</h1></div></a></li><li><a href=/on-call/ ><div class='IssueTitle large' style=color:#ef766e><div class='t-Caps meta'><span>Issue 1</span> <span>April 2017</span></div><h1 class='t-IssueTitle title'>On-Call</h1></div></a></li></ul></div></div></div><footer class=PageFooter><div class='u-Container ContentBody small'><svg style=display:none><symbol id=twitterIcon viewBox='0 0 32 32'><path d='M32.1 6c-1.2.5-2.5.9-3.8 1 1.4-.8 2.4-2.1 2.9-3.6-1.3.8-2.7 1.3-4.2 1.6a6.8 6.8 0 0 0-4.8-2c-3.6 0-6.6 3-6.6 6.6 0 .5.1 1 .2 1.5-5.5-.3-10.4-3-13.6-6.9-.6 1-.9 2.1-.9 3.3 0 2.3 1.2 4.3 2.9 5.5-1.1 0-2.1-.3-3-.8v.1c0 3.2 2.3 5.9 5.3 6.5-.6.2-1.1.2-1.7.2-.4 0-.8 0-1.2-.1.8 2.6 3.3 4.5 6.2 4.6-2.3 1.8-5.1 2.8-8.2 2.8-.5 0-1.1 0-1.6-.1C2.9 28 6.3 29 10 29c12.1 0 18.7-10 18.7-18.7v-.9c1.4-.9 2.5-2 3.4-3.4z' fill=currentColor /></symbol><symbol id=facebookIcon viewBox='0 0 32 32'><path d='M30.2 0H1.8C.8 0 0 .8 0 1.8v28.5c0 1 .8 1.8 1.8 1.8h15.3V19.6h-4.2v-4.8h4.2v-3.6c0-4.1 2.5-6.4 6.2-6.4 1.8 0 3.3.2 3.7.2v4.3h-2.6c-2 0-2.4 1-2.4 2.4v3.1h4.8l-.6 4.8H22V32h8.2c1 0 1.8-.8 1.8-1.8V1.8c0-1-.8-1.8-1.8-1.8z' fill=currentColor /></symbol><symbol id=rssIcon viewBox='0 0 32 32'><path d='M10.7 25.6c0 2.4-2 4.4-4.4 4.4S2 28 2 25.6s2-4.4 4.4-4.4 4.3 2 4.3 4.4zM6.1 2c-.6 0-1.3 0-2 .1-1.3.1-2.2 1.2-2.1 2.4.1 1.2 1.2 2.2 2.4 2.1.6 0 1.1-.1 1.6-.1 10.7 0 19.4 8.7 19.4 19.4 0 .5 0 1-.1 1.6-.1 1.2.8 2.3 2.1 2.4h.2c1.2 0 2.1-.9 2.2-2.1.1-.7.1-1.4.1-2C30 12.7 19.3 2 6.1 2zm-.7 9.6c-.4 0-.8 0-1.3.1-1.3.1-2.2 1.2-2.1 2.5.1 1.3 1.2 2.2 2.5 2.1h.9c5.7 0 10.4 4.7 10.4 10.4v.9c-.1 1.3.8 2.4 2.1 2.5h.2c1.2 0 2.2-.9 2.3-2.1 0-.4.1-.8.1-1.2-.1-8.4-6.9-15.2-15.1-15.2z' fill=currentColor /></symbol><symbol id=linkedInIcon viewBox='0 0 32 32'><path d='M29.6,0H2.4C1.1,0,0,1,0,2.3v27.4C0,31,1.1,32,2.4,32h27.3c1.3,0,2.4-1,2.4-2.3V2.3C32,1,30.9,0,29.6,0z M9.5,27.3H4.7V12 h4.8V27.3z M7.1,9.9c-1.5,0-2.8-1.2-2.8-2.8c0-1.5,1.2-2.8,2.8-2.8c1.5,0,2.8,1.2,2.8,2.8C9.9,8.7,8.6,9.9,7.1,9.9z M27.3,27.3 h-4.7v-7.4c0-1.8,0-4-2.5-4c-2.5,0-2.8,1.9-2.8,3.9v7.6h-4.7V12H17v2.1h0.1c0.6-1.2,2.2-2.5,4.5-2.5c4.8,0,5.7,3.2,5.7,7.3V27.3z' fill=currentColor /></symbol></svg><div class='column main'><section class=social><a href=https://twitter.com/incrementmag class=twitter><svg viewBox='0 0 32 32'><use xlink:href=#twitterIcon x=0 y=0></use></svg> <span>@incrementmag</span> </a><a href=https://facebook.com/incrementmag class=facebook><svg viewBox='0 0 32 32'><use xlink:href=#facebookIcon x=0 y=0></use></svg> <span>incrementmag</span> </a><a href=/feed.xml class=rss><svg viewBox='0 0 32 32'><use xlink:href=#rssIcon x=0 y=0></use></svg> <span>RSS Feed</span></a></section><section><h4>About</h4><p><em>Increment</em> is a print and digital magazine about how teams build and operate software systems at scale. <a href=/about/ >Learn more</a></p></section><section><h4>Work with us</h4><p>Interested in joining the team at Stripe? <a href=https://stripe.com/jobs>View job openings</a></p></section></div><p class='column copyright'><span>&copy; 2022 <em>Increment</em></span> <a href=https://stripe.com>Published by Stripe</a> <a href=https://stripe.com/privacy/media-policy>Privacy policy</a></p></div></footer></body></html>