700 lines
43 KiB
HTML
700 lines
43 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en" dir="ltr" prefix="og: https://ogp.me/ns#">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<meta name="description" content="Why does testing in production get such a bad rap when we all do it? Learn how to do it right.">
|
|
<link rel="canonical" href="https://opensource.com/article/17/8/testing-production">
|
|
<link rel="image_src" href="https://opensource.com/sites/default/files/lead-images/code_computer_development_programming.png">
|
|
<meta name="author" content="Charity">
|
|
<meta property="og:site_name" content="Opensource.com">
|
|
<meta property="og:url" content="https://opensource.com/article/17/8/testing-production">
|
|
<meta property="og:title" content="Testing in production: Yes, you can (and should)">
|
|
<meta property="og:description" content="Why does testing in production get such a bad rap when we all do it? Learn how to do it right.">
|
|
<meta property="og:image" content="https://opensource.com/sites/default/files/lead-images/code_computer_development_programming.png">
|
|
<meta name="twitter:card" content="summary">
|
|
<meta name="twitter:site" content="@opensourceway">
|
|
<meta name="twitter:description" content="Why does testing in production get such a bad rap when we all do it? Learn how to do it right.">
|
|
<meta name="twitter:title" content="Testing in production: Yes, you can (and should)">
|
|
<meta name="twitter:creator" content="@mipsytipsy">
|
|
<meta name="twitter:image" content="https://opensource.com/sites/default/files/lead-images/code_computer_development_programming.png">
|
|
<meta name="Generator" content="Drupal 10 (https://www.drupal.org)">
|
|
<meta name="MobileOptimized" content="width">
|
|
<meta name="HandheldFriendly" content="true">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<link rel="icon" href="/themes/osdc/assets/img/favicons/favicon.ico" type="image/vnd.microsoft.icon">
|
|
|
|
<title>Testing in production: Yes, you can (and should) | Opensource.com</title>
|
|
<link rel="stylesheet" media="all" href="/sites/default/files/css/css_v1Pk9uCDi2UfNhTAnRmgqTOT8FMeyU3TJ1MbtJc3rko.css?delta=0&language=en&theme=osdc&include=eJxVjkEOwyAMBD8E5UmVMVaCYjDCkIi-vrRJDrlYI-1o10WYXai9AL_KZMsxb2p0aKPkPCgZZFAdD06kCgupQankstQEHD9kRAO6hcUDn_yrPAl6W6XaRA0Mw5De3iEqyk51OMmEcomVGBoFi5Ib5Wb2SIe6_30lCZ2vGZSUZq6mTt0dtq09ee3l_jHmMGMKj3nPgtttZAlXlxJUXL9HlWc3">
|
|
<link rel="stylesheet" media="all" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
|
|
<link rel="stylesheet" media="all" href="/sites/default/files/css/css_pwV-EcCwseDS_ZsbU4fuelxCUlfePETEO0FOdvVn9o8.css?delta=2&language=en&theme=osdc&include=eJxVjkEOwyAMBD8E5UmVMVaCYjDCkIi-vrRJDrlYI-1o10WYXai9AL_KZMsxb2p0aKPkPCgZZFAdD06kCgupQankstQEHD9kRAO6hcUDn_yrPAl6W6XaRA0Mw5De3iEqyk51OMmEcomVGBoFi5Ib5Wb2SIe6_30lCZ2vGZSUZq6mTt0dtq09ee3l_jHmMGMKj3nPgtttZAlXlxJUXL9HlWc3">
|
|
<link rel="stylesheet" media="all" href="/sites/default/files/css/css_VoFCR57K3bDFm1q6VI9lyC_QbA1Le-TwFIbKJmkXjKg.css?delta=3&language=en&theme=osdc&include=eJxVjkEOwyAMBD8E5UmVMVaCYjDCkIi-vrRJDrlYI-1o10WYXai9AL_KZMsxb2p0aKPkPCgZZFAdD06kCgupQankstQEHD9kRAO6hcUDn_yrPAl6W6XaRA0Mw5De3iEqyk51OMmEcomVGBoFi5Ib5Wb2SIe6_30lCZ2vGZSUZq6mTt0dtq09ee3l_jHmMGMKj3nPgtttZAlXlxJUXL9HlWc3">
|
|
<link rel="stylesheet" media="all" href="https://fonts.googleapis.com/css2?family=Red+Hat+Text:wght@400;500;700&display=swap">
|
|
<link rel="stylesheet" media="all" href="https://fonts.googleapis.com/css2?family=Red+Hat+Mono:wght@400;500;700&display=swap">
|
|
<link rel="stylesheet" media="all" href="/sites/default/files/css/css_LdYFGLY4yno-Mg9yzhTjOUAxsL7THSG9U8GLP_rERRw.css?delta=6&language=en&theme=osdc&include=eJxVjkEOwyAMBD8E5UmVMVaCYjDCkIi-vrRJDrlYI-1o10WYXai9AL_KZMsxb2p0aKPkPCgZZFAdD06kCgupQankstQEHD9kRAO6hcUDn_yrPAl6W6XaRA0Mw5De3iEqyk51OMmEcomVGBoFi5Ib5Wb2SIe6_30lCZ2vGZSUZq6mTt0dtq09ee3l_jHmMGMKj3nPgtttZAlXlxJUXL9HlWc3">
|
|
|
|
<script src="/libraries/chapeaux--cpx-reporter/cpx-reporter.js?v=0.12.0" type="module"></script>
|
|
<script src="https://www.redhat.com/ma/dpal.js"></script>
|
|
|
|
<script async data-debug data-domain="redhat2.com" id="trustarc" src="https://static.redhat.com/libs/redhat/marketing/latest/trustarc/trustarc.js" type="text/javascript"></script>
|
|
|
|
<script type="module" src="/themes/osdc/node_modules/@patternfly/pfe-modal/dist/pfe-modal.min.js"></script>
|
|
</head>
|
|
<body class="path-node page-node-type-article">
|
|
<a href="#main-content" class="focusable skip-link">
|
|
Skip to main content
|
|
</a>
|
|
|
|
<div class="dialog-off-canvas-main-canvas" data-off-canvas-main-canvas>
|
|
<div class="layout-container">
|
|
|
|
<header>
|
|
<div class="region region-messages">
|
|
<div data-drupal-messages-fallback class="hidden"></div>
|
|
|
|
</div>
|
|
|
|
|
|
<div class="region-top-bar">
|
|
<div class="region wrapper region-user-menu">
|
|
<div class="hamburger__container">
|
|
<div class="hamburger">
|
|
<button id="mobnav-btn" aria-label="Toggle main menu" class="button--plain toggle-menu"></button>
|
|
<div class="bar"></div>
|
|
</div>
|
|
</div>
|
|
|
|
<a href="https://www.redhat.com/en?intcmp=701600000011l7VAAQ" target="_blank" class="flex-center support-logo">
|
|
<img alt="Supported by Red Hat" src="/themes/osdc/assets/img/l_supported-by-redhat-white.svg" title="Supported by Red Hat" width="125" height="56" class="support-logo__image">
|
|
</a>
|
|
|
|
<button id="toggle-search" aria-label="Toggle Search" class="button--plain toggle-search">
|
|
<i class="icon-search"></i>
|
|
</button>
|
|
<nav role="navigation" aria-labelledby="block-useraccountmenu-menu" id="block-useraccountmenu" class="block block-menu navigation menu--account">
|
|
|
|
<h2 class="visually-hidden" id="block-useraccountmenu-menu">User account menu</h2>
|
|
|
|
|
|
|
|
|
|
<ul class="account">
|
|
|
|
<li class="log-in account__item">
|
|
<a href="/user/login?current=/article/17/8/testing-production" class="account__link" data-drupal-link-system-path="user/login">Log in</a>
|
|
</li>
|
|
<li class="rss account__item">
|
|
<a href="/feed" title="RSS" class="account__link" data-drupal-link-system-path="rss.xml">RSS</a>
|
|
</li>
|
|
</ul>
|
|
|
|
|
|
|
|
</nav>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
<div class="region region-header">
|
|
<div id="block-osdc-branding" class="block block-system block-system-branding-block">
|
|
|
|
|
|
<div class="branding">
|
|
<a href="/" rel="home" aria-label="Opensource.com" class="branding__logo">
|
|
<img src="/themes/osdc/logo.svg" alt="Home">
|
|
</a>
|
|
<a href="https://www.redhat.com/en?intcmp=701600000011l7VAAQ" target="_blank" class="branding__supported-by">
|
|
<img src="/themes/osdc/assets/img/l_supported-by-redhat-white.svg" width="125" height="56" class="branding__supported-by__image" alt="Supported by Red Hat">
|
|
</a>
|
|
</div>
|
|
<div class="branding__slogan"></div>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
</header>
|
|
|
|
<div class="region region-main-nav">
|
|
<nav role="navigation" aria-labelledby="block-mainnavigation-menu" id="block-mainnavigation" class="block block-menu navigation menu--main">
|
|
|
|
<h2 class="visually-hidden" id="block-mainnavigation-menu">Main navigation</h2>
|
|
|
|
|
|
|
|
|
|
<ul class="main" id="main-menu">
|
|
|
|
<li class="main__item">
|
|
<a href="/" class="main__link" data-drupal-link-system-path="<front>">Articles</a>
|
|
</li>
|
|
<li class="main__item is-expanded">
|
|
<a href="/resources" class="main__link" data-drupal-link-system-path="node/17752">Resources</a>
|
|
|
|
<ul class="main__submenu">
|
|
|
|
<li class="main__item is-expanded">
|
|
<a href="/resources/what-open-source" class="main__link" data-drupal-link-system-path="node/17753">What is open source?</a>
|
|
|
|
<ul class="main__submenu">
|
|
|
|
<li class="main__item">
|
|
<a href="/open-source-way" class="main__link" data-drupal-link-system-path="node/17730">The open source way</a>
|
|
</li>
|
|
</ul>
|
|
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/resources/projects-and-applications" class="main__link" data-drupal-link-system-path="node/17760">Projects and applications</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/resources/organizations" class="main__link" data-drupal-link-system-path="node/17763">Organizations</a>
|
|
</li>
|
|
<li class="main__item is-expanded">
|
|
<a href="/alternatives" class="main__link" data-drupal-link-system-path="node/30871">Open source alternatives</a>
|
|
|
|
<ul class="main__submenu">
|
|
|
|
<li class="main__item">
|
|
<a href="/alternatives/adobe-acrobat" class="main__link" data-drupal-link-system-path="node/29486">Alternatives to Acrobat</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/autocad" class="main__link" data-drupal-link-system-path="node/29001">Alternatives to AutoCAD</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/dreamweaver" class="main__link" data-drupal-link-system-path="node/27646">Alternatives to Dreamweaver</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/gmail" class="main__link" data-drupal-link-system-path="node/22941">Alternatives to Gmail</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/matlab" class="main__link" data-drupal-link-system-path="node/29276">Alternatives to MATLAB</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/minecraft" class="main__link" data-drupal-link-system-path="node/22786">Alternatives to Minecraft</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/google-photos" class="main__link" data-drupal-link-system-path="node/27211">Alternatives to Google Photos</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/life/12/6/design-without-debt-five-tools-for-designers" class="main__link" data-drupal-link-system-path="node/16365">Alternatives to Photoshop</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/skype" class="main__link" data-drupal-link-system-path="node/45576">Alternatives to Skype</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/slack" class="main__link" data-drupal-link-system-path="node/23286">Alternatives to Slack</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives/trello" class="main__link" data-drupal-link-system-path="node/22546">Alternatives to Trello</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/alternatives" class="main__link" data-drupal-link-system-path="node/30871">More...</a>
|
|
</li>
|
|
</ul>
|
|
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/resources/linux" class="main__link" data-drupal-link-system-path="node/19796">Linux</a>
|
|
</li>
|
|
</ul>
|
|
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/downloads" class="main__link" data-drupal-link-system-path="downloads">Downloads</a>
|
|
</li>
|
|
<li class="main__item">
|
|
<a href="/faq" class="main__link" data-drupal-link-system-path="node/62466">Frequently Asked Questions</a>
|
|
</li>
|
|
</ul>
|
|
|
|
|
|
|
|
</nav>
|
|
<div class="views-exposed-form block block-views block-views-exposed-filter-blockacquia-search-page" data-drupal-selector="views-exposed-form-acquia-search-page" id="block-exposedformacquia-searchpage">
|
|
|
|
|
|
<form action="/site-search" method="get" id="views-exposed-form-acquia-search-page" accept-charset="UTF-8">
|
|
<div class="form--inline clearfix">
|
|
<div class="js-form-item form-item js-form-type-textfield form-type-textfield js-form-item-search form-item-search">
|
|
<label for="edit-search">Search</label>
|
|
<input placeholder="Search" data-drupal-selector="edit-search" type="text" id="edit-search" name="search" value size="30" maxlength="128" class="form-text">
|
|
|
|
</div>
|
|
<div data-drupal-selector="edit-actions" class="form-actions js-form-wrapper form-wrapper" id="edit-actions--4"><input data-drupal-selector="edit-submit-acquia-search" type="submit" id="edit-submit-acquia-search" value="Go" class="button js-form-submit form-submit">
|
|
</div>
|
|
|
|
</div>
|
|
|
|
</form>
|
|
|
|
</div>
|
|
|
|
|
|
</div>
|
|
|
|
|
|
<main>
|
|
<a id="main-content" tabindex="-1"></a>
|
|
<div class="wrapper wrapper--870">
|
|
<div class="region region-content">
|
|
<div id="block-osdc-page-title" class="block block-core block-page-title-block">
|
|
|
|
|
|
|
|
<h1 class="published page-title">
|
|
<span class="field field--name-title field--type-string field--label-hidden">Testing in production: Yes, you can (and should)</span>
|
|
|
|
</h1>
|
|
|
|
|
|
</div>
|
|
<div id="block-mainpagecontent" class="block block-system block-system-main-block">
|
|
|
|
|
|
|
|
<article class="node node--type-article node--promoted node--view-mode-full">
|
|
|
|
|
|
<div class="article__subhead">Why does testing in production get such a bad rap when we all do it? The key is to do it right.</div>
|
|
|
|
<footer class="author__meta">
|
|
<div class="author__submitted">
|
|
<div class="author__list">
|
|
By
|
|
<article class="profile">
|
|
<a href="/users/charity" hreflang="en">Charity</a>
|
|
|
|
</article>
|
|
|
|
</div>
|
|
<div class="author__meta-date-container">
|
|
<span class="author__meta-date">August 15, 2017</span> <span class="author__date-pipe">|</span>
|
|
<a href="#comments"><span class="author__comment-count">2 Comments </span></a>
|
|
<span class="author__date-pipe">|</span>
|
|
<pfe-readtime class="pf-global--FontSize--lg" for="#article_content" hidden>%t min read</pfe-readtime>
|
|
|
|
</div>
|
|
</div>
|
|
</footer>
|
|
<div id="article_content" class="author__content">
|
|
|
|
<div class="layout layout--onecol">
|
|
<div class="layout__region layout__region--content">
|
|
<div class="block block-layout-builder block-extra-field-blocknodearticlerate-rate-widget">
|
|
|
|
|
|
<div class="rate-widget thumbsup"><form class="node-article-38221-updown-rate-widget-0 vote-form" id="rate-widget-base-form" data-drupal-selector="node-article-38221-updown-rate-widget-0" action="/article/17/8/testing-production" method="post" accept-charset="UTF-8">
|
|
<div class="rate-widget-thumbs-up">
|
|
<div class="thumbsup-rating-wrapper rate-disabled" can-edit="false"><div class="form-item js-form-item form-type-radio js-form-type-radio form-item-value js-form-item-value form-disabled">
|
|
<label class="form-item js-form-item form-type-radio js-form-type-radio form-item-value js-form-item-value form-disabled rating-label thumbsup-rating-label thumbsup-rating-label-up vote-pending rate-thumbs-up-btn-up"><input twig-suggestion="rating-input" class="rating-input thumbsup-rating-input form-radio" data-drupal-selector="edit-value-1" disabled="disabled" type="radio" id="edit-value-1" name="value" value="1">
|
|
</label>
|
|
</div>
|
|
</div><input data-drupal-selector="form-9mfpykc-hpavohkc3uiwymjbh84piu0q9lvxap2ryjy" type="hidden" name="form_build_id" value="form-9mfPyKc-hPAVOhKC3uIwYMjBH84PIU0q9LvxAP2rYjY">
|
|
<input data-drupal-selector="edit-node-article-38221-updown-rate-widget-0" type="hidden" name="form_id" value="node_article_38221_updown_rate_widget_0">
|
|
<input class="thumbsup-rating-submit button button--primary js-form-submit form-submit" data-drupal-selector="edit-submit" data-disable-refocus="true" type="submit" id="edit-submit--2" name="op" value="Save">
|
|
<div class="vote-result js-form-wrapper form-wrapper" data-drupal-selector="edit-result" id="edit-result"><div class="rate-widget-thumbs-up rate-score">
|
|
484 readers like this. </div>
|
|
</div>
|
|
|
|
<div class="rate-score"></div>
|
|
</div>
|
|
|
|
|
|
</form>
|
|
</div>
|
|
|
|
</div>
|
|
<div class="block block-layout-builder block-extra-field-blocknodearticlecontent-moderation-control">
|
|
|
|
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
<div class="layout layout--onecol">
|
|
<div class="layout__region layout__region--content">
|
|
<div class="block block-layout-builder block-field-blocknodearticlefield-lead-image">
|
|
|
|
|
|
|
|
<div class="field field--name-field-lead-image field--type-entity-reference field--label-hidden field__item"><article class="media media--type-image media--view-mode-caption">
|
|
|
|
|
|
<div class="field field--name-field-media-image field--type-image field--label-hidden field__item"> <img loading="lazy" src="/sites/default/files/lead-images/code_computer_development_programming.png" width="520" height="292" alt="An introduction to GNU Screen" title="An introduction to GNU Screen">
|
|
|
|
</div>
|
|
|
|
<div class="field field--name-field-caption field--type-text-long field--label-hidden caption field__item"><span class="caption__byline">Image by: </span><p>Opensource.com</p>
|
|
</div>
|
|
|
|
</article>
|
|
</div>
|
|
|
|
</div>
|
|
<div class="block block-layout-builder block-field-blocknodearticlebody">
|
|
|
|
|
|
|
|
<div class="clearfix text-formatted field field--name-body field--type-text-with-summary field--label-hidden field__item"><p>I wrote a piece recently about why <a href="/article/17/7/state-systems-administration">we are all distributed systems engineers now</a>. To my surprise, lots of people objected to the observation that you <em>have</em> to test large distributed systems in production. </p>
|
|
<p>It seems testing in production has gotten a bad rap—despite the fact that we all do it, all the time.</p>
|
|
<p>Maybe we associate it with cowboy engineering. We hear "testing in production" and assume this means no unit tests, functional tests, or continuous integration.</p>
|
|
<p>It's good to try and catch things before production—we should do that too! But these things aren't mutually exclusive. Here are some things to consider about testing in production.</p>
|
|
<h2>1. You already do it</h2>
|
|
<p>There are lots of things you already test in prod—because there's no other way you can test them. Sure, you can spin up clones of various system components or entire systems, and capture real traffic to replay offline (the gold standard of systems testing). But many systems are too big, complex, and cost-prohibitive to clone.</p>
|
|
<p>Imagine trying to spin up a copy of Facebook for testing (with its multiple, globally distributed data centers). Imagine trying to spin up a copy of the national electrical grid. Even if you succeed, next you need the same number of clients, the same concurrency, same pipelining and usage patterns, etc. The unpredictability of user traffic makes it impossible to mock; even if you could perfectly reproduce yesterday's traffic, you still can't predict tomorrow's.</p>
|
|
<p>It's easy to get dragged down into bikeshedding about cloning environments and miss the real point: <strong>Only</strong><strong> production is production</strong>, and every time you deploy there you are testing a unique combination of deploy code + software + environment. (Just ask anyone who's ever confidently deployed to "Staging", and then "Producktion" (sic).) </p>
|
|
<h2>2. So does everyone else</h2>
|
|
<p>You can't spin up a copy of Facebook. You can't spin up a copy of the national power grid. Some things just aren't amenable to cloning. And that's fine. You simply can't usefully mimic the qualities of size and chaos that tease out the long, thin tail of bugs or behaviors you care about.</p>
|
|
<p>And you shouldn't try.</p>
|
|
<p>Facebook doesn't try to spin up a copy of Facebook either. They invest in the tools that allow thousands and thousands of engineers to deploy safely to production every day and observe people interacting with the code they wrote. So does Netflix. So does everyone who is fortunate enough to outgrow the delusion that this is a tractable problem.</p>
|
|
<h2>3. It's probably fine</h2>
|
|
<p>There's a lot of value in testing... to a point. But if you can catch 80% to 90% of the bugs with 10% to 20% of the effort—<a href="https://twitter.com/mipsytipsy/status/879314772277293056" target="_blank">and you can</a>—the rest is more usefully poured into making your systems resilient, not preventing failure.</p>
|
|
<p>You should be <strong>practicing failure regularly</strong>. Ideally, everyone who has access to production knows how to do a deploy and rollback, or how to get to a known-good state fast. They should know what a normal operating system looks like, and how to debug basic problems. Knowing how to deal with failure should not be rare.</p>
|
|
<p>If you test in production, dealing with failure won't be rare. I'm talking about things like, "Does this have a memory leak?" Maybe run it as a canary on five hosts overnight and see. "Does this functionality work as planned?" At some point, just ship it with a feature flag so only certain users can exercise it. Stuff like that. Practice shipping and fixing lots of small problems, instead of a few big and dramatic releases.</p>
|
|
<h2>4. You've got bigger problems</h2>
|
|
<p>You're shipping code every day and causing self-inflicted damage on the regular, and you can't tell what it's doing before, during, or after. It's not the breaking stuff that's the problem; you can break things safely. It's the second part—not knowing what it's doing—that's not OK. This bigger problem can be addressed by:</p>
|
|
<ul>
|
|
<li>Canarying. Automated canarying. Automated canarying in graduated levels with automatic promotion. Multiple canaries in simultaneous flight!</li>
|
|
<li>Making deploys more automated, robust, and fast (5 minutes on the upper bound is good)</li>
|
|
<li>Making rollbacks wicked fast and reliable</li>
|
|
<li>Using instrumentation, observability, and other early warning signs for staged canaries</li>
|
|
<li>Doing end-to-end health checks of key endpoints</li>
|
|
<li>Choosing good defaults, feature flags, developer tooling</li>
|
|
<li>Educating, sharing best practices, standardizing practices, making the easy/fast way the right way</li>
|
|
<li>Taking as much code and as many back-end components as possible out of the critical path</li>
|
|
<li>Limiting the blast radius of any given user or change</li>
|
|
<li>Exploring production, verifying that the expected changes are what actually happened. Knowing what normal looks like</li>
|
|
</ul>
|
|
<p>These things are all a <em>great</em> use of your time. Unlike staging and test environments, which are notoriously fragile and flaky and hard to keep in sync with prod.</p>
|
|
<h2>Do those things</h2>
|
|
<p>Release engineering is a systematically underinvested skillset at companies with more than 50 people. Your deploys are the cause of nearly all your failures because they inject chaos into your system. Having a staging copy of production is not going to do much to change that (and it adds a large category of problems colloquially known as "it looked just like production, so I just dropped that table...").</p>
|
|
<p>Embrace failure. Chaos and failure are your friends. The issue is not <em>if</em> you will fail, it is <em>when</em> you will fail, and <em>whether</em> you will notice. It's between whether it will annoy <em>all</em> of your users because the entire site is down, or if it will annoy only <em>a few</em> users until you fix it at your leisure the next morning.</p>
|
|
<p>Once upon a time, these were optional skills, even specialties. Not anymore. These are table stakes in your new career as a distributed systems engineer.</p>
|
|
<p>Lean into it. It's probably fine.</p>
|
|
</div>
|
|
|
|
</div>
|
|
<div class="block block-layout-builder block-field-blocknodearticlefield-listicle-title">
|
|
|
|
|
|
|
|
<div class="hidden field field--name-field-listicle-title field--type-string field--label-hidden field__item">What to read next</div>
|
|
|
|
</div>
|
|
<div class="block block-layout-builder block-field-blocknodearticlefield-tags">
|
|
|
|
|
|
|
|
<div class="field field--name-field-tags field--type-entity-reference field--label-inline clearfix">
|
|
<div class="field__label">Tags</div>
|
|
<div class="field__items">
|
|
<div class="field__item"><a href="/tags/programming" hreflang="en">Programming</a></div>
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
<div class="views-element-container author-block block block-views block-views-blockauthor-info-author-info">
|
|
<div class="author-block__wrapper">
|
|
|
|
|
|
<div><div class="view view-author-info view-id-author_info view-display-id-author_info js-view-dom-id-f342aa7bf6626c0215e14a3f5458fc2fab27623c97a12da5d387eb4e3f7c0e6d">
|
|
|
|
|
|
|
|
<div class="view-content">
|
|
<div class="views-row"><div class="views-field views-field-field-display-name"><div class="field-content"><a href="/users/charity" hreflang="en">Charity</a></div></div><div class="views-field views-field-field-user-picture"><div class="field-content"><article class="media media--type-image media--view-mode-_50x150">
|
|
|
|
|
|
<div class="field field--name-field-media-image field--type-image field--label-hidden field__item"> <img loading="lazy" src="/sites/default/files/styles/150x150/public/pictures/charity-stripe-hoodie-2.jpg?itok=5ZwaMcvl" width="150" height="150" alt="User profile image." class="image-style-_50x150">
|
|
|
|
|
|
</div>
|
|
|
|
</article>
|
|
</div></div><div class="views-field views-field-field-user-about"><span class="field-content">Engineer and cofounder/CEO of Honeycomb, a nextgen tool for helping software engineers understand their containers/schedulers/microservicified distributed systems and polyglot persistence layers. Likes: databases, operations under pressure, expensive whiskey. Hates: databases, flappy pages, cheap whiskey. Probably swears more than you.</span></div><div class="views-field views-field-nothing"><span class="field-content"><a href="/users/charity">More about me</a></span></div></div>
|
|
|
|
</div>
|
|
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<div class="views-element-container author-block block block-views block-views-blockauthor-info-block-1">
|
|
<div class="author-block__wrapper plain">
|
|
|
|
|
|
<div><div class="view view-author-info view-id-author_info view-display-id-block_1 js-view-dom-id-b4b3c78cc96bd6ab0dfba5391e3041d0978221bd2785321dced34c54ebcb195a">
|
|
|
|
|
|
|
|
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<div class="block block-layout-builder block-field-blocknodearticlefield-comments">
|
|
|
|
|
|
<section id="comments" class="field field--name-field-comments field--type-comment field--label-hidden comment-wrapper">
|
|
|
|
|
|
<div class="comments__count">
|
|
<h2>2 Comments</h2>
|
|
<div class="login">These comments are closed.</div>
|
|
</div>
|
|
|
|
|
|
<article data-comment-user-id="34306" id="comment-139271" class="comment js-comment">
|
|
<mark class="hidden" data-comment-timestamp="1502876400"></mark>
|
|
|
|
<footer class="comment__meta">
|
|
<article class="media media--type-image media--view-mode-byline">
|
|
|
|
|
|
<div class="field field--name-field-media-image field--type-image field--label-hidden field__item"> <img loading="lazy" src="/sites/default/files/styles/medium/public/osdc_default_avatar_1.png?itok=G0WcUo3c" width="100" height="100" alt="Avatar" title="Avatar" class="image-style-medium">
|
|
|
|
|
|
</div>
|
|
|
|
</article>
|
|
|
|
<div>
|
|
<div class="comment__submitted">
|
|
<a href="/users/lewiscowles1986">Lewis Cowles</a> | August 16, 2017
|
|
<div class="rate-widget thumbsup"><form class="comment-content-139271-updown-rate-widget-0 vote-form" id="rate-widget-base-form--2" data-drupal-selector="comment-content-139271-updown-rate-widget-0" action="/article/17/8/testing-production" method="post" accept-charset="UTF-8">
|
|
<div class="rate-widget-thumbs-up">
|
|
<div class="thumbsup-rating-wrapper rate-disabled" can-edit="false"><div class="form-item js-form-item form-type-radio js-form-type-radio form-item-value js-form-item-value form-disabled">
|
|
<label class="form-item js-form-item form-type-radio js-form-type-radio form-item-value js-form-item-value form-disabled rating-label thumbsup-rating-label thumbsup-rating-label-up vote-pending rate-thumbs-up-btn-up"><input twig-suggestion="rating-input" class="rating-input thumbsup-rating-input form-radio" data-drupal-selector="edit-value-1-2" disabled="disabled" type="radio" id="edit-value-1--2" name="value" value="1">
|
|
</label>
|
|
</div>
|
|
</div><input data-drupal-selector="form-thiqvvjzuuvndbitrxn8pkjn2k-txrcy0mqdn8ztihu" type="hidden" name="form_build_id" value="form-tHiQVvjzUuvnDbiTrxn8pkjN2K-tXRCY0mqDN8zTihU">
|
|
<input data-drupal-selector="edit-comment-content-139271-updown-rate-widget-0" type="hidden" name="form_id" value="comment_content_139271_updown_rate_widget_0">
|
|
<input class="thumbsup-rating-submit button button--primary js-form-submit form-submit" data-drupal-selector="edit-submit" data-disable-refocus="true" type="submit" id="edit-submit--4" name="op" value="Save">
|
|
<div class="vote-result js-form-wrapper form-wrapper" data-drupal-selector="edit-result" id="edit-result--2"><div class="rate-widget-thumbs-up rate-score">
|
|
No readers like this yet.
|
|
</div>
|
|
</div>
|
|
|
|
<div class="rate-score"></div>
|
|
</div>
|
|
|
|
|
|
</form>
|
|
</div>
|
|
|
|
</div>
|
|
<div class="content">
|
|
|
|
<div class="clearfix text-formatted field field--name-comment-body field--type-text-long field--label-hidden field__item"><p>It's a great article, but as it went on it sounded terrifying. I take it this is only for very large businesses that can afford to have staff on-call lots of the time to deal with potential fallout? Or are they beyond that? I'm just trying to wrap my head around at what point my own advice is, or becomes invalid or inadequate. I prefer to work with smaller businesses, so we get to avoid some of the complexities of facebook and national services mostly.</p>
|
|
<p>This doesn't say for example, don't test locally, then CI, then use staging. It seems that at a certain point it seems to suggest staging may be impractical for velocity, and end-user validation. Monitoring live is essential (we already agree there), with feature flags taking over some things being "tested", I wonder is this entirely through logging? I suppose the problem is that it's hard to visualise. Facebook might not clone itself, but I'd have a hard time accepting even at their gargantuan scale it's possible to log everything.</p>
|
|
<p>It's hardly a case for attaching a remote debugger to live, but I wish I had been able to glean a bit more what actually happens.</p>
|
|
<p>Very interesting, thank you for sharing.</p>
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</footer>
|
|
|
|
</article>
|
|
|
|
|
|
<article data-comment-user-id="0" id="comment-139576" class="comment js-comment by-anonymous">
|
|
<mark class="hidden" data-comment-timestamp="1503013806"></mark>
|
|
|
|
<footer class="comment__meta">
|
|
<article class="media media--type-image media--view-mode-byline">
|
|
|
|
|
|
<div class="field field--name-field-media-image field--type-image field--label-hidden field__item"> <img loading="lazy" src="/sites/default/files/styles/medium/public/osdc_default_avatar_1.png?itok=G0WcUo3c" width="100" height="100" alt="Avatar" title="Avatar" class="image-style-medium">
|
|
|
|
|
|
</div>
|
|
|
|
</article>
|
|
|
|
<div>
|
|
<div class="comment__submitted">
|
|
Eddie G. | August 17, 2017
|
|
<div class="rate-widget thumbsup"><form class="comment-content-139576-updown-rate-widget-0 vote-form" id="rate-widget-base-form--3" data-drupal-selector="comment-content-139576-updown-rate-widget-0" action="/article/17/8/testing-production" method="post" accept-charset="UTF-8">
|
|
<div class="rate-widget-thumbs-up">
|
|
<div class="thumbsup-rating-wrapper rate-disabled" can-edit="false"><div class="form-item js-form-item form-type-radio js-form-type-radio form-item-value js-form-item-value form-disabled">
|
|
<label class="form-item js-form-item form-type-radio js-form-type-radio form-item-value js-form-item-value form-disabled rating-label thumbsup-rating-label thumbsup-rating-label-up vote-pending rate-thumbs-up-btn-up"><input twig-suggestion="rating-input" class="rating-input thumbsup-rating-input form-radio" data-drupal-selector="edit-value-1-3" disabled="disabled" type="radio" id="edit-value-1--3" name="value" value="1">
|
|
</label>
|
|
</div>
|
|
</div><input data-drupal-selector="form-sfpkfpdmsylux4mkza54iajckyal3qk9gv9b7jfbe-s" type="hidden" name="form_build_id" value="form-SfPKFPDMsYLUx4mkza54iajCKyAl3qk9Gv9B7JFBe_s">
|
|
<input data-drupal-selector="edit-comment-content-139576-updown-rate-widget-0" type="hidden" name="form_id" value="comment_content_139576_updown_rate_widget_0">
|
|
<input class="thumbsup-rating-submit button button--primary js-form-submit form-submit" data-drupal-selector="edit-submit" data-disable-refocus="true" type="submit" id="edit-submit--6" name="op" value="Save">
|
|
<div class="vote-result js-form-wrapper form-wrapper" data-drupal-selector="edit-result" id="edit-result--3"><div class="rate-widget-thumbs-up rate-score">
|
|
No readers like this yet.
|
|
</div>
|
|
</div>
|
|
|
|
<div class="rate-score"></div>
|
|
</div>
|
|
|
|
|
|
</form>
|
|
</div>
|
|
|
|
</div>
|
|
<div class="content">
|
|
|
|
<div class="clearfix text-formatted field field--name-comment-body field--type-text-long field--label-hidden field__item"><p>I have no knowledge of deploying, testing, or even programming, but this is what I wanted to know regarding working in a medium to large business, as I'm aiming for some kind of programming degree with which to step into the arena of programmers. So this advice about deployments in or out of production is quite helpful! Thanks for your advice and insight!!</p>
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</footer>
|
|
|
|
</article>
|
|
|
|
|
|
|
|
</section>
|
|
|
|
</div>
|
|
<div class="views-element-container related-articles block block-views block-views-blockrelated-articles-related-articles">
|
|
|
|
<h2 class="related-articles__title">Related Content</h2>
|
|
|
|
<div><div class="view view-related-articles view-id-related_articles view-display-id-related_articles js-view-dom-id-8de8c96869e7dcb2e11023f593a7e430c7c77425816435ee93d717599ae5fcec">
|
|
|
|
|
|
|
|
<div class="view-content">
|
|
<div class="views-row"><div class="views-field views-field-field-lead-image"><div class="field-content"> <a href="/article/23/4/c-vs-go-programming-languages" hreflang="en"><img loading="lazy" src="/sites/default/files/styles/222x125/public/lead-images/collab-team-pair-programming-code-keyboard.png?itok=iEiqXXDv" width="222" height="125" alt="Pair programming" title="Pair programming" class="image-style-_22x125">
|
|
|
|
</a>
|
|
</div></div><div class="views-field views-field-title"><span class="field-content"><a href="/article/23/4/c-vs-go-programming-languages" hreflang="en">C vs. Go: Comparing programming languages</a></span></div></div>
|
|
<div class="views-row"><div class="views-field views-field-field-lead-image"><div class="field-content"> <a href="/article/23/4/learn-tcltk-wish-simple-game" hreflang="en"><img loading="lazy" src="/sites/default/files/styles/222x125/public/lead-images/programming_keyboard_coding.png?itok=OLBqPd92" width="222" height="125" alt="Programming keyboard." title="Programming keyboard." class="image-style-_22x125">
|
|
|
|
</a>
|
|
</div></div><div class="views-field views-field-title"><span class="field-content"><a href="/article/23/4/learn-tcltk-wish-simple-game" hreflang="en">Learn Tcl/Tk and Wish with this simple game</a></span></div></div>
|
|
<div class="views-row"><div class="views-field views-field-field-lead-image"><div class="field-content"> <a href="/article/23/4/basic-vs-fortran-77" hreflang="en"><img loading="lazy" src="/sites/default/files/styles/222x125/public/lead-images/code_computer_development_programming.png?itok=0kgDZpBf" width="222" height="125" alt="An introduction to GNU Screen" title="An introduction to GNU Screen" class="image-style-_22x125">
|
|
|
|
</a>
|
|
</div></div><div class="views-field views-field-title"><span class="field-content"><a href="/article/23/4/basic-vs-fortran-77" hreflang="en">BASIC vs. FORTRAN 77: Comparing programming blasts from the past</a></span></div></div>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="block block-layout-builder block-field-blocknodearticlefield-default-license">
|
|
|
|
|
|
|
|
<div class="field field--name-field-default-license field--type-list-string field--label-hidden field__item"><a rel="license" href="https://creativecommons.org/licenses/by-sa/4.0/">
|
|
<img alt="Creative Commons License" src="/themes/osdc/assets/img/cc-by-sa-4.png" title="This work is licensed under a Creative Commons Attribution-Share Alike 4.0 International License."></a>This work is licensed under a Creative Commons Attribution-Share Alike 4.0 International License.</div>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
</article>
|
|
|
|
|
|
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</main>
|
|
|
|
<footer class="global_footer">
|
|
<div class="wrapper">
|
|
<div class="region region-left-footer">
|
|
<div id="block-aboutthissitefooter" class="block block-block-content block-block-content1d00183d-1f84-4ed3-9367-c632014cd322">
|
|
|
|
<h2>About This Site</h2>
|
|
|
|
|
|
<div class="clearfix text-formatted field field--name-body field--type-text-with-summary field--label-hidden field__item"><p>The opinions expressed on this website are those of each author, not of the author's employer or of Red Hat.</p>
|
|
<p><a href="/">Opensource.com</a> aspires to publish all content under a <a href="https://creativecommons.org/licenses/">Creative Commons license</a> but may not be able to do so in all cases. You are responsible for ensuring that you have the necessary permission to reuse any work on this site. Red Hat and the Red Hat logo are trademarks of Red Hat, LLC, registered in the United States and other countries.</p>
|
|
<p>A note on advertising: Opensource.com does not sell advertising on the site or in any of its newsletters.</p>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="region region-right-footer">
|
|
<div id="block-sitebranding" class="block block-system block-system-branding-block">
|
|
|
|
|
|
<div class="branding">
|
|
<a href="/" rel="home" aria-label class="branding__logo">
|
|
<img src="/themes/osdc/logo.svg" alt="Home">
|
|
</a>
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="region region-copyright">
|
|
<div id="block-copyrightfooter" class="block block-block-content block-block-contentc0a744a4-2740-4f8a-a693-ee8d7cb5ff86">
|
|
|
|
|
|
|
|
<div class="clearfix text-formatted field field--name-body field--type-text-with-summary field--label-hidden field__item"><p>Copyright ©<span id="copyright-year">2021</span> Red Hat, LLC</p>
|
|
</div>
|
|
|
|
</div>
|
|
<nav role="navigation" aria-labelledby="block-legal-menu" id="block-legal" class="block block-menu navigation menu--legal">
|
|
|
|
<h2 class="visually-hidden" id="block-legal-menu">Legal</h2>
|
|
|
|
|
|
|
|
<ul class="menu">
|
|
<li class="menu-item">
|
|
<a href="/privacy-policy" data-drupal-link-system-path="node/17705">Privacy Policy</a>
|
|
</li>
|
|
<li class="menu-item">
|
|
<a href="/legal" data-drupal-link-system-path="legal">Terms of use</a>
|
|
</li>
|
|
<li class="menu-item">
|
|
<span id="teconsent"></span>
|
|
</li>
|
|
</ul>
|
|
|
|
|
|
|
|
</nav>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
<div id="consent_blackbar"></div>
|
|
|
|
</footer>
|
|
</div>
|
|
</div>
|
|
|
|
<cpx-reporter event="User Detected" auto><script type="data">{"loggedIn":false}</script>
|
|
</cpx-reporter>
|
|
<cpx-reporter auto event="Page Load Completed"></cpx-reporter>
|
|
<script type="application/json" data-drupal-selector="drupal-settings-json">{"path":{"baseUrl":"\/","pathPrefix":"","currentPath":"node\/38221","currentPathIsAdmin":false,"isFront":false,"currentLanguage":"en"},"pluralDelimiter":"\u0003","suppressDeprecationErrors":true,"ajaxPageState":{"libraries":"eJxVUVtyxCAMuxCEI2UIuIFdg6kN2aanr_PYafsBIwlbBhHQi-xu8QImXDjXCLVDfPMCIn4FefNKUWt1k5pbg-7-4CnlNaGu_tB6KkWdXOTRPE43tctutwwvYC1g0HEduOr543MA79MHcblOqiKP-RtM9N2j34HnAl3x6kL7mhkasTbP6tyoqrfRIhp9jlkCbermVA6EhiQG50dPxHZBCs9_ymF6Cfcd5WIr0uLv5kZ4Iwb0Go8NpDfXmaco4Dkkc1S933tgi7k-xbA2uFE36mATYNO3n9LL9jTKIqMZTjo7wjZ7zWLvOYiL6vKrHxnMZwjunmxzh2ID5vDU35JdlF4_eeQr7tynQnEg_ACxI73O","theme":"osdc","theme_token":null},"ajaxTrustedUrl":{"form_action_p_pvdeGsVG5zNF_XLGPTvYSKCf43t8qZYSwcfZl2uzM":true,"\/article\/17\/8\/testing-production?ajax_form=1":true,"\/site-search":true},"ajax":{"edit-submit--6":{"disable-refocus":true,"callback":"::ajaxSubmit","event":"click","wrapper":"rate-widget-base-form--3","progress":{"type":null},"url":"\/article\/17\/8\/testing-production?ajax_form=1","httpMethod":"POST","dialogType":"ajax","submit":{"_triggering_element_name":"op","_triggering_element_value":"Save"}},"edit-submit--4":{"disable-refocus":true,"callback":"::ajaxSubmit","event":"click","wrapper":"rate-widget-base-form--2","progress":{"type":null},"url":"\/article\/17\/8\/testing-production?ajax_form=1","httpMethod":"POST","dialogType":"ajax","submit":{"_triggering_element_name":"op","_triggering_element_value":"Save"}},"edit-submit--2":{"disable-refocus":true,"callback":"::ajaxSubmit","event":"click","wrapper":"rate-widget-base-form","progress":{"type":null},"url":"\/article\/17\/8\/testing-production?ajax_form=1","httpMethod":"POST","dialogType":"ajax","submit":{"_triggering_element_name":"op","_triggering_element_value":"Save"}}},"user":{"uid":0,"permissionsHash":"bc13164d8931f8d14b743e3c22c9159308b51a70c1eafe240ce98221f01ff908"}}</script>
|
|
<script src="/sites/default/files/js/js_Wlv6wa8kL6PCkC1orXiTY8r2QgMkdPttdi_yc1ADois.js?scope=footer&delta=0&language=en&theme=osdc&include=eJxNT1sSwyAIvJDGIzlGaTRVsYhJvX3pa9oPZpZddoHg2GU3gWwBFrwZ3-6WoCGxkB5LwwqVFUXBAQ4bZMq-LMZjZdF0Yija5-SvEBT24M2WcXX5jd3giKSf-UoyijhMoNFcXj6tXqc-EpxAihyDGfVABh0hty91ao6jrH00ySAwSTZTlYj9NoDmckEqogToNbUGbP7wEtMWsxTv_feHE_fk5LsJcsoDo0hnbw"></script>
|
|
<script src="/themes/osdc/node_modules/%40patternfly/pfe-readtime/dist/pfe-readtime.min.js?sqka6y" type="module"></script>
|
|
<script src="/sites/default/files/js/js_1iq4NLQ-hqHdXTq7H1u5V5e5_FLPqt3wDRjw2y_2tk4.js?scope=footer&delta=2&language=en&theme=osdc&include=eJxNT1sSwyAIvJDGIzlGaTRVsYhJvX3pa9oPZpZddoHg2GU3gWwBFrwZ3-6WoCGxkB5LwwqVFUXBAQ4bZMq-LMZjZdF0Yija5-SvEBT24M2WcXX5jd3giKSf-UoyijhMoNFcXj6tXqc-EpxAihyDGfVABh0hty91ao6jrH00ySAwSTZTlYj9NoDmckEqogToNbUGbP7wEtMWsxTv_feHE_fk5LsJcsoDo0hnbw"></script>
|
|
|
|
</body>
|
|
</html>
|