784 lines
75 KiB
HTML
784 lines
75 KiB
HTML
<!doctype html>
|
||
<html xmlns:og="http://opengraphprotocol.org/schema/" xmlns:fb="http://www.facebook.com/2008/fbml" lang="en-US" >
|
||
|
||
<head>
|
||
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
|
||
|
||
<meta name="viewport" content="initial-scale=1">
|
||
|
||
<!-- This is Squarespace. --><!-- susan-fowler-b4lr -->
|
||
<base href="">
|
||
<meta charset="utf-8" />
|
||
<title>The Irreproducibility of Bugs in Large-Scale Production Systems — Susan Fowler</title>
|
||
<meta http-equiv="Accept-CH" content="Sec-CH-UA-Platform-Version, Sec-CH-UA-Model" /><link rel="icon" type="image/x-icon" href="https://assets.squarespace.com/universal/default-favicon.ico"/>
|
||
<link rel="canonical" href="https://www.susanjfowler.com/blog/2016/11/2/the-irreproducibility-of-bugs-in-large-scale-production-systems"/>
|
||
<meta name="google-site-verification" content="4ljeYcv6pAQpLIzdhnCPi8J50751fJPZ7hF0eysStlk" />
|
||
<meta property="og:site_name" content="Susan Fowler"/>
|
||
<meta property="og:title" content="The Irreproducibility of Bugs in Large-Scale Production Systems — Susan Fowler"/>
|
||
<meta property="og:latitude" content="40.7207559"/>
|
||
<meta property="og:longitude" content="-74.00076130000002"/>
|
||
<meta property="og:locality" content=""/>
|
||
<meta property="og:url" content="https://www.susanjfowler.com/blog/2016/11/2/the-irreproducibility-of-bugs-in-large-scale-production-systems"/>
|
||
<meta property="og:type" content="article"/>
|
||
<meta property="og:description" content="The crux of reproducibility when it comes to bugs is this: being able to reproduce a bug requires that the state of the system be nearly identical at the time of reproduction as it was at the time the bug originally occurred - something that is impossible to guarantee in large production systems..."/>
|
||
<meta property="og:image" content="http://static1.squarespace.com/static/57a6de391b631b6aa4789ebf/t/5dd82f25dcadfc116d1e5b99/1574448937925/91t6KiP1cEL.jpg?format=1500w"/>
|
||
<meta property="og:image:width" content="1500"/>
|
||
<meta property="og:image:height" content="2264"/>
|
||
<meta itemprop="name" content="The Irreproducibility of Bugs in Large-Scale Production Systems — Susan Fowler"/>
|
||
<meta itemprop="url" content="https://www.susanjfowler.com/blog/2016/11/2/the-irreproducibility-of-bugs-in-large-scale-production-systems"/>
|
||
<meta itemprop="description" content="The crux of reproducibility when it comes to bugs is this: being able to reproduce a bug requires that the state of the system be nearly identical at the time of reproduction as it was at the time the bug originally occurred - something that is impossible to guarantee in large production systems..."/>
|
||
<meta itemprop="thumbnailUrl" content="http://static1.squarespace.com/static/57a6de391b631b6aa4789ebf/t/5dd82f25dcadfc116d1e5b99/1574448937925/91t6KiP1cEL.jpg?format=1500w"/>
|
||
<link rel="image_src" href="http://static1.squarespace.com/static/57a6de391b631b6aa4789ebf/t/5dd82f25dcadfc116d1e5b99/1574448937925/91t6KiP1cEL.jpg?format=1500w" />
|
||
<meta itemprop="image" content="http://static1.squarespace.com/static/57a6de391b631b6aa4789ebf/t/5dd82f25dcadfc116d1e5b99/1574448937925/91t6KiP1cEL.jpg?format=1500w"/>
|
||
<meta itemprop="author" content="Susan Rigetti"/>
|
||
<meta itemprop="datePublished" content="2016-11-03T00:52:52-0700"/>
|
||
<meta itemprop="dateModified" content="2016-11-08T00:28:11-0800"/>
|
||
<meta itemprop="headline" content="The Irreproducibility of Bugs in Large-Scale Production Systems "/>
|
||
<meta itemprop="publisher" content="Susan Fowler"/>
|
||
<meta name="twitter:title" content="The Irreproducibility of Bugs in Large-Scale Production Systems — Susan Fowler"/>
|
||
<meta name="twitter:image" content="http://static1.squarespace.com/static/57a6de391b631b6aa4789ebf/t/5dd82f25dcadfc116d1e5b99/1574448937925/91t6KiP1cEL.jpg?format=1500w"/>
|
||
<meta name="twitter:url" content="https://www.susanjfowler.com/blog/2016/11/2/the-irreproducibility-of-bugs-in-large-scale-production-systems"/>
|
||
<meta name="twitter:card" content="summary"/>
|
||
<meta name="twitter:description" content="The crux of reproducibility when it comes to bugs is this: being able to reproduce a bug requires that the state of the system be nearly identical at the time of reproduction as it was at the time the bug originally occurred - something that is impossible to guarantee in large production systems..."/>
|
||
<meta name="description" content=" The crux of reproducibility when it comes to bugs is this: being able to reproduce a bug requires that the state of the system be nearly identical at the time of reproduction as it was at the time the bug originally occurred - something that is impossible to guarantee in large production systems... " />
|
||
<link rel="preconnect" href="https://images.squarespace-cdn.com">
|
||
<link rel="preconnect" href="https://use.typekit.net" crossorigin>
|
||
<link rel="preconnect" href="https://p.typekit.net" crossorigin>
|
||
<script type="text/javascript" src="//use.typekit.net/ik/5hWUCNeYwic1iSzXcBlxcfus-E-2JuLBCTJH42LKnICfeztgfFHN4UJLFRbh52jhWD9ajQZqZQwowAJhwDBKFcByF2St5AmKZy7dMPG0SamDjAtlZcNademkOAJXdYJ0SaBujW48Sagyjh90jhNlOeUzjhBC-eNDifUDiWFydKucdaiCZWJljRsCSKoDSWmyScmDSeBRZPoRdhXCSamDjAo7ZcNademkOcFzdPUaiaS0SamDjAtlZcNademkOAJXdYJ0SaBujW48Sagyjh90jhNlOYiaikoDiWFydcTcdaiCZWJ0jhNlJ6ZC-WZzS1q7fbRf9gMMeMb6MKG4fwFbIMMj2KMfH6qJUBMbMg6YJMJ7fbRB9gMMegI6MTMg8FHdzb9.js" async fetchpriority="high" onload="try{Typekit.load();}catch(e){} document.documentElement.classList.remove('wf-loading');"></script>
|
||
<script>document.documentElement.classList.add('wf-loading')</script>
|
||
<style>@keyframes fonts-loading { 0%, 99% { color: transparent; } } html.wf-loading * { animation: fonts-loading 3s; }</style>
|
||
<link rel="preconnect" href="https://file.squarespace-cdn.com" crossorigin>
|
||
<style data-sqsp-font-ids="6a341ee2fac6fe7d51c142b4">
|
||
/* latin */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: italic;
|
||
font-weight: 300;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/fde71b6f-ee98-40e8-8312-130961a742de/latin.woff2') format('woff2');
|
||
unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD;
|
||
}
|
||
/* cyrillic-ext */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: italic;
|
||
font-weight: 300;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/0332d807-ff5c-4af1-b22e-bd1fa3506c4a/cyrillic-ext.woff2') format('woff2');
|
||
unicode-range: U+0460-052F, U+1C80-1C8A, U+20B4, U+2DE0-2DFF, U+A640-A69F, U+FE2E-FE2F;
|
||
}
|
||
/* vietnamese */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: italic;
|
||
font-weight: 300;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/49270ee4-3fac-4e05-84ec-31d51ee2d331/vietnamese.woff2') format('woff2');
|
||
unicode-range: U+0102-0103, U+0110-0111, U+0128-0129, U+0168-0169, U+01A0-01A1, U+01AF-01B0, U+0300-0301, U+0303-0304, U+0308-0309, U+0323, U+0329, U+1EA0-1EF9, U+20AB;
|
||
}
|
||
/* greek */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: italic;
|
||
font-weight: 300;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/3c8e1cea-de6c-4534-9d42-7629dfa5bda7/greek.woff2') format('woff2');
|
||
unicode-range: U+0370-0377, U+037A-037F, U+0384-038A, U+038C, U+038E-03A1, U+03A3-03FF;
|
||
}
|
||
/* latin-ext */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: italic;
|
||
font-weight: 300;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/0870e6a2-07b2-41dc-b61a-d4c7e49a7eeb/latin-ext.woff2') format('woff2');
|
||
unicode-range: U+0100-02BA, U+02BD-02C5, U+02C7-02CC, U+02CE-02D7, U+02DD-02FF, U+0304, U+0308, U+0329, U+1D00-1DBF, U+1E00-1E9F, U+1EF2-1EFF, U+2020, U+20A0-20AB, U+20AD-20C0, U+2113, U+2C60-2C7F, U+A720-A7FF;
|
||
}
|
||
/* cyrillic */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: italic;
|
||
font-weight: 300;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/b347f096-2c33-4074-88da-f496d56baa29/cyrillic.woff2') format('woff2');
|
||
unicode-range: U+0301, U+0400-045F, U+0490-0491, U+04B0-04B1, U+2116;
|
||
}
|
||
/* vietnamese */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 700;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/1d35083e-f339-47a5-ad96-8ec778fec4b2/vietnamese.woff2') format('woff2');
|
||
unicode-range: U+0102-0103, U+0110-0111, U+0128-0129, U+0168-0169, U+01A0-01A1, U+01AF-01B0, U+0300-0301, U+0303-0304, U+0308-0309, U+0323, U+0329, U+1EA0-1EF9, U+20AB;
|
||
}
|
||
/* latin-ext */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 700;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/83e85d60-7795-46cd-b706-4d0043313f1b/latin-ext.woff2') format('woff2');
|
||
unicode-range: U+0100-02BA, U+02BD-02C5, U+02C7-02CC, U+02CE-02D7, U+02DD-02FF, U+0304, U+0308, U+0329, U+1D00-1DBF, U+1E00-1E9F, U+1EF2-1EFF, U+2020, U+20A0-20AB, U+20AD-20C0, U+2113, U+2C60-2C7F, U+A720-A7FF;
|
||
}
|
||
/* cyrillic-ext */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 700;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/48047284-7452-4c96-a9b3-730c1d1c73d8/cyrillic-ext.woff2') format('woff2');
|
||
unicode-range: U+0460-052F, U+1C80-1C8A, U+20B4, U+2DE0-2DFF, U+A640-A69F, U+FE2E-FE2F;
|
||
}
|
||
/* greek-ext */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 700;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/8497bc18-0ec5-472d-ad52-0ec0c1a66457/greek-ext.woff2') format('woff2');
|
||
unicode-range: U+1F00-1FFF;
|
||
}
|
||
/* cyrillic */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 700;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/645fc68b-6060-4799-88e7-6974a5c20b24/cyrillic.woff2') format('woff2');
|
||
unicode-range: U+0301, U+0400-045F, U+0490-0491, U+04B0-04B1, U+2116;
|
||
}
|
||
/* latin */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 700;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/d27750f8-d5e8-43f1-b2fb-683fbc0e5d67/latin.woff2') format('woff2');
|
||
unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD;
|
||
}
|
||
/* greek */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 700;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/8975ffda-31ff-4288-9503-2f3d5e52b8ad/greek.woff2') format('woff2');
|
||
unicode-range: U+0370-0377, U+037A-037F, U+0384-038A, U+038C, U+038E-03A1, U+03A3-03FF;
|
||
}
|
||
/* latin */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 300;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/069869f2-ae25-4923-bf14-eef186410e5e/latin.woff2') format('woff2');
|
||
unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD;
|
||
}
|
||
/* greek */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 300;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/e468c517-4a67-4c56-9d79-8594c611509d/greek.woff2') format('woff2');
|
||
unicode-range: U+0370-0377, U+037A-037F, U+0384-038A, U+038C, U+038E-03A1, U+03A3-03FF;
|
||
}
|
||
/* latin-ext */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 300;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/f0b4aa62-7ab7-4adb-9326-323629fdfe18/latin-ext.woff2') format('woff2');
|
||
unicode-range: U+0100-02BA, U+02BD-02C5, U+02C7-02CC, U+02CE-02D7, U+02DD-02FF, U+0304, U+0308, U+0329, U+1D00-1DBF, U+1E00-1E9F, U+1EF2-1EFF, U+2020, U+20A0-20AB, U+20AD-20C0, U+2113, U+2C60-2C7F, U+A720-A7FF;
|
||
}
|
||
/* vietnamese */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 300;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/3d4babfc-31db-48bc-9fe6-e7a817f2f772/vietnamese.woff2') format('woff2');
|
||
unicode-range: U+0102-0103, U+0110-0111, U+0128-0129, U+0168-0169, U+01A0-01A1, U+01AF-01B0, U+0300-0301, U+0303-0304, U+0308-0309, U+0323, U+0329, U+1EA0-1EF9, U+20AB;
|
||
}
|
||
/* cyrillic */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 300;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/ec5dba65-d530-40af-a95b-41b29b367a75/cyrillic.woff2') format('woff2');
|
||
unicode-range: U+0301, U+0400-045F, U+0490-0491, U+04B0-04B1, U+2116;
|
||
}
|
||
/* greek-ext */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 300;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/1ec1dcc5-2033-4de0-a7bc-e44aec87537b/greek-ext.woff2') format('woff2');
|
||
unicode-range: U+1F00-1FFF;
|
||
}
|
||
/* cyrillic-ext */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 300;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/a2697b42-e6b7-4a20-918f-57bb88e6422e/cyrillic-ext.woff2') format('woff2');
|
||
unicode-range: U+0460-052F, U+1C80-1C8A, U+20B4, U+2DE0-2DFF, U+A640-A69F, U+FE2E-FE2F;
|
||
}
|
||
/* cyrillic-ext */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: italic;
|
||
font-weight: 700;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/cdde8e97-8f8e-4c8b-8e1c-83709464aa76/cyrillic-ext.woff2') format('woff2');
|
||
unicode-range: U+0460-052F, U+1C80-1C8A, U+20B4, U+2DE0-2DFF, U+A640-A69F, U+FE2E-FE2F;
|
||
}
|
||
/* latin-ext */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: italic;
|
||
font-weight: 700;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/414bf655-2b27-47dc-9608-c70e5629de8a/latin-ext.woff2') format('woff2');
|
||
unicode-range: U+0100-02BA, U+02BD-02C5, U+02C7-02CC, U+02CE-02D7, U+02DD-02FF, U+0304, U+0308, U+0329, U+1D00-1DBF, U+1E00-1E9F, U+1EF2-1EFF, U+2020, U+20A0-20AB, U+20AD-20C0, U+2113, U+2C60-2C7F, U+A720-A7FF;
|
||
}
|
||
/* vietnamese */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: italic;
|
||
font-weight: 700;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/8824166d-cc4f-4fab-b2b9-eb6f9e5f9e5e/vietnamese.woff2') format('woff2');
|
||
unicode-range: U+0102-0103, U+0110-0111, U+0128-0129, U+0168-0169, U+01A0-01A1, U+01AF-01B0, U+0300-0301, U+0303-0304, U+0308-0309, U+0323, U+0329, U+1EA0-1EF9, U+20AB;
|
||
}
|
||
/* cyrillic */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: italic;
|
||
font-weight: 700;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/42fe1dba-f785-4a67-a6b7-e65beed8c6b2/cyrillic.woff2') format('woff2');
|
||
unicode-range: U+0301, U+0400-045F, U+0490-0491, U+04B0-04B1, U+2116;
|
||
}
|
||
/* latin */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: italic;
|
||
font-weight: 700;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/7e74e5f2-d876-44c7-8585-b35ec27ad385/latin.woff2') format('woff2');
|
||
unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD;
|
||
}
|
||
/* greek */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: italic;
|
||
font-weight: 700;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/2350df97-8a06-4d18-a74b-1b80ea2e680e/greek.woff2') format('woff2');
|
||
unicode-range: U+0370-0377, U+037A-037F, U+0384-038A, U+038C, U+038E-03A1, U+03A3-03FF;
|
||
}
|
||
/* latin */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 400;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/b7dd2236-930f-4d81-9c77-7cec81260193/latin.woff2') format('woff2');
|
||
unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD;
|
||
}
|
||
/* greek-ext */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 400;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/9aa28dc4-4e92-41ce-a6c7-d21e231eb3aa/greek-ext.woff2') format('woff2');
|
||
unicode-range: U+1F00-1FFF;
|
||
}
|
||
/* latin-ext */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 400;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/31fb64b9-1014-4a40-b663-fb70f94ef99c/latin-ext.woff2') format('woff2');
|
||
unicode-range: U+0100-02BA, U+02BD-02C5, U+02C7-02CC, U+02CE-02D7, U+02DD-02FF, U+0304, U+0308, U+0329, U+1D00-1DBF, U+1E00-1E9F, U+1EF2-1EFF, U+2020, U+20A0-20AB, U+20AD-20C0, U+2113, U+2C60-2C7F, U+A720-A7FF;
|
||
}
|
||
/* vietnamese */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 400;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/320d8579-3d91-4288-a181-23f2b5d2f98e/vietnamese.woff2') format('woff2');
|
||
unicode-range: U+0102-0103, U+0110-0111, U+0128-0129, U+0168-0169, U+01A0-01A1, U+01AF-01B0, U+0300-0301, U+0303-0304, U+0308-0309, U+0323, U+0329, U+1EA0-1EF9, U+20AB;
|
||
}
|
||
/* greek */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 400;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/e55f857c-e886-49a2-84b4-4c6e6cbbe27e/greek.woff2') format('woff2');
|
||
unicode-range: U+0370-0377, U+037A-037F, U+0384-038A, U+038C, U+038E-03A1, U+03A3-03FF;
|
||
}
|
||
/* cyrillic-ext */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 400;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/bd06faa7-07c3-4b89-9173-94d0c781c0aa/cyrillic-ext.woff2') format('woff2');
|
||
unicode-range: U+0460-052F, U+1C80-1C8A, U+20B4, U+2DE0-2DFF, U+A640-A69F, U+FE2E-FE2F;
|
||
}
|
||
/* cyrillic */
|
||
@font-face {
|
||
font-family: 'Source Sans Pro';
|
||
font-style: normal;
|
||
font-weight: 400;
|
||
font-display: swap;
|
||
src: url('https://file.squarespace-cdn.com/content/v2/namespaces/fonts/libraries/sqsp/assets/7e399908-f138-4090-a532-e92b10631448/cyrillic.woff2') format('woff2');
|
||
unicode-range: U+0301, U+0400-045F, U+0490-0491, U+04B0-04B1, U+2116;
|
||
}
|
||
</style><script type="text/javascript" crossorigin="anonymous" defer="true" nomodule="nomodule" src="//assets.squarespace.com/@sqs/polyfiller/1.6/legacy.js"></script>
|
||
<script type="text/javascript" crossorigin="anonymous" defer="true" src="//assets.squarespace.com/@sqs/polyfiller/1.6/modern.js"></script>
|
||
<script data-name="static-context">Static = window.Static || {}; Static.SQUARESPACE_CONTEXT = {"betaFeatureFlags":["json_tweaks","output_template_css_assets","campaigns_discount_section_in_automations","enable_modernized_pdp_m3_layout_ux","enable_modernized_standalone_pages","order_status_page_checkout_landing_enabled","enable_form_submission_trigger","modernized-pdp-m2-enabled","i18n_beta_website_locales","enable_modernized_standalone_pages_kill_switch","campaigns_new_image_layout_picker","marketing_automations","campaigns_thumbnail_layout","campaigns_discount_section_in_blasts","campaigns_table_v2","commerce_product_payment_plan_editing","contacts_and_campaigns_redesign","marketing_landing_page","enable_modernized_pdp_m3_fluid","campaigns_import_discounts","campaigns_merch_state"],"facebookAppId":"314192535267336","facebookApiVersion":"v6.0","rollups":{"squarespace-announcement-bar":{"js":"//assets.squarespace.com/universal/scripts-compressed/announcement-bar-df4516beb5aa0028-min.en-US.js"},"squarespace-audio-player":{"css":"//assets.squarespace.com/universal/styles-compressed/audio-player-b05f5197a871c566-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/audio-player-2c392274facfe228-min.en-US.js"},"squarespace-blog-collection-list":{"css":"//assets.squarespace.com/universal/styles-compressed/blog-collection-list-b4046463b72f34e2-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/blog-collection-list-3a5374d437fc6d00-min.en-US.js"},"squarespace-calendar-block-renderer":{"css":"//assets.squarespace.com/universal/styles-compressed/calendar-block-renderer-c3fef2a497c8e56b-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/calendar-block-renderer-26d8e9b90fb9f480-min.en-US.js"},"squarespace-chartjs-helpers":{"css":"//assets.squarespace.com/universal/styles-compressed/chartjs-helpers-96b256171ee039c1-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/chartjs-helpers-522a53304a318d5d-min.en-US.js"},"squarespace-comments":{"css":"//assets.squarespace.com/universal/styles-compressed/comments-23ce383529b75c4f-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/comments-e7efb03b263e7183-min.en-US.js"},"squarespace-custom-css-popup":{"css":"//assets.squarespace.com/universal/styles-compressed/custom-css-popup-ff86295ac1cc1d28-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/custom-css-popup-6b1928f63f535a38-min.en-US.js"},"squarespace-dialog":{"css":"//assets.squarespace.com/universal/styles-compressed/dialog-86aacd645d83874c-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/dialog-5a370ceb3b783b61-min.en-US.js"},"squarespace-events-collection":{"css":"//assets.squarespace.com/universal/styles-compressed/events-collection-c3fef2a497c8e56b-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/events-collection-53a2a66e197347e0-min.en-US.js"},"squarespace-form-rendering-utils":{"js":"//assets.squarespace.com/universal/scripts-compressed/form-rendering-utils-4d2ae44cbdf7308c-min.en-US.js"},"squarespace-forms":{"css":"//assets.squarespace.com/universal/styles-compressed/forms-0afd3c6ac30bbab1-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/forms-6d9e12a028dd1efc-min.en-US.js"},"squarespace-gallery-collection-list":{"css":"//assets.squarespace.com/universal/styles-compressed/gallery-collection-list-b4046463b72f34e2-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/gallery-collection-list-e50dc2cea7153bb8-min.en-US.js"},"squarespace-image-zoom":{"css":"//assets.squarespace.com/universal/styles-compressed/image-zoom-b4046463b72f34e2-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/image-zoom-0ca1efc2ac068abc-min.en-US.js"},"squarespace-pinterest":{"css":"//assets.squarespace.com/universal/styles-compressed/pinterest-b4046463b72f34e2-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/pinterest-17423cbbf8d04380-min.en-US.js"},"squarespace-popup-overlay":{"css":"//assets.squarespace.com/universal/styles-compressed/popup-overlay-b742b752f5880972-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/popup-overlay-d14ed9411116d82d-min.en-US.js"},"squarespace-product-quick-view":{"css":"//assets.squarespace.com/universal/styles-compressed/product-quick-view-0afd3c6ac30bbab1-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/product-quick-view-292b172c00faedda-min.en-US.js"},"squarespace-products-collection-item-v2":{"css":"//assets.squarespace.com/universal/styles-compressed/products-collection-item-v2-b4046463b72f34e2-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/products-collection-item-v2-7fbb3f7b25936e86-min.en-US.js"},"squarespace-products-collection-list-v2":{"css":"//assets.squarespace.com/universal/styles-compressed/products-collection-list-v2-b4046463b72f34e2-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/products-collection-list-v2-a0f681feaaa1d600-min.en-US.js"},"squarespace-search-page":{"css":"//assets.squarespace.com/universal/styles-compressed/search-page-90a67fc09b9b32c6-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/search-page-1af2c4e17252dac5-min.en-US.js"},"squarespace-search-preview":{"js":"//assets.squarespace.com/universal/scripts-compressed/search-preview-0b8ae236f536a890-min.en-US.js"},"squarespace-simple-liking":{"css":"//assets.squarespace.com/universal/styles-compressed/simple-liking-701bf8bbc05ec6aa-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/simple-liking-75b6582783ed07a5-min.en-US.js"},"squarespace-social-buttons":{"css":"//assets.squarespace.com/universal/styles-compressed/social-buttons-95032e5fa98e47a5-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/social-buttons-322494b0519c3493-min.en-US.js"},"squarespace-tourdates":{"css":"//assets.squarespace.com/universal/styles-compressed/tourdates-b4046463b72f34e2-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/tourdates-268b851e0659685f-min.en-US.js"},"squarespace-website-overlays-manager":{"css":"//assets.squarespace.com/universal/styles-compressed/website-overlays-manager-07ea5a4e004e6710-min.en-US.css","js":"//assets.squarespace.com/universal/scripts-compressed/website-overlays-manager-b34cb8a41e4a5843-min.en-US.js"}},"pageType":50,"website":{"id":"57a6de391b631b6aa4789ebf","captchaSettings":{"enabledForDonations":false},"cloneable":false,"contentModifiedOn":1619028512381,"formattingLocale":"en-US","fullSiteTitle":"The Irreproducibility of Bugs in Large-Scale Production Systems \u2014 Susan Fowler","identifier":"susan-fowler-b4lr","imageMetadataProcessingEnabled":false,"isHstsEnabled":false,"language":"en-US","location":{"addressLine1":"","addressLine2":"","addressCountry":""},"primaryDomain":"www.susanjfowler.com","screenshotId":"18b81ae44a07c9c30d51add86c2e2b4c2fda75f44ffd83ef6d12493a48627109","shareButtonOptions":{"6":true,"3":true,"2":true,"1":true,"4":true},"showOwnerLogin":false,"siteDescription":"<p>Writer. Engineer. Fledgling Physicist. Amateur Myrmecologist. </p>","siteStatus":{},"siteTagLine":"Writer. Editor. Fledgling Physicist. Amateur Myrmecologist. ","siteTitle":"Susan Fowler","socialLogoImageId":"5dd82f25dcadfc116d1e5b99","sslSetting":3,"timeZone":"America/Los_Angeles","translationLocale":"en-US","typekitId":"","websiteType":1,"statsMigrated":false,"authenticUrl":"https://www.susanjfowler.com","baseUrl":"https://www.susanjfowler.com","internalUrl":"https://susan-fowler-b4lr.squarespace.com","machineTimeZoneOffset":-25200000,"socialLogoImageUrl":"//images.squarespace-cdn.com/content/v1/57a6de391b631b6aa4789ebf/1574448935393-AVZS84QWNCJQL9KMCY5M/91t6KiP1cEL.jpg","timeZoneAbbr":"PDT","timeZoneOffset":-25200000},"websiteSettings":{"id":"57a6de391b631b6aa4789ec1","websiteId":"57a6de391b631b6aa4789ebf","type":"Personal","subjects":[],"country":"US","state":"CA","simpleLikingEnabled":true,"mobileInfoBarSettings":{"isContactEmailEnabled":false,"isContactPhoneNumberEnabled":false,"isLocationEnabled":false,"isBusinessHoursEnabled":false},"commentLikesAllowed":true,"commentAnonAllowed":true,"commentThreaded":true,"commentApprovalRequired":true,"commentAvatarsOn":true,"commentSortType":2,"commentFlagThreshold":0,"commentFlagsAllowed":true,"commentEnableByDefault":true,"commentDisableAfterDaysDefault":0,"disqusShortname":"","commentsEnabled":true,"contactPhoneNumber":"","businessHours":{"monday":{"text":"","ranges":[{}]},"tuesday":{"text":"","ranges":[{}]},"wednesday":{"text":"","ranges":[{}]},"thursday":{"text":"","ranges":[{}]},"friday":{"text":"","ranges":[{}]},"saturday":{"text":"","ranges":[{}]},"sunday":{"text":"","ranges":[{}]}},"storeSettings":{"returnPolicy":null,"termsOfService":null,"privacyPolicy":null,"expressCheckout":false,"continueShoppingLinkUrl":"/","useLightCart":false,"showNoteField":false,"shippingCountryDefaultValue":"US","billToShippingDefaultValue":false,"showShippingPhoneNumber":true,"isShippingPhoneRequired":false,"showBillingPhoneNumber":true,"isBillingPhoneRequired":false,"currenciesSupported":["CHF","HKD","MXN","EUR","DKK","USD","CAD","MYR","NOK","THB","AUD","SGD","ILS","PLN","GBP","CZK","SEK","NZD","PHP","RUB"],"defaultCurrency":"USD","selectedCurrency":"USD","measurementStandard":1,"showCustomCheckoutForm":false,"checkoutPageMarketingOptInEnabled":false,"enableMailingListOptInByDefault":true,"sameAsRetailLocation":false,"merchandisingSettings":{"scarcityEnabledOnProductItems":false,"scarcityEnabledOnProductBlocks":false,"scarcityMessageType":"DEFAULT_SCARCITY_MESSAGE","scarcityThreshold":10,"multipleQuantityAllowedForServices":true,"restockNotificationsEnabled":false,"restockNotificationsSuccessText":"","restockNotificationsMailingListSignUpEnabled":false,"relatedProductsEnabled":false,"relatedProductsOrdering":"random","soldOutVariantsDropdownDisabled":false,"productComposerOptedIn":false,"productComposerABTestOptedOut":false,"productReviewsEnabled":false,"displayImportedProductReviewsEnabled":false,"hasOptedToCollectNativeReviews":false},"minimumOrderSubtotalEnabled":false,"addToCartConfirmationType":2,"isLive":false,"multipleQuantityAllowedForServices":true},"useEscapeKeyToLogin":true,"ssBadgeType":1,"ssBadgePosition":4,"ssBadgeVisibility":1,"ssBadgeDevices":1,"pinterestOverlayOptions":{"mode":"disabled"},"userAccountsSettings":{"loginAllowed":false,"signupAllowed":false}},"cookieSettings":{"isCookieBannerEnabled":false,"isRestrictiveCookiePolicyEnabled":false,"cookieBannerText":"<p>Select \u201CAccept all\u201D to agree to our use of cookies and similar technologies to enhance your browsing experience, security, analytics and customization. Select \u201DManage cookies\u201D to make more choices or opt out.</p>","cookieBannerTheme":"DARK","cookieBannerPosition":"BOTTOM","cookieBannerCtaText":"Accept all","cookieBannerAcceptType":"OPT_IN","cookieBannerOptOutCtaText":"Decline all","cookieBannerHasOptOut":false,"cookieBannerHasManageCookies":true,"cookieBannerManageCookiesLabel":"Manage cookies","cookieBannerSavedPreferencesText":"Cookie Preferences","cookieBannerSavedPreferencesLayout":"PILL"},"websiteCloneable":false,"collection":{"title":"Blog","id":"57a6e2b2579fb3dbed177a34","fullUrl":"/blog","type":1,"permissionType":1},"item":{"title":"The Irreproducibility of Bugs in Large-Scale Production Systems ","id":"581ad9b11b631b780e8f8466","fullUrl":"/blog/2016/11/2/the-irreproducibility-of-bugs-in-large-scale-production-systems","publicCommentCount":3,"commentState":1,"recordType":1},"subscribed":false,"appDomain":"squarespace.com","templateTweakable":true,"tweakJSON":{"outerPadding":"65px","pagePadding":"100px","product-gallery-auto-crop":"true","product-image-auto-crop":"false","topPadding":"80px","tweak-v1-related-products-title-spacing":"50px"},"templateId":"50521cf884aeb45fa5cfdb80","templateVersion":"7","pageFeatures":[1,2,4],"gmRenderKey":"QUl6YVN5Q0JUUk9xNkx1dkZfSUUxcjQ2LVQ0QWVUU1YtMGQ3bXk4","templateScriptsRootUrl":"https://static1.squarespace.com/static/ta/5052176b84aeb45fa5cfcc83/968/scripts/","impersonatedSession":false,"tzData":{"zones":[[-480,"US","P%sT",null]],"rules":{"US":[[1967,2006,null,"Oct","lastSun","2:00","0","S"],[1987,2006,null,"Apr","Sun>=1","2:00","1:00","D"],[2007,"max",null,"Mar","Sun>=8","2:00","1:00","D"],[2007,"max",null,"Nov","Sun>=1","2:00","0","S"]]}},"showAnnouncementBar":false,"recaptchaEnterpriseContext":{"recaptchaEnterpriseSiteKey":"6LdDFQwjAAAAAPigEvvPgEVbb7QBm-TkVJdDTlAv"},"i18nContext":{"timeZoneData":{"id":"America/Los_Angeles","name":"Pacific Time"}},"env":"PRODUCTION","visitorFormContext":{"formFieldFormats":{"initialPhoneFormat":{"id":0,"type":"PHONE_NUMBER","country":"US","labelLocale":"en-US","fields":[{"type":"SEPARATOR","label":"(","identifier":"LeftParen","length":0,"required":false,"metadata":{}},{"type":"FIELD","label":"1","identifier":"1","length":3,"required":false,"metadata":{}},{"type":"SEPARATOR","label":")","identifier":"RightParen","length":0,"required":false,"metadata":{}},{"type":"SEPARATOR","label":" ","identifier":"Space","length":0,"required":false,"metadata":{}},{"type":"FIELD","label":"2","identifier":"2","length":3,"required":false,"metadata":{}},{"type":"SEPARATOR","label":"-","identifier":"Dash","length":0,"required":false,"metadata":{}},{"type":"FIELD","label":"3","identifier":"3","length":14,"required":false,"metadata":{}}]},"initialNameOrder":"GIVEN_FIRST","initialAddressFormat":{"id":0,"type":"ADDRESS","country":"US","labelLocale":"en","fields":[{"type":"FIELD","label":"Address Line 1","identifier":"Line1","length":0,"required":true,"metadata":{"autocomplete":"address-line1"}},{"type":"SEPARATOR","label":"\n","identifier":"Newline","length":0,"required":false,"metadata":{}},{"type":"FIELD","label":"Address Line 2","identifier":"Line2","length":0,"required":false,"metadata":{"autocomplete":"address-line2"}},{"type":"SEPARATOR","label":"\n","identifier":"Newline","length":0,"required":false,"metadata":{}},{"type":"FIELD","label":"City","identifier":"City","length":0,"required":true,"metadata":{"autocomplete":"address-level2"}},{"type":"SEPARATOR","label":",","identifier":"Comma","length":0,"required":false,"metadata":{}},{"type":"SEPARATOR","label":" ","identifier":"Space","length":0,"required":false,"metadata":{}},{"type":"FIELD","label":"State","identifier":"State","length":0,"required":true,"metadata":{"autocomplete":"address-level1"}},{"type":"SEPARATOR","label":" ","identifier":"Space","length":0,"required":false,"metadata":{}},{"type":"FIELD","label":"ZIP Code","identifier":"Zip","length":0,"required":true,"metadata":{"autocomplete":"postal-code"}}]},"countries":[{"name":"Afghanistan","code":"AF","phoneCode":"+93"},{"name":"\u00C5land Islands","code":"AX","phoneCode":"+358"},{"name":"Albania","code":"AL","phoneCode":"+355"},{"name":"Algeria","code":"DZ","phoneCode":"+213"},{"name":"American Samoa","code":"AS","phoneCode":"+1"},{"name":"Andorra","code":"AD","phoneCode":"+376"},{"name":"Angola","code":"AO","phoneCode":"+244"},{"name":"Anguilla","code":"AI","phoneCode":"+1"},{"name":"Antigua & Barbuda","code":"AG","phoneCode":"+1"},{"name":"Argentina","code":"AR","phoneCode":"+54"},{"name":"Armenia","code":"AM","phoneCode":"+374"},{"name":"Aruba","code":"AW","phoneCode":"+297"},{"name":"Ascension Island","code":"AC","phoneCode":"+247"},{"name":"Australia","code":"AU","phoneCode":"+61"},{"name":"Austria","code":"AT","phoneCode":"+43"},{"name":"Azerbaijan","code":"AZ","phoneCode":"+994"},{"name":"Bahamas","code":"BS","phoneCode":"+1"},{"name":"Bahrain","code":"BH","phoneCode":"+973"},{"name":"Bangladesh","code":"BD","phoneCode":"+880"},{"name":"Barbados","code":"BB","phoneCode":"+1"},{"name":"Belarus","code":"BY","phoneCode":"+375"},{"name":"Belgium","code":"BE","phoneCode":"+32"},{"name":"Belize","code":"BZ","phoneCode":"+501"},{"name":"Benin","code":"BJ","phoneCode":"+229"},{"name":"Bermuda","code":"BM","phoneCode":"+1"},{"name":"Bhutan","code":"BT","phoneCode":"+975"},{"name":"Bolivia","code":"BO","phoneCode":"+591"},{"name":"Bosnia & Herzegovina","code":"BA","phoneCode":"+387"},{"name":"Botswana","code":"BW","phoneCode":"+267"},{"name":"Brazil","code":"BR","phoneCode":"+55"},{"name":"British Indian Ocean Territory","code":"IO","phoneCode":"+246"},{"name":"British Virgin Islands","code":"VG","phoneCode":"+1"},{"name":"Brunei","code":"BN","phoneCode":"+673"},{"name":"Bulgaria","code":"BG","phoneCode":"+359"},{"name":"Burkina Faso","code":"BF","phoneCode":"+226"},{"name":"Burundi","code":"BI","phoneCode":"+257"},{"name":"Cambodia","code":"KH","phoneCode":"+855"},{"name":"Cameroon","code":"CM","phoneCode":"+237"},{"name":"Canada","code":"CA","phoneCode":"+1"},{"name":"Cape Verde","code":"CV","phoneCode":"+238"},{"name":"Caribbean Netherlands","code":"BQ","phoneCode":"+599"},{"name":"Cayman Islands","code":"KY","phoneCode":"+1"},{"name":"Central African Republic","code":"CF","phoneCode":"+236"},{"name":"Chad","code":"TD","phoneCode":"+235"},{"name":"Chile","code":"CL","phoneCode":"+56"},{"name":"China","code":"CN","phoneCode":"+86"},{"name":"Christmas Island","code":"CX","phoneCode":"+61"},{"name":"Cocos (Keeling) Islands","code":"CC","phoneCode":"+61"},{"name":"Colombia","code":"CO","phoneCode":"+57"},{"name":"Comoros","code":"KM","phoneCode":"+269"},{"name":"Congo - Brazzaville","code":"CG","phoneCode":"+242"},{"name":"Congo - Kinshasa","code":"CD","phoneCode":"+243"},{"name":"Cook Islands","code":"CK","phoneCode":"+682"},{"name":"Costa Rica","code":"CR","phoneCode":"+506"},{"name":"C\u00F4te d\u2019Ivoire","code":"CI","phoneCode":"+225"},{"name":"Croatia","code":"HR","phoneCode":"+385"},{"name":"Cuba","code":"CU","phoneCode":"+53"},{"name":"Cura\u00E7ao","code":"CW","phoneCode":"+599"},{"name":"Cyprus","code":"CY","phoneCode":"+357"},{"name":"Czechia","code":"CZ","phoneCode":"+420"},{"name":"Denmark","code":"DK","phoneCode":"+45"},{"name":"Djibouti","code":"DJ","phoneCode":"+253"},{"name":"Dominica","code":"DM","phoneCode":"+1"},{"name":"Dominican Republic","code":"DO","phoneCode":"+1"},{"name":"Ecuador","code":"EC","phoneCode":"+593"},{"name":"Egypt","code":"EG","phoneCode":"+20"},{"name":"El Salvador","code":"SV","phoneCode":"+503"},{"name":"Equatorial Guinea","code":"GQ","phoneCode":"+240"},{"name":"Eritrea","code":"ER","phoneCode":"+291"},{"name":"Estonia","code":"EE","phoneCode":"+372"},{"name":"Eswatini","code":"SZ","phoneCode":"+268"},{"name":"Ethiopia","code":"ET","phoneCode":"+251"},{"name":"Falkland Islands","code":"FK","phoneCode":"+500"},{"name":"Faroe Islands","code":"FO","phoneCode":"+298"},{"name":"Fiji","code":"FJ","phoneCode":"+679"},{"name":"Finland","code":"FI","phoneCode":"+358"},{"name":"France","code":"FR","phoneCode":"+33"},{"name":"French Guiana","code":"GF","phoneCode":"+594"},{"name":"French Polynesia","code":"PF","phoneCode":"+689"},{"name":"Gabon","code":"GA","phoneCode":"+241"},{"name":"Gambia","code":"GM","phoneCode":"+220"},{"name":"Georgia","code":"GE","phoneCode":"+995"},{"name":"Germany","code":"DE","phoneCode":"+49"},{"name":"Ghana","code":"GH","phoneCode":"+233"},{"name":"Gibraltar","code":"GI","phoneCode":"+350"},{"name":"Greece","code":"GR","phoneCode":"+30"},{"name":"Greenland","code":"GL","phoneCode":"+299"},{"name":"Grenada","code":"GD","phoneCode":"+1"},{"name":"Guadeloupe","code":"GP","phoneCode":"+590"},{"name":"Guam","code":"GU","phoneCode":"+1"},{"name":"Guatemala","code":"GT","phoneCode":"+502"},{"name":"Guernsey","code":"GG","phoneCode":"+44"},{"name":"Guinea","code":"GN","phoneCode":"+224"},{"name":"Guinea-Bissau","code":"GW","phoneCode":"+245"},{"name":"Guyana","code":"GY","phoneCode":"+592"},{"name":"Haiti","code":"HT","phoneCode":"+509"},{"name":"Honduras","code":"HN","phoneCode":"+504"},{"name":"Hong Kong SAR China","code":"HK","phoneCode":"+852"},{"name":"Hungary","code":"HU","phoneCode":"+36"},{"name":"Iceland","code":"IS","phoneCode":"+354"},{"name":"India","code":"IN","phoneCode":"+91"},{"name":"Indonesia","code":"ID","phoneCode":"+62"},{"name":"Iran","code":"IR","phoneCode":"+98"},{"name":"Iraq","code":"IQ","phoneCode":"+964"},{"name":"Ireland","code":"IE","phoneCode":"+353"},{"name":"Isle of Man","code":"IM","phoneCode":"+44"},{"name":"Israel","code":"IL","phoneCode":"+972"},{"name":"Italy","code":"IT","phoneCode":"+39"},{"name":"Jamaica","code":"JM","phoneCode":"+1"},{"name":"Japan","code":"JP","phoneCode":"+81"},{"name":"Jersey","code":"JE","phoneCode":"+44"},{"name":"Jordan","code":"JO","phoneCode":"+962"},{"name":"Kazakhstan","code":"KZ","phoneCode":"+7"},{"name":"Kenya","code":"KE","phoneCode":"+254"},{"name":"Kiribati","code":"KI","phoneCode":"+686"},{"name":"Kosovo","code":"XK","phoneCode":"+383"},{"name":"Kuwait","code":"KW","phoneCode":"+965"},{"name":"Kyrgyzstan","code":"KG","phoneCode":"+996"},{"name":"Laos","code":"LA","phoneCode":"+856"},{"name":"Latvia","code":"LV","phoneCode":"+371"},{"name":"Lebanon","code":"LB","phoneCode":"+961"},{"name":"Lesotho","code":"LS","phoneCode":"+266"},{"name":"Liberia","code":"LR","phoneCode":"+231"},{"name":"Libya","code":"LY","phoneCode":"+218"},{"name":"Liechtenstein","code":"LI","phoneCode":"+423"},{"name":"Lithuania","code":"LT","phoneCode":"+370"},{"name":"Luxembourg","code":"LU","phoneCode":"+352"},{"name":"Macao SAR China","code":"MO","phoneCode":"+853"},{"name":"Madagascar","code":"MG","phoneCode":"+261"},{"name":"Malawi","code":"MW","phoneCode":"+265"},{"name":"Malaysia","code":"MY","phoneCode":"+60"},{"name":"Maldives","code":"MV","phoneCode":"+960"},{"name":"Mali","code":"ML","phoneCode":"+223"},{"name":"Malta","code":"MT","phoneCode":"+356"},{"name":"Marshall Islands","code":"MH","phoneCode":"+692"},{"name":"Martinique","code":"MQ","phoneCode":"+596"},{"name":"Mauritania","code":"MR","phoneCode":"+222"},{"name":"Mauritius","code":"MU","phoneCode":"+230"},{"name":"Mayotte","code":"YT","phoneCode":"+262"},{"name":"Mexico","code":"MX","phoneCode":"+52"},{"name":"Micronesia","code":"FM","phoneCode":"+691"},{"name":"Moldova","code":"MD","phoneCode":"+373"},{"name":"Monaco","code":"MC","phoneCode":"+377"},{"name":"Mongolia","code":"MN","phoneCode":"+976"},{"name":"Montenegro","code":"ME","phoneCode":"+382"},{"name":"Montserrat","code":"MS","phoneCode":"+1"},{"name":"Morocco","code":"MA","phoneCode":"+212"},{"name":"Mozambique","code":"MZ","phoneCode":"+258"},{"name":"Myanmar (Burma)","code":"MM","phoneCode":"+95"},{"name":"Namibia","code":"NA","phoneCode":"+264"},{"name":"Nauru","code":"NR","phoneCode":"+674"},{"name":"Nepal","code":"NP","phoneCode":"+977"},{"name":"Netherlands","code":"NL","phoneCode":"+31"},{"name":"New Caledonia","code":"NC","phoneCode":"+687"},{"name":"New Zealand","code":"NZ","phoneCode":"+64"},{"name":"Nicaragua","code":"NI","phoneCode":"+505"},{"name":"Niger","code":"NE","phoneCode":"+227"},{"name":"Nigeria","code":"NG","phoneCode":"+234"},{"name":"Niue","code":"NU","phoneCode":"+683"},{"name":"Norfolk Island","code":"NF","phoneCode":"+672"},{"name":"Northern Mariana Islands","code":"MP","phoneCode":"+1"},{"name":"North Korea","code":"KP","phoneCode":"+850"},{"name":"North Macedonia","code":"MK","phoneCode":"+389"},{"name":"Norway","code":"NO","phoneCode":"+47"},{"name":"Oman","code":"OM","phoneCode":"+968"},{"name":"Pakistan","code":"PK","phoneCode":"+92"},{"name":"Palau","code":"PW","phoneCode":"+680"},{"name":"Palestinian Territories","code":"PS","phoneCode":"+970"},{"name":"Panama","code":"PA","phoneCode":"+507"},{"name":"Papua New Guinea","code":"PG","phoneCode":"+675"},{"name":"Paraguay","code":"PY","phoneCode":"+595"},{"name":"Peru","code":"PE","phoneCode":"+51"},{"name":"Philippines","code":"PH","phoneCode":"+63"},{"name":"Poland","code":"PL","phoneCode":"+48"},{"name":"Portugal","code":"PT","phoneCode":"+351"},{"name":"Puerto Rico","code":"PR","phoneCode":"+1"},{"name":"Qatar","code":"QA","phoneCode":"+974"},{"name":"R\u00E9union","code":"RE","phoneCode":"+262"},{"name":"Romania","code":"RO","phoneCode":"+40"},{"name":"Russia","code":"RU","phoneCode":"+7"},{"name":"Rwanda","code":"RW","phoneCode":"+250"},{"name":"Samoa","code":"WS","phoneCode":"+685"},{"name":"San Marino","code":"SM","phoneCode":"+378"},{"name":"S\u00E3o Tom\u00E9 & Pr\u00EDncipe","code":"ST","phoneCode":"+239"},{"name":"Saudi Arabia","code":"SA","phoneCode":"+966"},{"name":"Senegal","code":"SN","phoneCode":"+221"},{"name":"Serbia","code":"RS","phoneCode":"+381"},{"name":"Seychelles","code":"SC","phoneCode":"+248"},{"name":"Sierra Leone","code":"SL","phoneCode":"+232"},{"name":"Singapore","code":"SG","phoneCode":"+65"},{"name":"Sint Maarten","code":"SX","phoneCode":"+1"},{"name":"Slovakia","code":"SK","phoneCode":"+421"},{"name":"Slovenia","code":"SI","phoneCode":"+386"},{"name":"Solomon Islands","code":"SB","phoneCode":"+677"},{"name":"Somalia","code":"SO","phoneCode":"+252"},{"name":"South Africa","code":"ZA","phoneCode":"+27"},{"name":"South Korea","code":"KR","phoneCode":"+82"},{"name":"South Sudan","code":"SS","phoneCode":"+211"},{"name":"Spain","code":"ES","phoneCode":"+34"},{"name":"Sri Lanka","code":"LK","phoneCode":"+94"},{"name":"St. Barth\u00E9lemy","code":"BL","phoneCode":"+590"},{"name":"St. Helena","code":"SH","phoneCode":"+290"},{"name":"St. Kitts & Nevis","code":"KN","phoneCode":"+1"},{"name":"St. Lucia","code":"LC","phoneCode":"+1"},{"name":"St. Martin","code":"MF","phoneCode":"+590"},{"name":"St. Pierre & Miquelon","code":"PM","phoneCode":"+508"},{"name":"St. Vincent & Grenadines","code":"VC","phoneCode":"+1"},{"name":"Sudan","code":"SD","phoneCode":"+249"},{"name":"Suriname","code":"SR","phoneCode":"+597"},{"name":"Svalbard & Jan Mayen","code":"SJ","phoneCode":"+47"},{"name":"Sweden","code":"SE","phoneCode":"+46"},{"name":"Switzerland","code":"CH","phoneCode":"+41"},{"name":"Syria","code":"SY","phoneCode":"+963"},{"name":"Taiwan","code":"TW","phoneCode":"+886"},{"name":"Tajikistan","code":"TJ","phoneCode":"+992"},{"name":"Tanzania","code":"TZ","phoneCode":"+255"},{"name":"Thailand","code":"TH","phoneCode":"+66"},{"name":"Timor-Leste","code":"TL","phoneCode":"+670"},{"name":"Togo","code":"TG","phoneCode":"+228"},{"name":"Tokelau","code":"TK","phoneCode":"+690"},{"name":"Tonga","code":"TO","phoneCode":"+676"},{"name":"Trinidad & Tobago","code":"TT","phoneCode":"+1"},{"name":"Tristan da Cunha","code":"TA","phoneCode":"+290"},{"name":"Tunisia","code":"TN","phoneCode":"+216"},{"name":"T\u00FCrkiye","code":"TR","phoneCode":"+90"},{"name":"Turkmenistan","code":"TM","phoneCode":"+993"},{"name":"Turks & Caicos Islands","code":"TC","phoneCode":"+1"},{"name":"Tuvalu","code":"TV","phoneCode":"+688"},{"name":"U.S. Virgin Islands","code":"VI","phoneCode":"+1"},{"name":"Uganda","code":"UG","phoneCode":"+256"},{"name":"Ukraine","code":"UA","phoneCode":"+380"},{"name":"United Arab Emirates","code":"AE","phoneCode":"+971"},{"name":"United Kingdom","code":"GB","phoneCode":"+44"},{"name":"United States","code":"US","phoneCode":"+1"},{"name":"Uruguay","code":"UY","phoneCode":"+598"},{"name":"Uzbekistan","code":"UZ","phoneCode":"+998"},{"name":"Vanuatu","code":"VU","phoneCode":"+678"},{"name":"Vatican City","code":"VA","phoneCode":"+39"},{"name":"Venezuela","code":"VE","phoneCode":"+58"},{"name":"Vietnam","code":"VN","phoneCode":"+84"},{"name":"Wallis & Futuna","code":"WF","phoneCode":"+681"},{"name":"Western Sahara","code":"EH","phoneCode":"+212"},{"name":"Yemen","code":"YE","phoneCode":"+967"},{"name":"Zambia","code":"ZM","phoneCode":"+260"},{"name":"Zimbabwe","code":"ZW","phoneCode":"+263"}]},"localizedStrings":{"validation":{"noValidSelection":"A valid selection must be made.","invalidUrl":"Must be a valid URL.","stringTooLong":"Value should have a length no longer than {0}.","containsInvalidKey":"{0} contains an invalid key.","invalidTwitterUsername":"Must be a valid Twitter username.","valueOutsideRange":"Value must be in the range {0} to {1}.","invalidPassword":"Passwords should not contain whitespace.","missingRequiredSubfields":"{0} is missing required subfields: {1}","invalidCurrency":"Currency value should be formatted like 1234 or 123.99.","invalidMapSize":"Value should contain exactly {0} elements.","subfieldsRequired":"All fields in {0} are required.","formSubmissionFailed":"Form submission failed. Review the following information: {0}.","invalidCountryCode":"Country code should have an optional plus and up to 4 digits.","invalidDate":"This is not a real date.","required":"{0} is required.","invalidStringLength":"Value should be {0} characters long.","invalidEmail":"Email addresses should follow the format user@domain.com.","invalidListLength":"Value should be {0} elements long.","allEmpty":"Please fill out at least one form field.","missingRequiredQuestion":"Missing a required question.","invalidQuestion":"Contained an invalid question.","captchaFailure":"Captcha validation failed. Please try again.","stringTooShort":"Value should have a length of at least {0}.","invalid":"{0} is not valid.","formErrors":"Form Errors","containsInvalidValue":"{0} contains an invalid value.","invalidUnsignedNumber":"Numbers must contain only digits and no other characters.","invalidName":"Valid names contain only letters, numbers, spaces, ', or - characters."},"submit":"Submit","status":{"title":"{@} Block","learnMore":"Learn more"},"name":{"firstName":"First Name","lastName":"Last Name"},"lightbox":{"openForm":"Open Form"},"likert":{"agree":"Agree","stronglyDisagree":"Strongly Disagree","disagree":"Disagree","stronglyAgree":"Strongly Agree","neutral":"Neutral"},"time":{"am":"AM","second":"Second","pm":"PM","minute":"Minute","amPm":"AM/PM","hour":"Hour"},"notFound":"Form not found.","date":{"yyyy":"YYYY","year":"Year","mm":"MM","day":"Day","month":"Month","dd":"DD"},"phone":{"country":"Country","number":"Number","prefix":"Prefix","areaCode":"Area Code","line":"Line"},"submitError":"Unable to submit form. Please try again later.","address":{"stateProvince":"State/Province","country":"Country","zipPostalCode":"Zip/Postal Code","address2":"Address 2","address1":"Address 1","city":"City"},"email":{"signUp":"Sign up for news and updates"},"required":"(required)","invalidData":"Invalid form data."}}};</script><script type="text/javascript">SQUARESPACE_ROLLUPS = {};</script>
|
||
<script>(function(rollups, name) { if (!rollups[name]) { rollups[name] = {}; } rollups[name].js = ["//assets.squarespace.com/universal/scripts-compressed/visitor-site-error-reporter-51e4c647fcd6492e-min.en-US.js"]; })(SQUARESPACE_ROLLUPS, 'squarespace-visitor_site_error_reporter');</script>
|
||
<script crossorigin="anonymous" src="//assets.squarespace.com/universal/scripts-compressed/visitor-site-error-reporter-51e4c647fcd6492e-min.en-US.js" defer ></script><script>(function(rollups, name) { if (!rollups[name]) { rollups[name] = {}; } rollups[name].js = ["//assets.squarespace.com/universal/scripts-compressed/extract-css-runtime-a62919b2710bbacb-min.en-US.js"]; })(SQUARESPACE_ROLLUPS, 'squarespace-extract_css_runtime');</script>
|
||
<script crossorigin="anonymous" src="//assets.squarespace.com/universal/scripts-compressed/extract-css-runtime-a62919b2710bbacb-min.en-US.js" defer ></script><script>(function(rollups, name) { if (!rollups[name]) { rollups[name] = {}; } rollups[name].js = ["//assets.squarespace.com/universal/scripts-compressed/extract-css-moment-js-vendor-a5cfdec1ae227f33-min.en-US.js"]; })(SQUARESPACE_ROLLUPS, 'squarespace-extract_css_moment_js_vendor');</script>
|
||
<script crossorigin="anonymous" src="//assets.squarespace.com/universal/scripts-compressed/extract-css-moment-js-vendor-a5cfdec1ae227f33-min.en-US.js" defer ></script><script>(function(rollups, name) { if (!rollups[name]) { rollups[name] = {}; } rollups[name].js = ["//assets.squarespace.com/universal/scripts-compressed/cldr-resource-pack-2c5e18cab442f0cf-min.en-US.js"]; })(SQUARESPACE_ROLLUPS, 'squarespace-cldr_resource_pack');</script>
|
||
<script crossorigin="anonymous" src="//assets.squarespace.com/universal/scripts-compressed/cldr-resource-pack-2c5e18cab442f0cf-min.en-US.js" defer ></script><script>(function(rollups, name) { if (!rollups[name]) { rollups[name] = {}; } rollups[name].js = ["//assets.squarespace.com/universal/scripts-compressed/common-vendors-stable-784b947826b4c445-min.en-US.js"]; })(SQUARESPACE_ROLLUPS, 'squarespace-common_vendors_stable');</script>
|
||
<script crossorigin="anonymous" src="//assets.squarespace.com/universal/scripts-compressed/common-vendors-stable-784b947826b4c445-min.en-US.js" defer ></script><script>(function(rollups, name) { if (!rollups[name]) { rollups[name] = {}; } rollups[name].js = ["//assets.squarespace.com/universal/scripts-compressed/common-vendors-b318b15483dd810a-min.en-US.js"]; })(SQUARESPACE_ROLLUPS, 'squarespace-common_vendors');</script>
|
||
<script crossorigin="anonymous" src="//assets.squarespace.com/universal/scripts-compressed/common-vendors-b318b15483dd810a-min.en-US.js" defer ></script><script>(function(rollups, name) { if (!rollups[name]) { rollups[name] = {}; } rollups[name].js = ["//assets.squarespace.com/universal/scripts-compressed/common-3845c04b3630118d-min.en-US.js"]; })(SQUARESPACE_ROLLUPS, 'squarespace-common');</script>
|
||
<script crossorigin="anonymous" src="//assets.squarespace.com/universal/scripts-compressed/common-3845c04b3630118d-min.en-US.js" defer ></script><script>(function(rollups, name) { if (!rollups[name]) { rollups[name] = {}; } rollups[name].js = ["//assets.squarespace.com/universal/scripts-compressed/performance-4eb6e657033ca06d-min.en-US.js"]; })(SQUARESPACE_ROLLUPS, 'squarespace-performance');</script>
|
||
<script crossorigin="anonymous" src="//assets.squarespace.com/universal/scripts-compressed/performance-4eb6e657033ca06d-min.en-US.js" defer ></script><script type="module">Squarespace.load(window);</script>
|
||
<script data-sqs-type="imageloader-bootstrapper" type="module">if(window.ImageLoader) window.ImageLoader.bootstrap({}, document);</script>
|
||
<script type="module">Squarespace.afterBodyLoad(Y);</script>
|
||
<link rel="alternate" type="application/rss+xml" title="RSS Feed" href="https://www.susanjfowler.com/blog?format=rss" />
|
||
<script type="application/ld+json">{"url":"https://www.susanjfowler.com","name":"Susan Fowler","description":"<p>Writer. Engineer. Fledgling Physicist. Amateur Myrmecologist. </p>","@context":"http://schema.org","@type":"WebSite"}</script><script type="application/ld+json">{"address":"","openingHours":", , , , , , ","@context":"http://schema.org","@type":"LocalBusiness"}</script><script type="application/ld+json">{"name":"The Irreproducibility of Bugs in Large-Scale Production Systems \u2014 Susan Fowler","url":"https://www.susanjfowler.com/blog/2016/11/2/the-irreproducibility-of-bugs-in-large-scale-production-systems","datePublished":"2016-11-03T00:52:52-0700","dateModified":"2016-11-08T00:28:11-0800","headline":"The Irreproducibility of Bugs in Large-Scale Production Systems ","author":"Susan Rigetti","publisher":{"name":"Susan Fowler","logo":{"@type":"ImageObject"},"@context":"http://schema.org","@type":"Organization"},"image":"http://static1.squarespace.com/static/57a6de391b631b6aa4789ebf/t/5dd82f25dcadfc116d1e5b99/1574448937925/91t6KiP1cEL.jpg?format=1500w","@context":"http://schema.org","@type":"Article"}</script><link rel="stylesheet" type="text/css" href="https://static1.squarespace.com/static/sitecss/57a6de391b631b6aa4789ebf/84/50521cf884aeb45fa5cfdb80/57a6de391b631b6aa4789f05/968/site.css?nocustom=true"/><script>Static.COOKIE_BANNER_CAPABLE = true;</script>
|
||
<!-- End of Squarespace Headers -->
|
||
<script defer="true" src="https://static1.squarespace.com/static/ta/5052176b84aeb45fa5cfcc83/968/scripts/site-bundle.js" type="text/javascript"></script>
|
||
</head>
|
||
|
||
<body class="show-products-category-navigation page-borders-thick canvas-style-masthead-logo-right header-subtitle-none banner-alignment-center blog-layout-center project-layout-left-sidebar thumbnails-on-open-page-show-all social-icon-style-round hide-info-footer hide-page-title hide-article-author event-thumbnails event-thumbnail-size-32-standard event-date-label event-list-show-cats event-list-date event-list-time event-list-address event-excerpts event-item-back-link product-list-titles-under product-list-alignment-center product-item-size-32-standard product-gallery-size-11-square product-gallery-auto-crop show-product-price show-product-item-nav product-social-sharing tweak-v1-related-products-image-aspect-ratio-11-square tweak-v1-related-products-details-alignment-center newsletter-style-dark opentable-style-light small-button-style-solid small-button-shape-square medium-button-style-solid medium-button-shape-square large-button-style-solid large-button-shape-square image-block-poster-text-alignment-center image-block-card-dynamic-font-sizing image-block-card-content-position-center image-block-card-text-alignment-left image-block-overlap-dynamic-font-sizing image-block-overlap-content-position-center image-block-overlap-text-alignment-left image-block-collage-dynamic-font-sizing image-block-collage-content-position-top image-block-collage-text-alignment-left image-block-stack-dynamic-font-sizing image-block-stack-text-alignment-left button-style-solid button-corner-style-square tweak-product-quick-view-button-style-floating tweak-product-quick-view-button-position-bottom tweak-product-quick-view-lightbox-excerpt-display-truncate tweak-product-quick-view-lightbox-show-arrows tweak-product-quick-view-lightbox-show-close-button tweak-product-quick-view-lightbox-controls-weight-light native-currency-code-usd view-item collection-type-blog collection-layout-default collection-57a6e2b2579fb3dbed177a34 mobile-style-available site-title" id="item-581ad9b11b631b780e8f8466">
|
||
|
||
<div id="canvas">
|
||
|
||
<div id="mobileNav" class="">
|
||
<div class="wrapper">
|
||
<nav class="main-nav mobileNav"><ul>
|
||
|
||
|
||
|
||
<li class="page-collection">
|
||
|
||
|
||
|
||
|
||
<a href="/">Home</a>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
<li class=" external-link">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<a href="https://susanrigetti.com/" target="_blank">New Website</a>
|
||
|
||
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</div>
|
||
</div>
|
||
<div id="mobileMenuLink"><a>Menu</a></div>
|
||
|
||
<header id="header" class="clear">
|
||
|
||
|
||
|
||
<div id="upper-logo">
|
||
<h1 class="logo" data-content-field="site-title">
|
||
<a href="/">Susan Fowler</a>
|
||
</h1>
|
||
</div>
|
||
<script type="module">
|
||
if (parseInt(Y.one('body').getComputedStyle('width'),10) <= 640) {
|
||
Y.use('squarespace-ui-base', function(Y) {
|
||
Y.one("#upper-logo .logo").plug(Y.Squarespace.TextShrink, {
|
||
parentEl: Y.one('#upper-logo')
|
||
});
|
||
});
|
||
}
|
||
</script>
|
||
|
||
|
||
<div class="site-info" data-content-field="business-information">
|
||
<div class="site-address">Street Address</div>
|
||
<div class="site-city-state">City, State, Zip</div>
|
||
<div class="site-phone">Phone Number</div>
|
||
</div>
|
||
|
||
|
||
<div class="site-tag-line">
|
||
<span>Writer. Editor. Fledgling Physicist. Amateur Myrmecologist. </span>
|
||
</div>
|
||
|
||
<div class="custom-info">
|
||
<div class="sqs-layout sqs-grid-12 columns-12" data-layout-label="Header Subtitle: Custom Content" data-type="block-field" data-updated-on="1381761023501" id="customInfoBlock"><div class="row sqs-row"><div class="col sqs-col-12 span-12"><div class="sqs-block html-block sqs-block-html" data-block-type="2" data-sqsp-block="text" id="block-81eee4bbda3d9efd0834"><div class="sqs-block-content">
|
||
|
||
<div class="sqs-html-content" data-sqsp-text-block-content>
|
||
<p class="text-align-center">Your Custom Text Here</p>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</div></div></div></div></div>
|
||
</div>
|
||
|
||
<div id="lower-logo">
|
||
<h1 class="logo" data-content-field="site-title"><a href="/">Susan Fowler</a></h1>
|
||
</div>
|
||
<script type="module">
|
||
Y.use('squarespace-ui-base', function(Y) {
|
||
Y.one("#lower-logo .logo").plug(Y.Squarespace.TextShrink, {
|
||
parentEl: Y.one('#lower-logo')
|
||
});
|
||
});
|
||
</script>
|
||
|
||
|
||
<div id="topNav">
|
||
<nav class="main-nav" data-content-field="navigation">
|
||
<ul>
|
||
|
||
|
||
|
||
<li class="page-collection">
|
||
|
||
|
||
|
||
|
||
<a href="/">Home</a>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
<li class=" external-link">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<a href="https://susanrigetti.com/" target="_blank">New Website</a>
|
||
|
||
|
||
|
||
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
<div class="page-divider"></div>
|
||
</nav>
|
||
</div>
|
||
|
||
|
||
</header>
|
||
|
||
<div class="page-divider top-divider"></div>
|
||
|
||
<!-- // page image or divider -->
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<section id="page" class="clear" role="main" data-content-field="main-content" data-collection-id="57a6e2b2579fb3dbed177a34" data-collection-id="57a6e2b2579fb3dbed177a34" data-edit-main-image="Banner" >
|
||
|
||
<!-- // CATEGORY NAV -->
|
||
|
||
|
||
<div class="article-wrapper">
|
||
|
||
|
||
|
||
<article class="hentry category-software-engineering tag-software-engineering tag-distributed-systems tag-sre tag-devops author-susan-rigetti post-type-text" id="article-581ad9b11b631b780e8f8466" data-item-id="581ad9b11b631b780e8f8466">
|
||
|
||
<!--POST HEADER-->
|
||
|
||
<header>
|
||
<h1 class="entry-title" itemprop="headline" data-content-field="title">
|
||
|
||
<a href="/blog/2016/11/2/the-irreproducibility-of-bugs-in-large-scale-production-systems">The Irreproducibility of Bugs in Large-Scale Production Systems </a>
|
||
|
||
</h1>
|
||
<div class="meta">
|
||
<span class="date"><time class="published dt-published" datetime="2016-11-03" itemprop="datePublished" pubdate>November 3, 2016</time></span>
|
||
<span class="author"><a href="/blog?author=57a6de37725e25e92a2d1b4c" rel="author">Susan Rigetti</a></span>
|
||
</div>
|
||
</header>
|
||
|
||
<!--SPECIAL CONTENT-->
|
||
|
||
|
||
|
||
<!--POST BODY-->
|
||
|
||
<div class="body entry-content"><div class="sqs-layout sqs-grid-12 columns-12" data-layout-label="Post Body" data-type="item" data-updated-on="1478158542201" id="item-581ad9b11b631b780e8f8466"><div class="row sqs-row"><div class="col sqs-col-12 span-12"><div class="sqs-block html-block sqs-block-html" data-block-type="2" data-sqsp-block="text" id="block-906ecea93d571b7b8c93"><div class="sqs-block-content">
|
||
|
||
<div class="sqs-html-content" data-sqsp-text-block-content>
|
||
<p>Before working at Uber, I'd only had experience with very small and simple production systems. If something went wrong with these systems, if I was paged because something was broken in production, I could easily track down the problem, reproduce the bug(s) with minimal effort, and then test and deploy a fix within a few minutes. When I joined Uber one year ago and began working on various parts of its then-1000-microservice ecosystem (Uber is probably around 1500 microservices now), one of the biggest surprises was realizing that the majority of bugs were irreproducible. In fact, I think one of the most important lessons I've learned this past year is that the "track down the problem -> reproduce the bug -> test and deploy the fix" cycle simply does not work in most large and complex production systems. </p><p>The first stage of the bug/incident/outage response process (tracking down the problem) is definitely difficult in large production systems, but it's not impossible. It takes good monitoring, great observability, and some really dedicated and experienced engineers, but it can be done, and most companies I know do it pretty well. Yes, things get difficult when you have large production ecosystems, and you need to deal with crazy capacity, latency, availability, and scalability issues that you wouldn't find in smaller systems or at smaller companies, but there's nothing about running large-scale production systems that makes it <em>impossible </em>to do this (in theory or in practice). The same is true for testing and deploying a fix once a fix has been identified (the last part of the cycle), which is probably the easiest part of the whole process: there's nothing inherently impossible or difficult about accomplishing this in a large system. The problem is found in the second step of the process: in large-scale production systems, the cycle breaks downs in the "reproduce the bug" phase. The cycle breaks down because it is practically impossible to reproduce bugs in large-scale production systems. </p><p>To see exactly how and why the cycle breaks down, let's do a little thought experiment. Let's suppose that we are developers working at a company with four hundred microservices (a rather small number of services). These microservices are developed by various different teams, and they all live on top of an application platform, which lives atop the infrastructure at the company. To make things easy, let's say that the application platform and the infrastructure are relatively simple. In a perfect, ideal, Platonic-form-heaven of a world, all of these microservices would be completely independent of one another, and wouldn't affect each other at all (neither negatively nor positively). The cold reality of microservice architecture is that microservices do not live in isolation, but have to depend on each other and interact, so what you end up with at the end of the day is a large set of intricately connected services and dependency chains that (when drawn on a whiteboard) eerily resembles a spiderweb. If one microservice in the chain goes down, all upstream microservices tend to go down with it. If one microservice changes something, the odds are really high that the change will affect its clients. </p><p>Now let's think about how these services are developed and deployed. Microservice architecture allows developers to deploy all of the time and to create new features really, really quickly. In the real world, this means that developers will be deploying more than once per week (sometimes once per day, and often multiple times per day). For our thought experiment, let's imagine that the microservices at our pretend company are each deployed once per day, and that each change is of nonzero significance. So now we have four hundred microservices, each being significantly changed every single day with every new deployment. In addition, the platform and infrastructure beneath the microservices is changing every day: new configurations are being deployed, monitoring is being changed, new hardware is being provisioned, resources are being allocated, libraries are changing - the list goes on forever. We can safely assume that in a large-scale production system like this, the state of every single component of the system changes every single day.</p><p> </p><h3 class="text-align-center"><em>"The crux of reproducibility when it comes to bugs is this: being able to reproduce a bug requires that the state of the system be nearly identical at the time of reproduction as it was at the time the bug originally occurred - something that is impossible to guarantee in large production systems."</em></h3><p> </p><p>In this system (just like in most large-scale production systems), the state of the system changes constantly throughout each day. There are no guarantees that the state of the system will stay the same from one moment to the next, and, given how interconnected and codependent all of the components of the system are, this fact matters very, very much when it comes to reproducing problems with the system. The crux of reproducibility when it comes to bugs is this: being able to reproduce a bug requires that the state of the system be nearly identical at the time of reproduction as it was at the time the bug originally occurred - something that is impossible to guarantee in large production systems. In most cases, a microservice or application in a large system will not be the same service or application that it was twelve hours ago, let alone several days ago, and reproducing problems <em>the same way </em>we are used to reproducing them in smaller, self-contained systems is impossible. </p><p>The solution to the irreproducibility-of-bugs problem (which I should probably call the "irreproducibility-of-state problem", now that I think about its name) comes from the way that we find ourselves working around the problem when we're trying to debug and mitigate production issues. In these situations, where we are faced with a problem and can't reproduce it on-the-fly, often the only way to determine the its root cause is to comb through the service-level or application-level logs, discover the state of the application or service at the time of the outage, and figure out why the service or application failed in that state.</p><p>If our logs don't accurately capture the state of our service at the time the problem occurred, then we're shit out of luck and our only way forward is to add some brand-new logging where we suspect there may be a problem, cross our fingers, and hope that our new logging catches it. This is obviously problematic, because it requires our system to fail at least one more time in order for us to root-cause the underlying issue. It's better to take a comprehensive and smart approach to logging from the get-go: we can design our logging such that we can determine from our logs <em>exactly </em>what went wrong and where things fell apart, and we can do this by making sure that our logging accurately captures the state of the system at the time. </p><p>I've spent a lot of time this year while writing my book <em><a target="_blank" href="http://shop.oreilly.com/product/0636920053675.do">Production-Ready Microservices</a> </em>trying to figure out a comprehensive list of things that systems should log in order to fit this requirement, and honestly I haven't been able to come up with one - the list of necessary and sufficient things that should be logged in order to adequately describe the state of a system is far too dependent on the specifics of any given system. All I have been able to do is come up with a general rule of thumb: design your logging such that you can determine the state of the system at the time an event is logged. </p><p> </p><h3 class="text-align-center"><em>"a general rule of thumb: design your logging such that you can determine the state of the system at the time an event is logged." </em></h3><p> </p><p>Before I end this post, I want to briefly mention that I've seen several microservice-architecture-based companies attack this problem by versioning microservices and/or their endpoints, and I want to caution against it. Avoid this approach if you can, because it tends to lead to really messy and unfortunately situations in which client services pin to specific (old, outdated) versions of microservices. Instead, make your logging decent enough that you can record the time adequately, which will allow you to correlate bugs and other problems with the state of the system at that time in the past. </p><p>Good luck out there, and happy logging!</p><p><em>P.S. I wrote a whole book full of stuff like this post about building good, production-ready microservices called </em><a target="_blank" href="http://shop.oreilly.com/product/0636920053675.do">Production-Ready Microservices</a>. <em>If you'd like a discount code for the book, <a href="mailto:susan@susanjfowler.com">send me an email</a> - I have a few discount codes to give away. </em></p><p> </p>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</div></div></div></div></div></div>
|
||
|
||
|
||
|
||
<!--POST FOOTER-->
|
||
|
||
<footer class="clear">
|
||
<div class="meta">
|
||
|
||
|
||
|
||
<div class="first meta-row">
|
||
<span class="categories"><span class="categories-title">In</span> <a href="/blog/category/Software+Engineering" rel="tag">Software Engineering</a></span>
|
||
<span class="tags"><span class="tags-title">Tags</span> <a href="/blog/tag/software+engineering" rel="tag">software engineering</a>, <a href="/blog/tag/distributed+systems" rel="tag">distributed systems</a>, <a href="/blog/tag/SRE" rel="tag">SRE</a>, <a href="/blog/tag/devops" rel="tag">devops</a></span>
|
||
</div>
|
||
|
||
<div class="second meta-row">
|
||
<span class="squarespace-social-buttons inline-style" data-system-data-id="" data-asset-url="https://static1.squarespace.com/static/57a6de391b631b6aa4789ebf/57a6e2b2579fb3dbed177a34/581ad9b11b631b780e8f8466/1478593691007/" data-record-type="1" data-full-url="/blog/2016/11/2/the-irreproducibility-of-bugs-in-large-scale-production-systems" data-title="The Irreproducibility of Bugs in Large-Scale Production Systems "></span>
|
||
|
||
<span class="sqs-simple-like" data-item-id="581ad9b11b631b780e8f8466" data-like-count="13">
|
||
<span class="like-icon"></span>
|
||
<span class="like-count"></span>
|
||
</span>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
</article>
|
||
|
||
|
||
|
||
<!--PAGINATION-->
|
||
|
||
|
||
<nav class="pagination clear">
|
||
<a class="prev-item" href="/blog/2016/12/12/the-52-books-of-2016">← The 52 Books of 2016</a><a class="next-item" href="/blog/2016/10/13/the-ops-identity-crisis">The Ops Identity Crisis →</a>
|
||
</nav>
|
||
|
||
|
||
|
||
<!-- COMMENTS -->
|
||
|
||
<section id="comments-581ad9b11b631b780e8f8466" class="comments-wrapper">
|
||
|
||
<div class="squarespace-comments" id="comments-581ad9b11b631b780e8f8466" data-item-id="581ad9b11b631b780e8f8466" data-public-comment-count="3" data-comment-state="1"></div>
|
||
|
||
</section>
|
||
|
||
|
||
</div><!-- /article-wrapper -->
|
||
|
||
<aside id="sidebar"><div class="sqs-layout sqs-grid-1 columns-1 empty" data-layout-label="Sidebar Content" data-type="block-field" id="sidebarBlocks"><div class="row sqs-row"><div class="col sqs-col-1 span-1"></div></div></div></aside>
|
||
|
||
</section>
|
||
|
||
<div class="sqs-layout sqs-grid-12 columns-12 empty" data-layout-label="Blog Footer Content" data-type="block-field" id="item-581ad9b11b631b780e8f8466"><div class="row sqs-row"><div class="col sqs-col-12 span-12"></div></div></div>
|
||
|
||
<!-- <div class="page-divider bottom-divider"></div> -->
|
||
|
||
<div class="info-footer-wrapper clear">
|
||
<div class="info-footer">
|
||
<div class="sqs-layout sqs-grid-12 columns-12" data-layout-label="Info Footer Content" data-type="block-field" data-updated-on="1470554197839" id="infoFooterBlock"><div class="row sqs-row"><div class="col sqs-col-12 span-12"><div class="sqs-block html-block sqs-block-html" data-block-type="2" data-sqsp-block="text" id="block-yui_3_17_2_1_1411670010341_19727"><div class="sqs-block-content">
|
||
|
||
<div class="sqs-html-content" data-sqsp-text-block-content>
|
||
<p>Text here</p>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</div></div></div></div></div>
|
||
|
||
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<footer id="footer" class="clear">
|
||
<div class="sqs-layout sqs-grid-12 columns-12" data-layout-label="Footer Content" data-type="block-field" data-updated-on="1767710408396" id="footerBlock"><div class="row sqs-row"><div class="col sqs-col-12 span-12"><div class="sqs-block html-block sqs-block-html" data-block-type="2" data-border-radii="{"topLeft":{"unit":"px","value":0.0},"topRight":{"unit":"px","value":0.0},"bottomLeft":{"unit":"px","value":0.0},"bottomRight":{"unit":"px","value":0.0}}" data-sqsp-block="text" id="block-yui_3_17_2_1_1619027894026_2085"><div class="sqs-block-content">
|
||
|
||
<div class="sqs-html-content" data-sqsp-text-block-content>
|
||
<p class="" style="white-space:pre-wrap;"><em>Please note: Some links on this site may contain Amazon or Bookshop referral codes that generate small commissions — usually around a few cents — if you buy the books. These commissions help me cover hosting expenses for susanrigetti.com and susanjfowler.com.</em></p>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</div></div><div class="sqs-block html-block sqs-block-html" data-block-type="2" data-border-radii="{"topLeft":{"unit":"px","value":0.0},"topRight":{"unit":"px","value":0.0},"bottomLeft":{"unit":"px","value":0.0},"bottomRight":{"unit":"px","value":0.0}}" data-sqsp-block="text" id="block-yui_3_17_2_1_1411672901316_31172"><div class="sqs-block-content">
|
||
|
||
<div class="sqs-html-content" data-sqsp-text-block-content>
|
||
<p style="text-align:center;white-space:pre-wrap;" class="">COPYRIGHT 2016-2026 Susan Fowler</p>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</div></div></div></div></div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
<div></div>
|
||
|
||
|
||
|
||
|
||
|
||
</body>
|
||
|
||
</html>
|