Files
nexus/sreweekly/articles/217/05-the-tail-at-scale.html
2026-09-12 17:23:01 +08:00

463 lines
75 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html dir="ltr" lang="en-CA" class="no-js">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="profile" href="https://gmpg.org/xfn/11">
<script>
(function(html){html.className = html.className.replace(/\bno-js\b/,'js')})(document.documentElement);
//# sourceURL=twentysixteen_javascript_detection
</script>
<title>The Tail at Scale | bill duncan's blog</title>
<!-- All in One SEO 5.0.1.1 - aioseo.com -->
<meta name="description" content="Some of the math behind nines used to establish back end service level objectives (SLO)." />
<meta name="robots" content="max-image-preview:large" />
<meta name="author" content="bduncan"/>
<meta name="keywords" content="slo,service level objectives,backend,long tail,tail at scale" />
<link rel="canonical" href="https://billduncan.org/the-tail-at-scale/" />
<meta name="generator" content="All in One SEO (AIOSEO) 5.0.1.1" />
<script type="application/ld+json" class="aioseo-schema">
{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/billduncan.org\/the-tail-at-scale\/#article","name":"The Tail at Scale | bill duncan's blog","headline":"The Tail at Scale","author":{"@id":"https:\/\/billduncan.org\/author\/ve3ied\/#author"},"publisher":{"@id":"https:\/\/billduncan.org\/#organization"},"image":{"@type":"ImageObject","url":"https:\/\/billduncan.org\/wp-content\/uploads\/2020\/04\/VennDiagram_900.png","@id":"https:\/\/billduncan.org\/the-tail-at-scale\/#articleImage","width":499,"height":217,"caption":"System on left depends on two services, on right we have 3 services."},"datePublished":"2020-04-28T20:07:13-04:00","dateModified":"2023-03-12T20:04:14-04:00","inLanguage":"en-CA","commentCount":3,"mainEntityOfPage":{"@id":"https:\/\/billduncan.org\/the-tail-at-scale\/#webpage"},"isPartOf":{"@id":"https:\/\/billduncan.org\/the-tail-at-scale\/#webpage"},"articleSection":"devops, SRE, System Performance"},{"@type":"BreadcrumbList","@id":"https:\/\/billduncan.org\/the-tail-at-scale\/#breadcrumblist","itemListElement":[{"@type":"ListItem","@id":"https:\/\/billduncan.org#listItem","position":1,"name":"Home","item":"https:\/\/billduncan.org","nextItem":{"@type":"ListItem","@id":"https:\/\/billduncan.org\/category\/system-performance\/#listItem","name":"System Performance"}},{"@type":"ListItem","@id":"https:\/\/billduncan.org\/category\/system-performance\/#listItem","position":2,"name":"System Performance","item":"https:\/\/billduncan.org\/category\/system-performance\/","nextItem":{"@type":"ListItem","@id":"https:\/\/billduncan.org\/the-tail-at-scale\/#listItem","name":"The Tail at Scale"},"previousItem":{"@type":"ListItem","@id":"https:\/\/billduncan.org#listItem","name":"Home"}},{"@type":"ListItem","@id":"https:\/\/billduncan.org\/the-tail-at-scale\/#listItem","position":3,"name":"The Tail at Scale","previousItem":{"@type":"ListItem","@id":"https:\/\/billduncan.org\/category\/system-performance\/#listItem","name":"System Performance"}}]},{"@type":"Organization","@id":"https:\/\/billduncan.org\/#organization","name":"bill duncan's blog","description":"Performance Matters (and Other Tidbits from the Trenches)","url":"https:\/\/billduncan.org\/"},{"@type":"Person","@id":"https:\/\/billduncan.org\/author\/ve3ied\/#author","url":"https:\/\/billduncan.org\/author\/ve3ied\/","name":"bduncan","image":{"@type":"ImageObject","@id":"https:\/\/billduncan.org\/the-tail-at-scale\/#authorImage","url":"https:\/\/secure.gravatar.com\/avatar\/9ce8613e8528c4c7ee8688ca76ed48b04e3bca24bc5aab38416266ea89876147?s=96&d=mm&r=pg","width":96,"height":96,"caption":"bduncan"}},{"@type":"WebPage","@id":"https:\/\/billduncan.org\/the-tail-at-scale\/#webpage","url":"https:\/\/billduncan.org\/the-tail-at-scale\/","name":"The Tail at Scale | bill duncan's blog","description":"Some of the math behind nines used to establish back end service level objectives (SLO).","inLanguage":"en-CA","isPartOf":{"@id":"https:\/\/billduncan.org\/#website"},"breadcrumb":{"@id":"https:\/\/billduncan.org\/the-tail-at-scale\/#breadcrumblist"},"author":{"@id":"https:\/\/billduncan.org\/author\/ve3ied\/#author"},"creator":{"@id":"https:\/\/billduncan.org\/author\/ve3ied\/#author"},"datePublished":"2020-04-28T20:07:13-04:00","dateModified":"2023-03-12T20:04:14-04:00"},{"@type":"WebSite","@id":"https:\/\/billduncan.org\/#website","url":"https:\/\/billduncan.org\/","name":"bill duncan's blog","description":"Performance Matters (and Other Tidbits from the Trenches)","inLanguage":"en-CA","publisher":{"@id":"https:\/\/billduncan.org\/#organization"}}]}
</script>
<!-- All in One SEO -->
<link rel='dns-prefetch' href='//cdn.trustedsite.com' />
<link rel='dns-prefetch' href='//platform-api.sharethis.com' />
<link rel="alternate" type="application/rss+xml" title="bill duncan&#039;s blog &raquo; Feed" href="https://billduncan.org/feed/" />
<link rel="alternate" type="application/rss+xml" title="bill duncan&#039;s blog &raquo; Comments Feed" href="https://billduncan.org/comments/feed/" />
<link rel="alternate" type="application/rss+xml" title="bill duncan&#039;s blog &raquo; The Tail at Scale Comments Feed" href="https://billduncan.org/the-tail-at-scale/feed/" />
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://billduncan.org/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fbillduncan.org%2Fthe-tail-at-scale%2F" />
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://billduncan.org/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fbillduncan.org%2Fthe-tail-at-scale%2F&#038;format=xml" />
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<link rel='stylesheet' id='wp-quicklatex-format-css' href='https://billduncan.org/wp-content/plugins/wp-quicklatex/css/quicklatex-format.css?ver=7.0.4' media='all' />
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style*=border-color]){border-style:solid}html :where([style*=border-top-color]){border-top-style:solid}html :where([style*=border-right-color]){border-right-style:solid}html :where([style*=border-bottom-color]){border-bottom-style:solid}html :where([style*=border-left-color]){border-left-style:solid}html :where([style*=border-width]){border-style:solid}html :where([style*=border-top-width]){border-top-style:solid}html :where([style*=border-right-width]){border-right-style:solid}html :where([style*=border-bottom-width]){border-bottom-style:solid}html :where([style*=border-left-width]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
</style>
<style id="classic-theme-styles-inline-css">
/*! This file is auto-generated */
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
/*# sourceURL=/wp-includes/css/classic-themes.min.css */
</style>
<link rel='stylesheet' id='font-awesome-css' href='https://billduncan.org/wp-content/plugins/contact-widgets/assets/css/font-awesome.min.css?ver=4.7.0' media='all' />
<style id="site-designer-shared-pattern-classes-inline-css">
/* Dark cover/section overlay + on-dark text. */
body .wp-site-blocks .is-style-overlay-dark .wp-block-cover__background { background-color: var(--wp--preset--color--base-3); color: var(--wp--preset--color--contrast-3); }
.wp-block-cover.is-style-overlay-dark .wp-block-cover__background.has-background-dim { opacity: 0.50 !important; }
:is(.wp-block-designsetgo-section, .wp-block-designsetgo-scroll-slides).is-style-overlay-dark { --dsgo-overlay-color: var(--wp--preset--color--base-3); --dsgo-overlay-opacity: 0.50; color: var(--wp--preset--color--contrast-3); }
.dsgo-image-accordion-item.is-style-overlay-dark { --dsgo-overlay-color: var(--wp--preset--color--base-3); }
.wp-block-group.has-background.is-style-overlay-dark { box-shadow: inset 0 0 0 9999px color-mix(in srgb, var(--wp--preset--color--base-3) 50%, transparent); }
body .wp-site-blocks .is-style-overlay-dark, body .wp-site-blocks .is-style-on-dark { --dsgo-text-color: var(--wp--preset--color--contrast-3); color: var(--wp--preset--color--contrast-3) !important; }
body .wp-site-blocks .is-style-overlay-dark :is(h1,h2,h3,h4,h5,h6,p,li,blockquote,cite), body .wp-site-blocks .is-style-on-dark :is(h1,h2,h3,h4,h5,h6,p,li,blockquote,cite) { color: var(--wp--preset--color--contrast-3) !important; }
/* Solid dark section background + light text. */
.is-style-bg-dark { background-color: var(--wp--preset--color--contrast); color: var(--wp--preset--color--base); }
.is-style-bg-dark a { color: var(--wp--preset--color--base); }
/*# sourceURL=site-designer-shared-pattern-classes-inline-css */
</style>
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #fff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--color--dark-gray: #1a1a1a;--wp--preset--color--medium-gray: #686868;--wp--preset--color--light-gray: #e5e5e5;--wp--preset--color--blue-gray: #4d545c;--wp--preset--color--bright-blue: #007acc;--wp--preset--color--light-blue: #9adffd;--wp--preset--color--dark-brown: #402b30;--wp--preset--color--medium-brown: #774e24;--wp--preset--color--dark-red: #640c1f;--wp--preset--color--bright-red: #ff675f;--wp--preset--color--yellow: #ffef8e;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-dark-gray-color{color: var(--wp--preset--color--dark-gray) !important;}.has-medium-gray-color{color: var(--wp--preset--color--medium-gray) !important;}.has-light-gray-color{color: var(--wp--preset--color--light-gray) !important;}.has-blue-gray-color{color: var(--wp--preset--color--blue-gray) !important;}.has-bright-blue-color{color: var(--wp--preset--color--bright-blue) !important;}.has-light-blue-color{color: var(--wp--preset--color--light-blue) !important;}.has-dark-brown-color{color: var(--wp--preset--color--dark-brown) !important;}.has-medium-brown-color{color: var(--wp--preset--color--medium-brown) !important;}.has-dark-red-color{color: var(--wp--preset--color--dark-red) !important;}.has-bright-red-color{color: var(--wp--preset--color--bright-red) !important;}.has-yellow-color{color: var(--wp--preset--color--yellow) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-dark-gray-background-color{background-color: var(--wp--preset--color--dark-gray) !important;}.has-medium-gray-background-color{background-color: var(--wp--preset--color--medium-gray) !important;}.has-light-gray-background-color{background-color: var(--wp--preset--color--light-gray) !important;}.has-blue-gray-background-color{background-color: var(--wp--preset--color--blue-gray) !important;}.has-bright-blue-background-color{background-color: var(--wp--preset--color--bright-blue) !important;}.has-light-blue-background-color{background-color: var(--wp--preset--color--light-blue) !important;}.has-dark-brown-background-color{background-color: var(--wp--preset--color--dark-brown) !important;}.has-medium-brown-background-color{background-color: var(--wp--preset--color--medium-brown) !important;}.has-dark-red-background-color{background-color: var(--wp--preset--color--dark-red) !important;}.has-bright-red-background-color{background-color: var(--wp--preset--color--bright-red) !important;}.has-yellow-background-color{background-color: var(--wp--preset--color--yellow) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-dark-gray-border-color{border-color: var(--wp--preset--color--dark-gray) !important;}.has-medium-gray-border-color{border-color: var(--wp--preset--color--medium-gray) !important;}.has-light-gray-border-color{border-color: var(--wp--preset--color--light-gray) !important;}.has-blue-gray-border-color{border-color: var(--wp--preset--color--blue-gray) !important;}.has-bright-blue-border-color{border-color: var(--wp--preset--color--bright-blue) !important;}.has-light-blue-border-color{border-color: var(--wp--preset--color--light-blue) !important;}.has-dark-brown-border-color{border-color: var(--wp--preset--color--dark-brown) !important;}.has-medium-brown-border-color{border-color: var(--wp--preset--color--medium-brown) !important;}.has-dark-red-border-color{border-color: var(--wp--preset--color--dark-red) !important;}.has-bright-red-border-color{border-color: var(--wp--preset--color--bright-red) !important;}.has-yellow-border-color{border-color: var(--wp--preset--color--yellow) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
/*# sourceURL=global-styles-inline-css */
</style>
<link rel='stylesheet' id='mc4wp-form-themes-css' href='https://billduncan.org/wp-content/plugins/mailchimp-for-wp/assets/css/form-themes.css?ver=4.14.1' media='all' />
<link rel='stylesheet' id='wp-components-css' href='https://billduncan.org/wp-includes/css/dist/components/style.min.css?ver=7.0.4' media='all' />
<link rel='stylesheet' id='godaddy-styles-css' href='https://billduncan.org/wp-content/mu-plugins/vendor/wpex/godaddy-launch/includes/Dependencies/GoDaddy/Styles/build/latest.css?ver=2.0.2' media='all' />
<link rel='stylesheet' id='twentysixteen-fonts-css' href='https://billduncan.org/wp-content/themes/twentysixteen/fonts/merriweather-plus-montserrat-plus-inconsolata.css?ver=20230328' media='all' />
<link rel='stylesheet' id='genericons-css' href='https://billduncan.org/wp-content/themes/twentysixteen/genericons/genericons.css?ver=20251101' media='all' />
<link rel='stylesheet' id='twentysixteen-style-css' href='https://billduncan.org/wp-content/themes/twentysixteen/style.css?ver=20260819' media='all' />
<link rel='stylesheet' id='twentysixteen-block-style-css' href='https://billduncan.org/wp-content/themes/twentysixteen/css/blocks.css?ver=20260105' media='all' />
<script id="jquery-core-js" src="https://billduncan.org/wp-includes/js/jquery/jquery.min.js?ver=3.7.1"></script>
<script id="jquery-migrate-js" src="https://billduncan.org/wp-includes/js/jquery/jquery-migrate.min.js?ver=3.4.1"></script>
<script id="twentysixteen-script-js-extra">
var screenReaderText = {"expand":"expand child menu","collapse":"collapse child menu"};
//# sourceURL=twentysixteen-script-js-extra
</script>
<script data-wp-strategy="defer" defer id="twentysixteen-script-js" src="https://billduncan.org/wp-content/themes/twentysixteen/js/functions.js?ver=20230629"></script>
<script id="googleanalytics-platform-sharethis-js" src="//platform-api.sharethis.com/js/sharethis.js#source=googleanalytics-wordpress#product=ga&#038;property=5b1f3e1fe9c37c00114cb833"></script>
<script id="wp-quicklatex-frontend-js" src="https://billduncan.org/wp-content/plugins/wp-quicklatex/js/wp-quicklatex-frontend.js?ver=1.0"></script>
<link rel="https://api.w.org/" href="https://billduncan.org/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://billduncan.org/wp-json/wp/v2/posts/850" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://billduncan.org/xmlrpc.php?rsd" />
<meta name="generator" content="WordPress 7.0.4" />
<link rel='shortlink' href='https://billduncan.org/?p=850' />
<script>
(function() {
(function (i, s, o, g, r, a, m) {
i['GoogleAnalyticsObject'] = r;
i[r] = i[r] || function () {
(i[r].q = i[r].q || []).push(arguments)
}, i[r].l = 1 * new Date();
a = s.createElement(o),
m = s.getElementsByTagName(o)[0];
a.async = 1;
a.src = g;
m.parentNode.insertBefore(a, m)
})(window, document, 'script', 'https://google-analytics.com/analytics.js', 'ga');
ga('create', 'UA-120366658-1', 'auto');
ga('send', 'pageview');
})();
</script>
<style id="wp-site-designer-contrast-fallback"> .wp-block-group:not([data-dsgo-inline-bg]),.wp-block-column:not([data-dsgo-inline-bg]),.wp-block-columns:not([data-dsgo-inline-bg]){--dsgo-text-color:initial;} :is(.wp-block-designsetgo-section,.wp-block-group,.wp-block-column).has-background:not([class*="-background-color"]):not(.has-text-color):not(.is-style-footer-section):not(.is-style-header-section),.wp-block-cover:not(.has-text-color){color:var(--wp--preset--color--contrast-3) !important;--dsgo-text-color:var(--wp--preset--color--contrast-3);} body .wp-site-blocks :is(:is(.wp-block-designsetgo-section,.wp-block-group,.wp-block-column).has-background:not([class*="-background-color"]):not(.has-text-color):not(.is-style-footer-section):not(.is-style-header-section),.wp-block-cover) h1:not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(:is(.wp-block-designsetgo-section,.wp-block-group,.wp-block-column).has-background:not([class*="-background-color"]):not(.has-text-color):not(.is-style-footer-section):not(.is-style-header-section),.wp-block-cover) h2:not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(:is(.wp-block-designsetgo-section,.wp-block-group,.wp-block-column).has-background:not([class*="-background-color"]):not(.has-text-color):not(.is-style-footer-section):not(.is-style-header-section),.wp-block-cover) h3:not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(:is(.wp-block-designsetgo-section,.wp-block-group,.wp-block-column).has-background:not([class*="-background-color"]):not(.has-text-color):not(.is-style-footer-section):not(.is-style-header-section),.wp-block-cover) h4:not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(:is(.wp-block-designsetgo-section,.wp-block-group,.wp-block-column).has-background:not([class*="-background-color"]):not(.has-text-color):not(.is-style-footer-section):not(.is-style-header-section),.wp-block-cover) h5:not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(:is(.wp-block-designsetgo-section,.wp-block-group,.wp-block-column).has-background:not([class*="-background-color"]):not(.has-text-color):not(.is-style-footer-section):not(.is-style-header-section),.wp-block-cover) h6:not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(:is(.wp-block-designsetgo-section,.wp-block-group,.wp-block-column).has-background:not([class*="-background-color"]):not(.has-text-color):not(.is-style-footer-section):not(.is-style-header-section),.wp-block-cover) p:not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(:is(.wp-block-designsetgo-section,.wp-block-group,.wp-block-column).has-background:not([class*="-background-color"]):not(.has-text-color):not(.is-style-footer-section):not(.is-style-header-section),.wp-block-cover) a:where(:not(.wp-element-button):not(.wp-block-social-link-anchor)):not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(:is(.wp-block-designsetgo-section,.wp-block-group,.wp-block-column).has-background:not([class*="-background-color"]):not(.has-text-color):not(.is-style-footer-section):not(.is-style-header-section),.wp-block-cover) .wp-element-caption:not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)){color:var(--dsgo-text-color,inherit) !important;} body .wp-site-blocks :is(.wp-block-designsetgo-section,.wp-block-group,.wp-block-column)[data-dsgo-inline-bg] .wp-block-button:is([class*="is-style-outline"],[class*="is-style-secondary"]) .wp-element-button{color:var(--dsgo-text-color,inherit) !important;border-color:currentColor !important;}body .wp-site-blocks :is(.wp-block-designsetgo-section,.wp-block-group,.wp-block-column)[data-dsgo-inline-bg] .wp-block-button:is([class*="is-style-outline"],[class*="is-style-secondary"]) .wp-element-button:hover{opacity:0.75;} body .wp-site-blocks [data-dsgo-inline-bg] h1:not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks [data-dsgo-inline-bg] h2:not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks [data-dsgo-inline-bg] h3:not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks [data-dsgo-inline-bg] h4:not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks [data-dsgo-inline-bg] h5:not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks [data-dsgo-inline-bg] h6:not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks [data-dsgo-inline-bg] p:not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks [data-dsgo-inline-bg] a:where(:not(.wp-element-button):not(.wp-block-social-link-anchor)):not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks [data-dsgo-inline-bg] .wp-element-caption:not(.has-text-color):where(:not(.elementor *):not([data-elementor-type] *)){color:var(--dsgo-text-color,inherit) !important;} .wp-block-designsetgo-section.has-background:not([data-dsgo-inline-bg]):not(.has-text-color){color:var(--wp--preset--color--contrast);--dsgo-text-color:var(--wp--preset--color--contrast);} :is(.wp-block-group,.wp-block-column).is-style-section-1:not(:is(.has-white-background-color,.has-black-background-color)),.has-background:is(.wp-block-group,.wp-block-column).is-style-section-1:not(:is(.has-white-background-color,.has-black-background-color)),.has-text-color:is(.wp-block-group,.wp-block-column).is-style-section-1:not(:is(.has-white-background-color,.has-black-background-color)){background-color:var(--wp--preset--color--accent-5);color:var(--wp--preset--color--contrast);} body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-1:not(:is(.has-white-background-color,.has-black-background-color)) h1:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-1:not(:is(.has-white-background-color,.has-black-background-color)) h2:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-1:not(:is(.has-white-background-color,.has-black-background-color)) h3:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-1:not(:is(.has-white-background-color,.has-black-background-color)) h4:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-1:not(:is(.has-white-background-color,.has-black-background-color)) h5:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-1:not(:is(.has-white-background-color,.has-black-background-color)) h6:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-1:not(:is(.has-white-background-color,.has-black-background-color)) p:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-1:not(:is(.has-white-background-color,.has-black-background-color)) a:where(:not(.wp-element-button):not(.wp-block-social-link-anchor)):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-1:not(:is(.has-white-background-color,.has-black-background-color)) .wp-element-caption:where(:not(.elementor *):not([data-elementor-type] *)){color:var(--wp--preset--color--contrast);} :is(.wp-block-group,.wp-block-column).is-style-section-2:not(:is(.has-white-background-color,.has-black-background-color)),.has-background:is(.wp-block-group,.wp-block-column).is-style-section-2:not(:is(.has-white-background-color,.has-black-background-color)),.has-text-color:is(.wp-block-group,.wp-block-column).is-style-section-2:not(:is(.has-white-background-color,.has-black-background-color)){background-color:var(--wp--preset--color--accent-2);color:var(--wp--preset--color--contrast);} body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-2:not(:is(.has-white-background-color,.has-black-background-color)) h1:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-2:not(:is(.has-white-background-color,.has-black-background-color)) h2:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-2:not(:is(.has-white-background-color,.has-black-background-color)) h3:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-2:not(:is(.has-white-background-color,.has-black-background-color)) h4:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-2:not(:is(.has-white-background-color,.has-black-background-color)) h5:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-2:not(:is(.has-white-background-color,.has-black-background-color)) h6:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-2:not(:is(.has-white-background-color,.has-black-background-color)) p:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-2:not(:is(.has-white-background-color,.has-black-background-color)) a:where(:not(.wp-element-button):not(.wp-block-social-link-anchor)):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-2:not(:is(.has-white-background-color,.has-black-background-color)) .wp-element-caption:where(:not(.elementor *):not([data-elementor-type] *)){color:var(--wp--preset--color--contrast);} :is(.wp-block-group,.wp-block-column).is-style-section-3:not(:is(.has-white-background-color,.has-black-background-color)),.has-background:is(.wp-block-group,.wp-block-column).is-style-section-3:not(:is(.has-white-background-color,.has-black-background-color)),.has-text-color:is(.wp-block-group,.wp-block-column).is-style-section-3:not(:is(.has-white-background-color,.has-black-background-color)){background-color:var(--wp--preset--color--accent-1);color:var(--wp--preset--color--contrast);} body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-3:not(:is(.has-white-background-color,.has-black-background-color)) h1:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-3:not(:is(.has-white-background-color,.has-black-background-color)) h2:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-3:not(:is(.has-white-background-color,.has-black-background-color)) h3:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-3:not(:is(.has-white-background-color,.has-black-background-color)) h4:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-3:not(:is(.has-white-background-color,.has-black-background-color)) h5:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-3:not(:is(.has-white-background-color,.has-black-background-color)) h6:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-3:not(:is(.has-white-background-color,.has-black-background-color)) p:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-3:not(:is(.has-white-background-color,.has-black-background-color)) a:where(:not(.wp-element-button):not(.wp-block-social-link-anchor)):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-3:not(:is(.has-white-background-color,.has-black-background-color)) .wp-element-caption:where(:not(.elementor *):not([data-elementor-type] *)){color:var(--wp--preset--color--contrast);} :is(.wp-block-group,.wp-block-column).is-style-section-4:not(:is(.has-white-background-color,.has-black-background-color)),.has-background:is(.wp-block-group,.wp-block-column).is-style-section-4:not(:is(.has-white-background-color,.has-black-background-color)),.has-text-color:is(.wp-block-group,.wp-block-column).is-style-section-4:not(:is(.has-white-background-color,.has-black-background-color)){background-color:var(--wp--preset--color--accent-3);color:var(--wp--preset--color--accent-2);} body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-4:not(:is(.has-white-background-color,.has-black-background-color)) h1:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-4:not(:is(.has-white-background-color,.has-black-background-color)) h2:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-4:not(:is(.has-white-background-color,.has-black-background-color)) h3:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-4:not(:is(.has-white-background-color,.has-black-background-color)) h4:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-4:not(:is(.has-white-background-color,.has-black-background-color)) h5:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-4:not(:is(.has-white-background-color,.has-black-background-color)) h6:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-4:not(:is(.has-white-background-color,.has-black-background-color)) p:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-4:not(:is(.has-white-background-color,.has-black-background-color)) a:where(:not(.wp-element-button):not(.wp-block-social-link-anchor)):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-4:not(:is(.has-white-background-color,.has-black-background-color)) .wp-element-caption:where(:not(.elementor *):not([data-elementor-type] *)){color:var(--wp--preset--color--accent-2);} :is(.wp-block-group,.wp-block-column).is-style-section-5:not(:is(.has-white-background-color,.has-black-background-color)),.has-background:is(.wp-block-group,.wp-block-column).is-style-section-5:not(:is(.has-white-background-color,.has-black-background-color)),.has-text-color:is(.wp-block-group,.wp-block-column).is-style-section-5:not(:is(.has-white-background-color,.has-black-background-color)){background-color:var(--wp--preset--color--contrast);color:var(--wp--preset--color--base);} body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-5:not(:is(.has-white-background-color,.has-black-background-color)) h1:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-5:not(:is(.has-white-background-color,.has-black-background-color)) h2:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-5:not(:is(.has-white-background-color,.has-black-background-color)) h3:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-5:not(:is(.has-white-background-color,.has-black-background-color)) h4:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-5:not(:is(.has-white-background-color,.has-black-background-color)) h5:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-5:not(:is(.has-white-background-color,.has-black-background-color)) h6:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-5:not(:is(.has-white-background-color,.has-black-background-color)) p:where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-5:not(:is(.has-white-background-color,.has-black-background-color)) a:where(:not(.wp-element-button):not(.wp-block-social-link-anchor)):where(:not(.elementor *):not([data-elementor-type] *)),body .wp-site-blocks :is(.wp-block-group,.wp-block-column).is-style-section-5:not(:is(.has-white-background-color,.has-black-background-color)) .wp-element-caption:where(:not(.elementor *):not([data-elementor-type] *)){color:var(--wp--preset--color--base);} body .wp-site-blocks input:not([type="submit"]):not([type="button"]):not([type="reset"]):not([type="checkbox"]):not([type="radio"]):not([type="file"]):not([type="image"]):not([type="hidden"]),body .wp-site-blocks textarea,body .wp-site-blocks select{background-color:var(--wp--preset--color--base);color:var(--wp--preset--color--contrast);border:1px solid color-mix(in srgb, var(--wp--preset--color--contrast) 30%, transparent);} .wp-site-blocks input[type="submit"]:not(.wp-element-button),.wp-site-blocks button[type="submit"]:not(.wp-element-button){background-color:var(--wp--preset--color--accent-2);color:var(--wp--preset--color--base);border-width:0;padding:12px 30px;font-family:inherit;font-size:var(--wp--preset--font-size--medium, 1rem);line-height:inherit;cursor:pointer;}</style>
<style id="site-designer-logo-constraints">
.wp-block-site-logo.is-default-size img {
width: auto;
height: auto;
max-width: 180px;
max-height: 80px;
}
.wp-block-site-logo img {
height: auto;
}
</style>
</head>
<body class="wp-singular post-template-default single single-post postid-850 single-format-standard wp-embed-responsive wp-theme-twentysixteen">
<div id="page" class="site">
<div class="site-inner">
<a class="skip-link screen-reader-text" href="#content">
Skip to content </a>
<header id="masthead" class="site-header">
<div class="site-header-main">
<div class="site-branding">
<p class="site-title"><a href="https://billduncan.org/" rel="home" >bill duncan&#039;s blog</a></p>
<p class="site-description">Performance Matters (and Other Tidbits from the Trenches)</p>
</div><!-- .site-branding -->
<button id="menu-toggle" class="menu-toggle">Menu</button>
<div id="site-header-menu" class="site-header-menu">
<nav id="site-navigation" class="main-navigation" aria-label="Primary Menu">
<div class="menu-primary-menu-container"><ul id="menu-primary-menu" class="primary-menu"><li id="menu-item-15" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-home menu-item-15"><a href="https://billduncan.org">Home</a></li>
<li id="menu-item-13" class="menu-item menu-item-type-post_type menu-item-object-page menu-item-13"><a href="https://billduncan.org/about/">About Me</a></li>
<li id="menu-item-12" class="menu-item menu-item-type-post_type menu-item-object-page menu-item-12"><a href="https://billduncan.org/contact/">Contact bill duncan</a></li>
<li id="menu-item-87" class="menu-item menu-item-type-post_type menu-item-object-page menu-item-87"><a href="https://billduncan.org/references/">Links</a></li>
</ul></div> </nav><!-- .main-navigation -->
</div><!-- .site-header-menu -->
</div><!-- .site-header-main -->
<div class="header-image">
<a href="https://billduncan.org/" rel="home" >
<img src="https://billduncan.org/wp-content/uploads/2018/10/k05d1996-cropped.jpg" width="1198" height="280" alt="bill duncan&#039;s blog" sizes="(max-width: 709px) 85vw, (max-width: 909px) 81vw, (max-width: 1362px) 88vw, 1200px" srcset="https://billduncan.org/wp-content/uploads/2018/10/k05d1996-cropped.jpg 1198w, https://billduncan.org/wp-content/uploads/2018/10/k05d1996-cropped-300x70.jpg 300w, https://billduncan.org/wp-content/uploads/2018/10/k05d1996-cropped-768x179.jpg 768w, https://billduncan.org/wp-content/uploads/2018/10/k05d1996-cropped-1024x239.jpg 1024w" decoding="async" fetchpriority="high" /> </a>
</div><!-- .header-image -->
</header><!-- .site-header -->
<div id="content" class="site-content">
<div id="primary" class="content-area">
<main id="main" class="site-main">
<article id="post-850" class="post-850 post type-post status-publish format-standard hentry category-devops category-sre category-system-performance">
<header class="entry-header">
<h1 class="entry-title">The Tail at Scale</h1> </header><!-- .entry-header -->
<div class="entry-content">
<p>The landmark &#8220;Tail at Scale&#8221;[1] article was missing some of the math. We&#8217;re diving into it a bit here to show how the math can be used in setting objectives for latency budgets in back end systems.<span id="more-850"></span></p>
<p>Note that if you can give partial results to users before all the back end responses are in, <em>great</em>! You&#8217;re well on your way to mitigating the long tail.</p>
<p>Also note that this post does not discuss setting user level SLOs or SLAs. There are many resources available that discuss this and it&#8217;s more of a business or marketing related issue. This post sheds some light on what objectives are necessary from the back end to support whatever user level objectives have already been decided.</p>
<h2>TL;DR</h2>
<p>Once you&#8217;ve determined the SLO or SLAs for the user experience, you can derive what is required from the back end to support it. Let&#8217;s say that you determine that the user requests should meet the SLO target 95% of the time (whatever the target is). If the service accesses 50 back end systems to service the requests, then they will need at least 3 nines of probability of hitting the target. If it accesses 500 systems, then 4 nines of probability of hitting the target SLO will be required.</p>
<p><code><img decoding="async" src="https://billduncan.org/wp-content/ql-cache/quicklatex.com-3507a9edc6575b66aa5b17491d5fe0b4_l3.png" class="ql-img-inline-formula quicklatex-auto-format" alt="&#83;&#76;&#79;&#32;&#61;&#32;&#101;&#32;&#94;&#32;&#123;&#40;&#108;&#111;&#103;&#95;&#101;&#40;&#83;&#76;&#65;&#41;&#32;&#47;&#32;&#78;&#41;&#125;" title="Rendered by QuickLaTeX.com" height="17" width="161" style="vertical-align: 0px;"/></code><br />
..or simply:<br />
<code><img decoding="async" src="https://billduncan.org/wp-content/ql-cache/quicklatex.com-9d013f8f5db5046e2a96fdb046aeb9d0_l3.png" class="ql-img-inline-formula quicklatex-auto-format" alt="&#83;&#76;&#79;&#32;&#61;&#32;&#83;&#76;&#65;&#32;&#94;&#32;&#123;&#40;&#49;&#32;&#47;&#32;&#78;&#41;&#125;" title="Rendered by QuickLaTeX.com" height="17" width="134" style="vertical-align: 0px;"/></code><br />
..where SLA represents the customer facing objective/agreement, N is the fanout for back end services and SLO is the objective for each of the back end services.</p>
<p>You can calculate this from the command line with &#8220;bc -l&#8221; or use your favourite calculator..</p>
<pre>scale=4
e( l(0.95) / 50 )
.9990
scale=5
e( l(0.95) / 500 )
.99990
</pre>
<p>&nbsp;</p>
<h2>Quick Probabilities Refresher</h2>
<div class="page" title="Page 52">
<div class="layoutArea">
<div class="column">
<ul>
<li><code><img loading="lazy" decoding="async" src="https://billduncan.org/wp-content/ql-cache/quicklatex.com-2123ea76a17a4cd4fb699e1350e7252b_l3.png" class="ql-img-inline-formula quicklatex-auto-format" alt="&#48;&#32;&#92;&#108;&#101;&#113;&#32;&#80;&#40;&#69;&#41;&#32;&#92;&#108;&#101;&#113;&#32;&#49;" title="Rendered by QuickLaTeX.com" height="18" width="106" style="vertical-align: -4px;"/></code>  [The probability of an event is always between 0 and 1.]</li>
<li><code><img loading="lazy" decoding="async" src="https://billduncan.org/wp-content/ql-cache/quicklatex.com-69fca08bbf24782f353b322aba645a05_l3.png" class="ql-img-inline-formula quicklatex-auto-format" alt="&#80;&#40;&#83;&#41;&#32;&#61;&#32;&#49;" title="Rendered by QuickLaTeX.com" height="18" width="71" style="vertical-align: -4px;"/></code>  [The probability of the sample space is always 1.]</li>
<li><code><img loading="lazy" decoding="async" src="https://billduncan.org/wp-content/ql-cache/quicklatex.com-c9bc8b460ac2eb0366821f4ae6e81dd0_l3.png" class="ql-img-inline-formula quicklatex-auto-format" alt="&#80;&#40;&#69;&#41;&#32;&#43;&#32;&#80;&#40;&#92;&#115;&#105;&#109;&#32;&#69;&#41;&#32;&#61;&#32;&#49;" title="Rendered by QuickLaTeX.com" height="18" width="155" style="vertical-align: -4px;"/></code> [The probability of an event and its complement is always 1.]</li>
</ul>
<figure id="attachment_888" aria-describedby="caption-attachment-888" style="width: 499px" class="wp-caption alignnone"><img loading="lazy" decoding="async" class="size-full wp-image-888" src="https://billduncan.org/wp-content/uploads/2020/04/VennDiagram_900.png" alt="Venn Diagrams of independent events which are not mutually exclusive" width="499" height="217" srcset="https://billduncan.org/wp-content/uploads/2020/04/VennDiagram_900.png 499w, https://billduncan.org/wp-content/uploads/2020/04/VennDiagram_900-300x130.png 300w" sizes="auto, (max-width: 499px) 85vw, 499px" /><figcaption id="caption-attachment-888" class="wp-caption-text">System on left depends on two services, on right we have 3 services.</figcaption></figure>
</div>
</div>
</div>
<p>As an exaggerated case, if we have two systems that fail SLO with a probability of 0.5 (they each fail half the time), then the combined failure probability will be given by the following union of events.</p>
<p><code><img loading="lazy" decoding="async" src="https://billduncan.org/wp-content/ql-cache/quicklatex.com-1ef088b8135b16b0ad4f35bf7d5e2188_l3.png" class="ql-img-inline-formula quicklatex-auto-format" alt="&#80;&#40;&#65;&#32;&#92;&#99;&#117;&#112;&#32;&#66;&#41;&#32;&#61;&#32;&#80;&#40;&#65;&#41;&#32;&#43;&#32;&#80;&#40;&#66;&#41;&#32;&#45;&#32;&#80;&#40;&#65;&#32;&#92;&#99;&#97;&#112;&#32;&#66;&#41;" title="Rendered by QuickLaTeX.com" height="18" width="299" style="vertical-align: -4px;"/></code></p>
<p>It doesn&#8217;t matter if only one fails or both fail, the overall user experience will be a fail. However, we only want to count the overlap once, so need to subtract out the area of intersection. Obviously, in this exaggerated case, (like coin tosses), the user experience would fail SLO 75% of the time.</p>
<p><code>0.75 = 0.5 + 0.5 - 0.25</code></p>
<p>Where it gets &#8220;interesting&#8221;, is if you have dozens or hundreds of services that must respond before the deadline of the SLO. It would be a combination explosion to calculate this way as we can start to see from the diagram on right with only 3 services.</p>
<p>It&#8217;s easier to find the intersection set of the inverse probabilities; the probability that they will each succeed in meeting the SLO. (This is also generally easier to glean from available data.)  Assuming that the events are independent, the probability of both succeeding is simply multiplying the success rates to get the overall success rate. In our simplified extreme example above the user would experience SLO success 25% of the time, <code>0.5^2</code>.</p>
<p><code><img loading="lazy" decoding="async" src="https://billduncan.org/wp-content/ql-cache/quicklatex.com-74ac8fd061f76303478ea17f967ea06d_l3.png" class="ql-img-inline-formula quicklatex-auto-format" alt="&#80;&#40;&#32;&#92;&#115;&#105;&#109;&#32;&#65;&#32;&#92;&#59;&#32;&#92;&#99;&#97;&#112;&#32;&#32;&#92;&#115;&#105;&#109;&#32;&#66;&#32;&#41;&#32;&#61;&#32;&#80;&#40;&#32;&#92;&#115;&#105;&#109;&#32;&#65;&#32;&#41;&#32;&#92;&#116;&#105;&#109;&#101;&#115;&#32;&#80;&#40;&#32;&#92;&#115;&#105;&#109;&#32;&#66;&#32;&#41;" title="Rendered by QuickLaTeX.com" height="18" width="279" style="vertical-align: -4px;"/></code></p>
<p>It&#8217;s the intersection set of when they both succeed. Subtract from one to find the failure rate. Much easier to calculate.</p>
<h2>Setting SLO for Backend Systems</h2>
<p>Assuming that all the back end services are equal and independent[2], we can use the same method to come up with a rough objective budget for back end services using probabilities.</p>
<p>Using a real example from my past, I&#8217;ll look at the objectives for the back end as well as a mitigation strategy for improving the customer experience.</p>
<p>The system had thousands of lucene index applications in the back end which consumed most of the SLO budget. Requests could contain hundreds of terms and hit many hundreds of these apps for the larger customers. All queries must complete before the response is sent to the user. The target SLO (as well as SLA for some customers) was 90% of the queries finishing in under 20 seconds.</p>
<p>We rarely had trouble with smaller customers as the queries would only be hitting a few dozen back end systems. Using the formulae above, 3 &#8220;nines&#8221; or 99.9% objective for the back end systems meeting their latency budget would suffice. Larger customers might fan the query out to 500+ systems. Meeting the customer SLO would require 4 &#8220;nines&#8221; or 99.99% from the back end which was always a &#8220;challenge&#8221;.</p>
<p>Improving the &#8220;average response times&#8221; wasn&#8217;t going to make much of a difference of course. 95% of the responses from the back end were available within half the SLO budget for the back end already. It was the outliers that killed us and they were unpredictable. &#8220;Noisy neighbour&#8221;, &#8220;garbage collection&#8221; and other events that would <em>always</em> be a factor and wouldn&#8217;t go away by improving average performance.</p>
<p>The paper alludes to a solution with &#8220;Hedged requests&#8221;; sending the same request to multiple replicas. Unfortunately, the simplest scenario would double the traffic and make &#8220;noisy neighbour&#8221; issues even more prevalent.</p>
<p>The paper also mentions deferring the secondary requests. This starts to make sense. Since 95% of the requests were returned in half the time budget, for 500 systems we&#8217;d generally only need to send out about 25 requests to the replicas. We&#8217;d already established that 95% of the systems responded in 10 seconds, so most of the time the remaining 25 systems would respond in the remaining 10 seconds to meet the overall SLO of 20 seconds.</p>
<h2>Summary</h2>
<p>If you haven&#8217;t already, read the Tail at Scale[1] paper and Adrian Colyer&#8217;s[3] notes on it.</p>
<p>Understand that there are always going to be variances and the best way to model the performance for a large fanout of back end systems is by using probabilities.</p>
<p><em>Probably.</em></p>
<hr />
<p>[1] <a title="The Tail at Scale" href="https://www2.cs.duke.edu/courses/cps296.4/fall13/838-CloudPapers/dean_longtail.pdf" target="_blank" rel="noopener noreferrer">The Tail at Scale</a> Dean and Barroso 2013</p>
<p>[2] Never really true actually, but close enough for estimating.</p>
<p>[3] <a title="The Tail at Scale" href="https://blog.acolyer.org/2015/01/15/the-tail-at-scale/">the morning paper</a> Adrian Colyer</p>
</div><!-- .entry-content -->
<footer class="entry-footer">
<span class="byline"><img alt='' src='https://secure.gravatar.com/avatar/9ce8613e8528c4c7ee8688ca76ed48b04e3bca24bc5aab38416266ea89876147?s=49&#038;d=mm&#038;r=pg' srcset='https://secure.gravatar.com/avatar/9ce8613e8528c4c7ee8688ca76ed48b04e3bca24bc5aab38416266ea89876147?s=98&#038;d=mm&#038;r=pg 2x' class='avatar avatar-49 photo' height='49' width='49' loading='lazy' decoding='async'/><span class="screen-reader-text">Author </span><span class="author vcard"><a class="url fn n" href="https://billduncan.org/author/ve3ied/">bduncan</a></span></span><span class="posted-on"><span class="screen-reader-text">Posted on </span><a href="https://billduncan.org/the-tail-at-scale/" rel="bookmark"><time class="entry-date published" datetime="2020-04-28T20:07:13-04:00">2020-04-28</time><time class="updated" datetime="2023-03-12T20:04:14-04:00">2023-03-12</time></a></span><span class="cat-links"><span class="screen-reader-text">Categories </span><a href="https://billduncan.org/category/devops/" rel="category tag">devops</a>, <a href="https://billduncan.org/category/sre/" rel="category tag">SRE</a>, <a href="https://billduncan.org/category/system-performance/" rel="category tag">System Performance</a></span> </footer><!-- .entry-footer -->
</article><!-- #post-850 -->
<div id="comments" class="comments-area">
<h2 class="comments-title">
3 thoughts on &ldquo;The Tail at Scale&rdquo; </h2>
<ol class="comment-list">
<li id="comment-345" class="pingback even thread-even depth-1">
<div class="comment-body">
Pingback: <a href="http://sreweekly.com/sre-weekly-issue-217/" class="url" rel="ugc external nofollow">SRE Weekly Issue #217 &#8211; SRE WEEKLY</a> </div>
</li><!-- #comment-## -->
<li id="comment-351" class="pingback odd alt thread-odd thread-alt depth-1">
<div class="comment-body">
Pingback: <a href="https://billduncan.org/the-tail-at-scale-revisited/" class="url" rel="ugc">The Tail at Scale Revisited | | bill duncan&#039;s blog</a> </div>
</li><!-- #comment-## -->
<li id="comment-362" class="pingback even thread-even depth-1">
<div class="comment-body">
Pingback: <a href="https://billduncan.org/the-tail-at-scale-approximation/" class="url" rel="ugc">The Tail at Scale Approximation | bill duncan&#039;s blog</a> </div>
</li><!-- #comment-## -->
</ol><!-- .comment-list -->
<p class="no-comments">Comments are closed.</p>
</div><!-- .comments-area -->
<nav class="navigation post-navigation" aria-label="Posts">
<h2 class="screen-reader-text">Post navigation</h2>
<div class="nav-links"><div class="nav-previous"><a href="https://billduncan.org/hiring-questions-problem-3/" rel="prev"><span class="meta-nav" aria-hidden="true">Previous</span> <span class="screen-reader-text">Previous post:</span> <span class="post-title">Hiring Questions, Problem 3</span></a></div><div class="nav-next"><a href="https://billduncan.org/the-tail-at-scale-revisited/" rel="next"><span class="meta-nav" aria-hidden="true">Next</span> <span class="screen-reader-text">Next post:</span> <span class="post-title">The Tail at Scale Revisited</span></a></div></div>
</nav>
</main><!-- .site-main -->
</div><!-- .content-area -->
<aside id="secondary" class="sidebar widget-area">
<section id="search-2" class="widget widget_search">
<form role="search" method="get" class="search-form" action="https://billduncan.org/">
<label>
<span class="screen-reader-text">
Search for: </span>
<input type="search" class="search-field" placeholder="Search &hellip;" value="" name="s" />
</label>
<button type="submit" class="search-submit"><span class="screen-reader-text">
Search </span></button>
</form>
</section>
<section id="recent-posts-2" class="widget widget_recent_entries">
<h2 class="widget-title">Recent Posts</h2><nav aria-label="Recent Posts">
<ul>
<li>
<a href="https://billduncan.org/latencylens/">latencylens</a>
</li>
<li>
<a href="https://billduncan.org/type-conversion/">Type Conversion</a>
</li>
<li>
<a href="https://billduncan.org/ai-and-sre/">AI and SRE</a>
</li>
</ul>
</nav></section><section id="archives-2" class="widget widget_archive"><h2 class="widget-title">Archives</h2><nav aria-label="Archives">
<ul>
<li><a href='https://billduncan.org/2026/09/'>September 2026</a></li>
<li><a href='https://billduncan.org/2026/08/'>August 2026</a></li>
<li><a href='https://billduncan.org/2023/04/'>April 2023</a></li>
<li><a href='https://billduncan.org/2023/01/'>January 2023</a></li>
<li><a href='https://billduncan.org/2022/09/'>September 2022</a></li>
<li><a href='https://billduncan.org/2022/08/'>August 2022</a></li>
<li><a href='https://billduncan.org/2021/07/'>July 2021</a></li>
<li><a href='https://billduncan.org/2021/02/'>February 2021</a></li>
<li><a href='https://billduncan.org/2020/12/'>December 2020</a></li>
<li><a href='https://billduncan.org/2020/07/'>July 2020</a></li>
<li><a href='https://billduncan.org/2020/06/'>June 2020</a></li>
<li><a href='https://billduncan.org/2020/05/'>May 2020</a></li>
<li><a href='https://billduncan.org/2020/04/'>April 2020</a></li>
<li><a href='https://billduncan.org/2020/03/'>March 2020</a></li>
<li><a href='https://billduncan.org/2020/02/'>February 2020</a></li>
<li><a href='https://billduncan.org/2019/12/'>December 2019</a></li>
<li><a href='https://billduncan.org/2019/11/'>November 2019</a></li>
<li><a href='https://billduncan.org/2019/10/'>October 2019</a></li>
<li><a href='https://billduncan.org/2019/05/'>May 2019</a></li>
<li><a href='https://billduncan.org/2019/01/'>January 2019</a></li>
<li><a href='https://billduncan.org/2018/12/'>December 2018</a></li>
<li><a href='https://billduncan.org/2018/11/'>November 2018</a></li>
<li><a href='https://billduncan.org/2018/10/'>October 2018</a></li>
<li><a href='https://billduncan.org/2018/09/'>September 2018</a></li>
<li><a href='https://billduncan.org/2018/08/'>August 2018</a></li>
<li><a href='https://billduncan.org/2018/07/'>July 2018</a></li>
<li><a href='https://billduncan.org/2018/06/'>June 2018</a></li>
<li><a href='https://billduncan.org/2018/05/'>May 2018</a></li>
<li><a href='https://billduncan.org/2018/04/'>April 2018</a></li>
<li><a href='https://billduncan.org/2018/03/'>March 2018</a></li>
</ul>
</nav></section><section id="categories-2" class="widget widget_categories"><h2 class="widget-title">Categories</h2><nav aria-label="Categories">
<ul>
<li class="cat-item cat-item-97"><a href="https://billduncan.org/category/ai/">AI</a> (1)
</li>
<li class="cat-item cat-item-87"><a href="https://billduncan.org/category/aviation/">Aviation</a> (8)
</li>
<li class="cat-item cat-item-12"><a href="https://billduncan.org/category/code-bits/">Code Bits</a> (14)
</li>
<li class="cat-item cat-item-17"><a href="https://billduncan.org/category/data-corruption/">Data Corruption</a> (2)
</li>
<li class="cat-item cat-item-4"><a href="https://billduncan.org/category/detecting-performance-issues/">Detecting Performance Issues</a> (14)
</li>
<li class="cat-item cat-item-43"><a href="https://billduncan.org/category/devops/">devops</a> (27)
</li>
<li class="cat-item cat-item-11"><a href="https://billduncan.org/category/diagnosing-issues/">Diagnosing Issues</a> (9)
</li>
<li class="cat-item cat-item-51"><a href="https://billduncan.org/category/documentation/">Documentation</a> (5)
</li>
<li class="cat-item cat-item-67"><a href="https://billduncan.org/category/event-logs/">Event logs</a> (2)
</li>
<li class="cat-item cat-item-53"><a href="https://billduncan.org/category/geolocating/">Geolocating</a> (1)
</li>
<li class="cat-item cat-item-47"><a href="https://billduncan.org/category/golang/">golang</a> (1)
</li>
<li class="cat-item cat-item-16"><a href="https://billduncan.org/category/just-for-fun/">Just-for-Fun</a> (12)
</li>
<li class="cat-item cat-item-89"><a href="https://billduncan.org/category/k8s/">K8S</a> (2)
</li>
<li class="cat-item cat-item-68"><a href="https://billduncan.org/category/monitoring/">Monitoring</a> (4)
</li>
<li class="cat-item cat-item-58"><a href="https://billduncan.org/category/ot/">OT</a> (6)
</li>
<li class="cat-item cat-item-46"><a href="https://billduncan.org/category/problem-solving/">Problem Solving</a> (5)
</li>
<li class="cat-item cat-item-21"><a href="https://billduncan.org/category/r/">R</a> (2)
</li>
<li class="cat-item cat-item-60"><a href="https://billduncan.org/category/reading-week/">Reading Week</a> (6)
</li>
<li class="cat-item cat-item-78"><a href="https://billduncan.org/category/remotework/">RemoteWork</a> (2)
</li>
<li class="cat-item cat-item-44"><a href="https://billduncan.org/category/rtfm/">RTFM</a> (5)
</li>
<li class="cat-item cat-item-59"><a href="https://billduncan.org/category/security/">Security</a> (2)
</li>
<li class="cat-item cat-item-45"><a href="https://billduncan.org/category/sre/">SRE</a> (34)
</li>
<li class="cat-item cat-item-10"><a href="https://billduncan.org/category/system-performance/">System Performance</a> (13)
</li>
<li class="cat-item cat-item-1"><a href="https://billduncan.org/category/uncategorized/">Uncategorized</a> (2)
</li>
<li class="cat-item cat-item-61"><a href="https://billduncan.org/category/wfm/">WFM</a> (5)
</li>
</ul>
</nav></section><section id="archives-3" class="widget widget_archive"><h2 class="widget-title">Archives</h2><nav aria-label="Archives">
<ul>
<li><a href='https://billduncan.org/2026/09/'>September 2026</a></li>
<li><a href='https://billduncan.org/2026/08/'>August 2026</a></li>
<li><a href='https://billduncan.org/2023/04/'>April 2023</a></li>
<li><a href='https://billduncan.org/2023/01/'>January 2023</a></li>
<li><a href='https://billduncan.org/2022/09/'>September 2022</a></li>
<li><a href='https://billduncan.org/2022/08/'>August 2022</a></li>
<li><a href='https://billduncan.org/2021/07/'>July 2021</a></li>
<li><a href='https://billduncan.org/2021/02/'>February 2021</a></li>
<li><a href='https://billduncan.org/2020/12/'>December 2020</a></li>
<li><a href='https://billduncan.org/2020/07/'>July 2020</a></li>
<li><a href='https://billduncan.org/2020/06/'>June 2020</a></li>
<li><a href='https://billduncan.org/2020/05/'>May 2020</a></li>
<li><a href='https://billduncan.org/2020/04/'>April 2020</a></li>
<li><a href='https://billduncan.org/2020/03/'>March 2020</a></li>
<li><a href='https://billduncan.org/2020/02/'>February 2020</a></li>
<li><a href='https://billduncan.org/2019/12/'>December 2019</a></li>
<li><a href='https://billduncan.org/2019/11/'>November 2019</a></li>
<li><a href='https://billduncan.org/2019/10/'>October 2019</a></li>
<li><a href='https://billduncan.org/2019/05/'>May 2019</a></li>
<li><a href='https://billduncan.org/2019/01/'>January 2019</a></li>
<li><a href='https://billduncan.org/2018/12/'>December 2018</a></li>
<li><a href='https://billduncan.org/2018/11/'>November 2018</a></li>
<li><a href='https://billduncan.org/2018/10/'>October 2018</a></li>
<li><a href='https://billduncan.org/2018/09/'>September 2018</a></li>
<li><a href='https://billduncan.org/2018/08/'>August 2018</a></li>
<li><a href='https://billduncan.org/2018/07/'>July 2018</a></li>
<li><a href='https://billduncan.org/2018/06/'>June 2018</a></li>
<li><a href='https://billduncan.org/2018/05/'>May 2018</a></li>
<li><a href='https://billduncan.org/2018/04/'>April 2018</a></li>
<li><a href='https://billduncan.org/2018/03/'>March 2018</a></li>
</ul>
</nav></section> </aside><!-- .sidebar .widget-area -->
</div><!-- .site-content -->
<footer id="colophon" class="site-footer">
<nav class="main-navigation" aria-label="Footer Primary Menu">
<div class="menu-primary-menu-container"><ul id="menu-primary-menu-1" class="primary-menu"><li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-home menu-item-15"><a href="https://billduncan.org">Home</a></li>
<li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-13"><a href="https://billduncan.org/about/">About Me</a></li>
<li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-12"><a href="https://billduncan.org/contact/">Contact bill duncan</a></li>
<li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-87"><a href="https://billduncan.org/references/">Links</a></li>
</ul></div> </nav><!-- .main-navigation -->
<div class="site-info">
<span class="site-title"><a href="https://billduncan.org/" rel="home">bill duncan&#039;s blog</a></span>
<a href="https://wordpress.org/" class="imprint">
Proudly powered by WordPress </a>
</div><!-- .site-info -->
</footer><!-- .site-footer -->
</div><!-- .site-inner -->
</div><!-- .site -->
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/twentysixteen/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<script async data-wp-strategy="async" id="trustedsite-badge-js" src="https://cdn.trustedsite.com/js/1.js"></script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://billduncan.org/wp-includes/js/wp-emoji-release.min.js?ver=7.0.4"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=https://billduncan.org/wp-includes/js/wp-emoji-loader.min.js
</script>
</body>
</html>