Files
nexus/sreweekly/articles/336/05-latency-and-throughput-optimized-clusters-under-load.html
2026-09-12 17:23:01 +08:00

1327 lines
132 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="profile" href="http://gmpg.org/xfn/11">
<link rel="pingback" href="https://blog.danslimmon.com/xmlrpc.php">
<title>Latency- and Throughput-Optimized Clusters Under Load &#8211; Dan Slimmon</title>
<script type="text/javascript">
WebFontConfig = {"google":{"families":["Montserrat:r:latin,latin-ext","Montserrat:r,i,b,bi:latin,latin-ext"]},"api_url":"https:\/\/fonts-api.wp.com\/css"};
(function() {
var wf = document.createElement('script');
wf.src = '/wp-content/plugins/custom-fonts/js/webfont.js';
wf.type = 'text/javascript';
wf.async = 'true';
var s = document.getElementsByTagName('script')[0];
s.parentNode.insertBefore(wf, s);
})();
</script><style id="jetpack-custom-fonts-css">.wf-active body{font-family:"Montserrat",sans-serif}.wf-active code, .wf-active kbd, .wf-active pre, .wf-active samp{font-family:"Montserrat",sans-serif}.wf-active .main-navigation ul > li + li:before{font-family:"Montserrat",sans-serif}.wf-active .entry-meta > span{font-family:"Montserrat",sans-serif}.wf-active .page-links{font-family:"Montserrat",sans-serif}.wf-active .tagcloud a, .wf-active .tags-links a{font-family:"Montserrat",sans-serif}.wf-active .post-navigation span{font-family:"Montserrat",sans-serif}.wf-active .entry-content{font-family:"Montserrat",sans-serif}.wf-active .entry-content .more-link{font-family:"Montserrat",sans-serif}.wf-active .entry-content blockquote cite{font-family:"Montserrat",sans-serif}.wf-active code, .wf-active kbd, .wf-active pre{font-family:"Montserrat",sans-serif}.wf-active .gallery-caption, .wf-active .wp-caption-text{font-family:"Montserrat",sans-serif}.wf-active .entry-content fieldset legend{font-family:"Montserrat",sans-serif}.wf-active .entry-content label{font-family:"Montserrat",sans-serif}.wf-active .entry-content input, .wf-active .entry-content textarea{font-family:"Montserrat",sans-serif}.wf-active .comments-title span a{font-family:"Montserrat",sans-serif}.wf-active .comment-metadata, .wf-active .comment-reply-link{font-family:"Montserrat",sans-serif}.wf-active .comment-navigation a{font-family:"Montserrat",sans-serif}.wf-active .comment-respond form label{font-family:"Montserrat",sans-serif}.wf-active .comment-respond input[type="text"], .wf-active .comment-respond textarea{font-family:"Montserrat",sans-serif}.wf-active .comment-respond input[type="submit"]{font-family:"Montserrat",sans-serif}.wf-active .comment-list #cancel-comment-reply-link{font-family:"Montserrat",sans-serif}.wf-active .search-field{font-family:"Montserrat",sans-serif}.wf-active .posts-navigation span{font-family:"Montserrat",sans-serif}.wf-active #infinite-handle span{font-family:"Montserrat",sans-serif}.wf-active .widget{font-family:"Montserrat",sans-serif}.wf-active .widget_authors a strong, .wf-active .widget_recent_comments .comment-author-link, .wf-active .widget_recent_comments .comment-author-link a{font-family:"Montserrat",sans-serif}.wf-active .widget .search-field{font-family:"Montserrat",sans-serif}.wf-active .widget_recent_entries a, .wf-active .widget_recent_entries a:focus, .wf-active .widget_recent_entries a:hover, .wf-active .widget_recent_entries a:visited{font-family:"Montserrat",sans-serif}.wf-active .widget_rss li a{font-family:"Montserrat",sans-serif}.wf-active .site-info{font-family:"Montserrat",sans-serif}@media screen and (max-width: 1000px){.wf-active .mobile-search .search-field{font-family:"Montserrat",sans-serif}}.wf-active h1, .wf-active h2, .wf-active h3, .wf-active h4, .wf-active h5, .wf-active h6{font-family:"Montserrat",sans-serif;font-style:normal;font-weight:400}.wf-active .site-title{font-weight:400;font-style:normal}.wf-active .site-description{font-style:normal;font-weight:400}.wf-active .entry-title{font-weight:400;font-style:normal}.wf-active .post-navigation h4{font-weight:400;font-style:normal}.wf-active .entry-content h1, .wf-active .entry-content h2, .wf-active .entry-content h3, .wf-active .entry-content h4, .wf-active .entry-content h5{font-weight:400;font-style:normal}.wf-active .entry-content h6{font-weight:400;font-style:normal}.wf-active .entry-content h1{font-style:normal;font-weight:400}.wf-active .entry-content h2{font-style:normal;font-weight:400}.wf-active .entry-content h3{font-style:normal;font-weight:400}.wf-active .entry-content h4{font-style:normal;font-weight:400}.wf-active .entry-content h5{font-style:normal;font-weight:400}.wf-active .entry-content h6{font-family:"Montserrat",sans-serif;font-weight:400;font-style:normal}.wf-active .entry-content th{font-weight:400;font-style:normal}.wf-active .comment-reply-title, .wf-active .comments-title{font-weight:400;font-style:normal}.wf-active .page-title{font-family:"Montserrat",sans-serif;font-weight:400;font-style:normal}.wf-active .widget-title{font-family:"Montserrat",sans-serif;font-weight:400;font-style:normal}@media screen and (max-width: 600px){.wf-active .site-title{font-style:normal;font-weight:400}}@media screen and (max-width: 600px){.wf-active .entry-title{font-style:normal;font-weight:400}}@media screen and (max-width: 500px){.wf-active .comment-reply-title, .wf-active .comments-title{font-style:normal;font-weight:400}}@media screen and (max-width: 500px){.wf-active .entry-title{font-style:normal;font-weight:400}}@media screen and (max-width: 500px){.wf-active .entry-content h1{font-style:normal;font-weight:400}}@media screen and (max-width: 500px){.wf-active .entry-content h2{font-style:normal;font-weight:400}}@media screen and (max-width: 500px){.wf-active .entry-content h3{font-style:normal;font-weight:400}}@media screen and (max-width: 500px){.wf-active .entry-content h4{font-style:normal;font-weight:400}}@media screen and (max-width: 500px){.wf-active .entry-content h5{font-style:normal;font-weight:400}}@media screen and (max-width: 440px){.wf-active .comment-reply-title, .wf-active .comments-title{font-style:normal;font-weight:400}}@media screen and (max-width: 440px){.wf-active .entry-title{font-style:normal;font-weight:400}}</style>
<meta name='robots' content='max-image-preview:large' />
<!-- Async WordPress.com Remote Login -->
<script id="wpcom_remote_login_js">
var wpcom_remote_login_extra_auth = '';
function wpcom_remote_login_remove_dom_node_id( element_id ) {
var dom_node = document.getElementById( element_id );
if ( dom_node ) { dom_node.parentNode.removeChild( dom_node ); }
}
function wpcom_remote_login_remove_dom_node_classes( class_name ) {
var dom_nodes = document.querySelectorAll( '.' + class_name );
for ( var i = 0; i < dom_nodes.length; i++ ) {
dom_nodes[ i ].parentNode.removeChild( dom_nodes[ i ] );
}
}
function wpcom_remote_login_final_cleanup() {
wpcom_remote_login_remove_dom_node_classes( "wpcom_remote_login_msg" );
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" );
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_validate" );
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_js" );
wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_iframe" );
wpcom_remote_login_remove_dom_node_id( "wpcom_request_access_styles" );
}
// Watch for messages back from the remote login
window.addEventListener( "message", function( e ) {
if ( e.origin === "https://r-login.wordpress.com" ) {
var data = {};
try {
data = JSON.parse( e.data );
} catch( e ) {
wpcom_remote_login_final_cleanup();
return;
}
if ( data.msg === 'LOGIN' ) {
// Clean up the login check iframe
wpcom_remote_login_remove_dom_node_id( "wpcom_remote_login_key" );
var id_regex = new RegExp( /^[0-9]+$/ );
var token_regex = new RegExp( /^.*|.*|.*$/ );
if (
token_regex.test( data.token )
&& id_regex.test( data.wpcomid )
) {
// We have everything we need to ask for a login
var script = document.createElement( "script" );
script.setAttribute( "id", "wpcom_remote_login_validate" );
script.src = '/remote-login.php?wpcom_remote_login=validate'
+ '&wpcomid=' + data.wpcomid
+ '&token=' + encodeURIComponent( data.token )
+ '&host=' + window.location.protocol
+ '//' + window.location.hostname
+ '&postid=2294'
+ '&is_singular=1';
document.body.appendChild( script );
}
return;
}
// Safari ITP, not logged in, so redirect
if ( data.msg === 'LOGIN-REDIRECT' ) {
window.location = 'https://wordpress.com/log-in?redirect_to=' + window.location.href;
return;
}
// Safari ITP, storage access failed, remove the request
if ( data.msg === 'LOGIN-REMOVE' ) {
var css_zap = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } @media screen and ( max-width: 782px ) { html { margin-top: 0 !important; } * html body { margin-top: 0 !important; } }';
var style_zap = document.createElement( 'style' );
style_zap.type = 'text/css';
style_zap.appendChild( document.createTextNode( css_zap ) );
document.body.appendChild( style_zap );
var e = document.getElementById( 'wpcom_request_access_iframe' );
e.parentNode.removeChild( e );
document.cookie = 'wordpress_com_login_access=denied; path=/; max-age=31536000';
return;
}
// Safari ITP
if ( data.msg === 'REQUEST_ACCESS' ) {
console.log( 'request access: safari' );
// Check ITP iframe enable/disable knob
if ( wpcom_remote_login_extra_auth !== 'safari_itp_iframe' ) {
return;
}
// If we are in a "private window" there is no ITP.
var private_window = false;
try {
var opendb = window.openDatabase( null, null, null, null );
} catch( e ) {
private_window = true;
}
if ( private_window ) {
console.log( 'private window' );
return;
}
var iframe = document.createElement( 'iframe' );
iframe.id = 'wpcom_request_access_iframe';
iframe.setAttribute( 'scrolling', 'no' );
iframe.setAttribute( 'sandbox', 'allow-storage-access-by-user-activation allow-scripts allow-same-origin allow-top-navigation-by-user-activation' );
iframe.src = 'https://r-login.wordpress.com/remote-login.php?wpcom_remote_login=request_access&origin=' + encodeURIComponent( data.origin ) + '&wpcomid=' + encodeURIComponent( data.wpcomid );
var css = 'html { -webkit-transition: margin-top 1s; transition: margin-top 1s; } /* 9001 */ html { margin-top: 46px !important; } * html body { margin-top: 46px !important; } @media screen and ( max-width: 660px ) { html { margin-top: 71px !important; } * html body { margin-top: 71px !important; } #wpcom_request_access_iframe { display: block; height: 71px !important; } } #wpcom_request_access_iframe { border: 0px; height: 46px; position: fixed; top: 0; left: 0; width: 100%; min-width: 100%; z-index: 99999; background: #23282d; } ';
var style = document.createElement( 'style' );
style.type = 'text/css';
style.id = 'wpcom_request_access_styles';
style.appendChild( document.createTextNode( css ) );
document.body.appendChild( style );
document.body.appendChild( iframe );
}
if ( data.msg === 'DONE' ) {
wpcom_remote_login_final_cleanup();
}
}
}, false );
// Inject the remote login iframe after the page has had a chance to load
// more critical resources
window.addEventListener( "DOMContentLoaded", function( e ) {
var iframe = document.createElement( "iframe" );
iframe.style.display = "none";
iframe.setAttribute( "scrolling", "no" );
iframe.setAttribute( "id", "wpcom_remote_login_key" );
iframe.src = "https://r-login.wordpress.com/remote-login.php"
+ "?wpcom_remote_login=key"
+ "&origin=aHR0cHM6Ly9ibG9nLmRhbnNsaW1tb24uY29t"
+ "&wpcomid=38620006"
+ "&time=" + Math.floor( Date.now() / 1000 );
document.body.appendChild( iframe );
}, false );
</script>
<link rel='dns-prefetch' href='//s2.wp.com' />
<link rel='dns-prefetch' href='//widgets.wp.com' />
<link rel='dns-prefetch' href='//fonts-api.wp.com' />
<link rel='dns-prefetch' href='//s1.wp.com' />
<link rel='dns-prefetch' href='//s0.wp.com' />
<link rel="alternate" type="application/rss+xml" title="Dan Slimmon &raquo; Feed" href="https://blog.danslimmon.com/feed/" />
<link rel="alternate" type="application/rss+xml" title="Dan Slimmon &raquo; Comments Feed" href="https://blog.danslimmon.com/comments/feed/" />
<link rel="alternate" type="application/rss+xml" title="Dan Slimmon &raquo; Latency- and Throughput-Optimized Clusters Under&nbsp;Load Comments Feed" href="https://blog.danslimmon.com/2019/03/25/latency-and-throughput-optimized-clusters-under-load/feed/" />
<script type="text/javascript">
/* <![CDATA[ */
function addLoadEvent(func) {
var oldonload = window.onload;
if (typeof window.onload != 'function') {
window.onload = func;
} else {
window.onload = function () {
oldonload();
func();
}
}
}
/* ]]> */
</script>
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-2-1' href='/wp-content/mu-plugins/likes/jetpack-likes.css?m=1743883414i&cssminify=yes' type='text/css' media='all' />
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/common.min.css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-0-2' href='/_static/??-eJy1zEEOwiAQQNELWSZVa3FhPAtQMh2lMJmBmt7eJq7c6/LnJw9e3IWSa8wVODWkrIBtTx8F9yMR1uPJXE0PvlGaQOuWooJPJTy7RF6cbIBSGn+WWSiboHqAn8hzdBNl/IvNThyK4/lbvy+3frR2tMNwvjzecYlpQQ==&cssminify=yes' type='text/css' media='all' />
<style id="wp-block-library-inline-css-extra">
.has-text-align-justify {
text-align:justify;
}
.has-text-align-justify{text-align:justify;}
/*# sourceURL=wp-block-library-inline-css */
</style>
<style id="classic-theme-styles-inline-css">
.wp-block-button__link{background-color:#32373c;border-radius:9999px;box-shadow:none;color:#fff;font-size:1.125em;padding:calc(.667em + 2px) calc(1.333em + 2px);text-decoration:none}.wp-block-file__button{background:#32373c;color:#fff}.wp-block-accordion-heading{margin:0}.wp-block-accordion-heading__toggle{background-color:inherit!important;color:inherit!important}.wp-block-accordion-heading__toggle:not(:focus-visible){outline:none}.wp-block-accordion-heading__toggle:focus,.wp-block-accordion-heading__toggle:hover{background-color:inherit!important;border:none;box-shadow:none;color:inherit;padding:var(--wp--preset--spacing--20,1em) 0;text-decoration:none}.wp-block-accordion-heading__toggle:focus-visible{outline:auto;outline-offset:0}.wp-block-tab:not(.has-text-color){color:inherit!important}.wp-block-tab:not(.has-background){background-color:inherit!important}.wp-block-tab:focus,.wp-block-tab:hover{text-decoration:none}.wp-block-tab:focus-visible{outline:auto;outline-offset:0}
/*# sourceURL=/wp-content/plugins/gutenberg-core/v23.9.1/build/styles/block-library/classic.min.css */
</style>
<style id="coblocks-inline-format-inline-css">
.uppercase{text-transform:uppercase}
/*# sourceURL=coblocks-inline-format-inline-css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-12-1' href='/_static/??-eJx9jdEKwjAMRX/IGEXF7UH8lrYL2tm0oWnZ7y8D8UHElwsn9x6Ci0AouVFuyB0k9UfMiqFUsjuLa2gLpik6SsQ22wfVHf7WFjEHvJdKqmDJsTO0p4n6z5upiQuvNyOXssXUEylq9xpqlBZL/pAnsN6lb96e3Pl2vF5Oh2E8j8O8AvKaVKM=&cssminify=yes' type='text/css' media='all' />
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--font-family--albert-sans: 'Albert Sans', sans-serif;--wp--preset--font-family--alegreya: Alegreya, serif;--wp--preset--font-family--arvo: Arvo, serif;--wp--preset--font-family--bodoni-moda: 'Bodoni Moda', serif;--wp--preset--font-family--bricolage-grotesque: 'Bricolage Grotesque', sans-serif;--wp--preset--font-family--cabin: Cabin, sans-serif;--wp--preset--font-family--chivo: Chivo, sans-serif;--wp--preset--font-family--commissioner: Commissioner, sans-serif;--wp--preset--font-family--cormorant: Cormorant, serif;--wp--preset--font-family--courier-prime: 'Courier Prime', monospace;--wp--preset--font-family--crimson-pro: 'Crimson Pro', serif;--wp--preset--font-family--dm-mono: 'DM Mono', monospace;--wp--preset--font-family--dm-sans: 'DM Sans', sans-serif;--wp--preset--font-family--dm-serif-display: 'DM Serif Display', serif;--wp--preset--font-family--domine: Domine, serif;--wp--preset--font-family--eb-garamond: 'EB Garamond', serif;--wp--preset--font-family--epilogue: Epilogue, sans-serif;--wp--preset--font-family--fahkwang: Fahkwang, sans-serif;--wp--preset--font-family--figtree: Figtree, sans-serif;--wp--preset--font-family--fira-sans: 'Fira Sans', sans-serif;--wp--preset--font-family--fjalla-one: 'Fjalla One', sans-serif;--wp--preset--font-family--fraunces: Fraunces, serif;--wp--preset--font-family--gabarito: Gabarito, system-ui;--wp--preset--font-family--ibm-plex-mono: 'IBM Plex Mono', monospace;--wp--preset--font-family--ibm-plex-sans: 'IBM Plex Sans', sans-serif;--wp--preset--font-family--ibarra-real-nova: 'Ibarra Real Nova', serif;--wp--preset--font-family--instrument-serif: 'Instrument Serif', serif;--wp--preset--font-family--inter: Inter, sans-serif;--wp--preset--font-family--josefin-sans: 'Josefin Sans', sans-serif;--wp--preset--font-family--jost: Jost, sans-serif;--wp--preset--font-family--libre-baskerville: 'Libre Baskerville', serif;--wp--preset--font-family--libre-franklin: 'Libre Franklin', sans-serif;--wp--preset--font-family--literata: Literata, serif;--wp--preset--font-family--lora: Lora, serif;--wp--preset--font-family--merriweather: Merriweather, serif;--wp--preset--font-family--montserrat: Montserrat, sans-serif;--wp--preset--font-family--newsreader: Newsreader, serif;--wp--preset--font-family--noto-sans-mono: 'Noto Sans Mono', sans-serif;--wp--preset--font-family--nunito: Nunito, sans-serif;--wp--preset--font-family--open-sans: 'Open Sans', sans-serif;--wp--preset--font-family--overpass: Overpass, sans-serif;--wp--preset--font-family--pt-serif: 'PT Serif', serif;--wp--preset--font-family--petrona: Petrona, serif;--wp--preset--font-family--piazzolla: Piazzolla, serif;--wp--preset--font-family--playfair-display: 'Playfair Display', serif;--wp--preset--font-family--plus-jakarta-sans: 'Plus Jakarta Sans', sans-serif;--wp--preset--font-family--poppins: Poppins, sans-serif;--wp--preset--font-family--raleway: Raleway, sans-serif;--wp--preset--font-family--roboto: Roboto, sans-serif;--wp--preset--font-family--roboto-slab: 'Roboto Slab', serif;--wp--preset--font-family--rubik: Rubik, sans-serif;--wp--preset--font-family--rufina: Rufina, serif;--wp--preset--font-family--sora: Sora, sans-serif;--wp--preset--font-family--source-sans-3: 'Source Sans 3', sans-serif;--wp--preset--font-family--source-serif-4: 'Source Serif 4', serif;--wp--preset--font-family--space-mono: 'Space Mono', monospace;--wp--preset--font-family--syne: Syne, sans-serif;--wp--preset--font-family--texturina: Texturina, serif;--wp--preset--font-family--urbanist: Urbanist, sans-serif;--wp--preset--font-family--work-sans: 'Work Sans', sans-serif;--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:where(body) { margin: 0; }:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{padding-top: 0px;padding-right: 0px;padding-bottom: 0px;padding-left: 0px;}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}.has-albert-sans-font-family{font-family: var(--wp--preset--font-family--albert-sans) !important;}.has-alegreya-font-family{font-family: var(--wp--preset--font-family--alegreya) !important;}.has-arvo-font-family{font-family: var(--wp--preset--font-family--arvo) !important;}.has-bodoni-moda-font-family{font-family: var(--wp--preset--font-family--bodoni-moda) !important;}.has-bricolage-grotesque-font-family{font-family: var(--wp--preset--font-family--bricolage-grotesque) !important;}.has-cabin-font-family{font-family: var(--wp--preset--font-family--cabin) !important;}.has-chivo-font-family{font-family: var(--wp--preset--font-family--chivo) !important;}.has-commissioner-font-family{font-family: var(--wp--preset--font-family--commissioner) !important;}.has-cormorant-font-family{font-family: var(--wp--preset--font-family--cormorant) !important;}.has-courier-prime-font-family{font-family: var(--wp--preset--font-family--courier-prime) !important;}.has-crimson-pro-font-family{font-family: var(--wp--preset--font-family--crimson-pro) !important;}.has-dm-mono-font-family{font-family: var(--wp--preset--font-family--dm-mono) !important;}.has-dm-sans-font-family{font-family: var(--wp--preset--font-family--dm-sans) !important;}.has-dm-serif-display-font-family{font-family: var(--wp--preset--font-family--dm-serif-display) !important;}.has-domine-font-family{font-family: var(--wp--preset--font-family--domine) !important;}.has-eb-garamond-font-family{font-family: var(--wp--preset--font-family--eb-garamond) !important;}.has-epilogue-font-family{font-family: var(--wp--preset--font-family--epilogue) !important;}.has-fahkwang-font-family{font-family: var(--wp--preset--font-family--fahkwang) !important;}.has-figtree-font-family{font-family: var(--wp--preset--font-family--figtree) !important;}.has-fira-sans-font-family{font-family: var(--wp--preset--font-family--fira-sans) !important;}.has-fjalla-one-font-family{font-family: var(--wp--preset--font-family--fjalla-one) !important;}.has-fraunces-font-family{font-family: var(--wp--preset--font-family--fraunces) !important;}.has-gabarito-font-family{font-family: var(--wp--preset--font-family--gabarito) !important;}.has-ibm-plex-mono-font-family{font-family: var(--wp--preset--font-family--ibm-plex-mono) !important;}.has-ibm-plex-sans-font-family{font-family: var(--wp--preset--font-family--ibm-plex-sans) !important;}.has-ibarra-real-nova-font-family{font-family: var(--wp--preset--font-family--ibarra-real-nova) !important;}.has-instrument-serif-font-family{font-family: var(--wp--preset--font-family--instrument-serif) !important;}.has-inter-font-family{font-family: var(--wp--preset--font-family--inter) !important;}.has-josefin-sans-font-family{font-family: var(--wp--preset--font-family--josefin-sans) !important;}.has-jost-font-family{font-family: var(--wp--preset--font-family--jost) !important;}.has-libre-baskerville-font-family{font-family: var(--wp--preset--font-family--libre-baskerville) !important;}.has-libre-franklin-font-family{font-family: var(--wp--preset--font-family--libre-franklin) !important;}.has-literata-font-family{font-family: var(--wp--preset--font-family--literata) !important;}.has-lora-font-family{font-family: var(--wp--preset--font-family--lora) !important;}.has-merriweather-font-family{font-family: var(--wp--preset--font-family--merriweather) !important;}.has-montserrat-font-family{font-family: var(--wp--preset--font-family--montserrat) !important;}.has-newsreader-font-family{font-family: var(--wp--preset--font-family--newsreader) !important;}.has-noto-sans-mono-font-family{font-family: var(--wp--preset--font-family--noto-sans-mono) !important;}.has-nunito-font-family{font-family: var(--wp--preset--font-family--nunito) !important;}.has-open-sans-font-family{font-family: var(--wp--preset--font-family--open-sans) !important;}.has-overpass-font-family{font-family: var(--wp--preset--font-family--overpass) !important;}.has-pt-serif-font-family{font-family: var(--wp--preset--font-family--pt-serif) !important;}.has-petrona-font-family{font-family: var(--wp--preset--font-family--petrona) !important;}.has-piazzolla-font-family{font-family: var(--wp--preset--font-family--piazzolla) !important;}.has-playfair-display-font-family{font-family: var(--wp--preset--font-family--playfair-display) !important;}.has-plus-jakarta-sans-font-family{font-family: var(--wp--preset--font-family--plus-jakarta-sans) !important;}.has-poppins-font-family{font-family: var(--wp--preset--font-family--poppins) !important;}.has-raleway-font-family{font-family: var(--wp--preset--font-family--raleway) !important;}.has-roboto-font-family{font-family: var(--wp--preset--font-family--roboto) !important;}.has-roboto-slab-font-family{font-family: var(--wp--preset--font-family--roboto-slab) !important;}.has-rubik-font-family{font-family: var(--wp--preset--font-family--rubik) !important;}.has-rufina-font-family{font-family: var(--wp--preset--font-family--rufina) !important;}.has-sora-font-family{font-family: var(--wp--preset--font-family--sora) !important;}.has-source-sans-3-font-family{font-family: var(--wp--preset--font-family--source-sans-3) !important;}.has-source-serif-4-font-family{font-family: var(--wp--preset--font-family--source-serif-4) !important;}.has-space-mono-font-family{font-family: var(--wp--preset--font-family--space-mono) !important;}.has-syne-font-family{font-family: var(--wp--preset--font-family--syne) !important;}.has-texturina-font-family{font-family: var(--wp--preset--font-family--texturina) !important;}.has-urbanist-font-family{font-family: var(--wp--preset--font-family--urbanist) !important;}.has-work-sans-font-family{font-family: var(--wp--preset--font-family--work-sans) !important;}
/*# sourceURL=global-styles-inline-css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-14-1' href='/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/moon/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/verbum-comments/verbum-comments.css?m=1785895867i&cssminify=yes' type='text/css' media='all' />
<link rel='stylesheet' id='verbum-gutenberg-css-css' href='https://widgets.wp.com/verbum-block-editor/block-editor.css?ver=1771405207' media='all' />
<link crossorigin='anonymous' rel='stylesheet' id='all-css-16-1' href='/wp-content/themes/pub/lovecraft/style.css?m=1758759644i&cssminify=yes' type='text/css' media='all' />
<link rel='stylesheet' id='lovecraft-fonts-css' href='https://fonts-api.wp.com/css?family=Playfair+Display%3A400%2C700%2C400italic%7CLato%3A400%2C400italic%2C700%2C700italic%2C900%2C900italic&#038;subset=latin%2Clatin-ext&#038;ver=7.1.1-alpha-63555' media='all' />
<link crossorigin='anonymous' rel='stylesheet' id='all-css-18-1' href='/_static/??-eJx9jsEKwjAQRH/IuC1C60X8FEmXbUzdZEOytfj3puAhoHibB/OGgS0ZlKgUFcJqEq/OxwILabL4+DAEkQg3HxEcRcq+GuV3PGIpB2hG9U6BCqR1ApYnYbazwr5U9MVktoQSvqTmSaaJxdXooLYa/Cc5EsOCVn393YKZ2fq8q9dw6cexH7rhdO6WN9IbYnE=&cssminify=yes' type='text/css' media='all' />
<link crossorigin='anonymous' rel='stylesheet' id='print-css-19-1' href='/wp-content/mu-plugins/global-print/global-print.css?m=1465851035i&cssminify=yes' type='text/css' media='print' />
<style id="jetpack-global-styles-frontend-style-inline-css">
:root { --font-headings: unset; --font-base: unset; --font-headings-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif; --font-base-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif;}
/*# sourceURL=jetpack-global-styles-frontend-style-inline-css */
</style>
<link crossorigin='anonymous' rel='stylesheet' id='all-css-22-1' href='/_static/??-eJyNjcEKAjEMRH/ImhVlqwfxU6TbhrZrmhTTIv69u+JFvHgZ5sHwBh7VeOGG3KB0U6nHzAoztur87cNQRNYInVBBk7tjcCE83zVz3HrVDfxvumb2MPVMAVR8dmRIougX/DhbwrK8pwNEksnROriU884e9+PJDnaYXxwFS20=&cssminify=yes' type='text/css' media='all' />
<script id="wpcom-actionbar-placeholder-js-extra">
var actionbardata = {"siteID":"38620006","postID":"2294","siteURL":"https://blog.danslimmon.com","xhrURL":"https://blog.danslimmon.com/wp-admin/admin-ajax.php","nonce":"e8f3aebd24","isLoggedIn":"","statusMessage":"","subsEmailDefault":"instantly","proxyScriptUrl":"https://s0.wp.com/wp-content/js/wpcom-proxy-request.js?m=1513050504i&amp;ver=20211021","shortlink":"https://wp.me/p2C2Pk-B0","i18n":{"followedText":"New posts from this site will now appear in your \u003Ca href=\"https://wordpress.com/reader\"\u003EReader\u003C/a\u003E","foldBar":"Collapse this bar","unfoldBar":"Expand this bar","shortLinkCopied":"Shortlink copied to clipboard."}};
//# sourceURL=wpcom-actionbar-placeholder-js-extra
</script>
<script id="jetpack-mu-wpcom-settings-js-before">
var JETPACK_MU_WPCOM_SETTINGS = {"assetsUrl":"https://s1.wp.com/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/moon/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/"};
//# sourceURL=jetpack-mu-wpcom-settings-js-before
</script>
<script crossorigin='anonymous' type='text/javascript' src='/wp-content/js/rlt-proxy.js?m=1720530689i'></script>
<script id="rlt-proxy-js-after">
rltInitialize( {"token":null,"iframeOrigins":["https:\/\/widgets.wp.com"]} );
//# sourceURL=rlt-proxy-js-after
</script>
<link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://danslimmon.wordpress.com/xmlrpc.php?rsd" />
<meta name="generator" content="WordPress.com" />
<link rel="canonical" href="https://blog.danslimmon.com/2019/03/25/latency-and-throughput-optimized-clusters-under-load/" />
<link rel='shortlink' href='https://wp.me/p2C2Pk-B0' />
<link rel="alternate" type="application/json+oembed" href="https://public-api.wordpress.com/oembed/?format=json&amp;url=https%3A%2F%2Fblog.danslimmon.com%2F2019%2F03%2F25%2Flatency-and-throughput-optimized-clusters-under-load%2F&amp;for=wpcom-auto-discovery" /><link rel="alternate" type="application/xml+oembed" href="https://public-api.wordpress.com/oembed/?format=xml&amp;url=https%3A%2F%2Fblog.danslimmon.com%2F2019%2F03%2F25%2Flatency-and-throughput-optimized-clusters-under-load%2F&amp;for=wpcom-auto-discovery" />
<!-- Jetpack Open Graph Tags -->
<meta property="og:type" content="article" />
<meta property="og:title" content="Latency- and Throughput-Optimized Clusters Under Load" />
<meta property="og:url" content="https://blog.danslimmon.com/2019/03/25/latency-and-throughput-optimized-clusters-under-load/" />
<meta property="og:description" content="It&#8217;s good to have accurate and thorough metrics for our systems. But that&#8217;s only where observability starts. In order to get value out of metrics, we have to focus on the right ones: th…" />
<meta property="article:published_time" content="2019-03-25T16:44:22+00:00" />
<meta property="article:modified_time" content="2019-03-25T16:44:22+00:00" />
<meta property="og:site_name" content="Dan Slimmon" />
<meta property="og:image" content="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-11-blog-charts-latency-oriented-healthy.png" />
<meta property="og:image:width" content="800" />
<meta property="og:image:height" content="632" />
<meta property="og:image:alt" content="2019-03-11 blog charts latency-oriented healthy" />
<meta property="og:locale" content="en_US" />
<meta property="article:publisher" content="https://www.facebook.com/WordPresscom" />
<meta name="twitter:text:title" content="Latency- and Throughput-Optimized Clusters Under&nbsp;Load" />
<meta name="twitter:image" content="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-11-blog-charts-latency-oriented-healthy.png?w=640" />
<meta name="twitter:image:alt" content="2019-03-11 blog charts latency-oriented healthy" />
<meta name="twitter:card" content="summary_large_image" />
<!-- End Jetpack Open Graph Tags -->
<link rel='openid.server' href='https://blog.danslimmon.com/?openidserver=1' />
<link rel='openid.delegate' href='https://blog.danslimmon.com/' />
<link rel="search" type="application/opensearchdescription+xml" href="https://blog.danslimmon.com/osd.xml" title="Dan Slimmon" />
<link rel="search" type="application/opensearchdescription+xml" href="https://s1.wp.com/opensearch.xml" title="WordPress.com" />
<style type="text/css">
.recentcomments a {
display: inline !important;
padding: 0 !important;
margin: 0 !important;
}
table.recentcommentsavatartop img.avatar, table.recentcommentsavatarend img.avatar {
border: 0px;
margin: 0;
}
table.recentcommentsavatartop a, table.recentcommentsavatarend a {
border: 0px !important;
background-color: transparent !important;
}
td.recentcommentsavatarend, td.recentcommentsavatartop {
padding: 0px 0px 1px 0px;
margin: 0px;
}
td.recentcommentstextend {
border: none !important;
padding: 0px 0px 2px 10px;
}
.rtl td.recentcommentstextend {
padding: 0px 10px 2px 0px;
}
td.recentcommentstexttop {
border: none;
padding: 0px 0px 0px 10px;
}
.rtl td.recentcommentstexttop {
padding: 0px 10px 0px 0px;
}
</style>
<meta name="description" content="It&#039;s good to have accurate and thorough metrics for our systems. But that&#039;s only where observability starts. In order to get value out of metrics, we have to focus on the right ones: the ones that tell us about outcomes that matter to our users. In The Latency/Throughput Tradeoff, we talked about why a given&hellip;" />
<style type="text/css" id="custom-colors-css">.entry-content .page-links a:hover { color: #FFFFFF;}
.footer-widgets .tagcloud a:hover { color: #FFFFFF;}
.tags-links a:hover, .tagcloud a:hover { color: #FFFFFF;}
.sticky-post, .sticky-post .genericon { color: #FFFFFF;}
#infinite-handle span:hover { color: #FFFFFF;}
.main-navigation ul ul a, .main-navigation ul ul a:visited { color: #FFFFFF;}
.button:hover, button:hover, input[type="submit"]:hover, input[type="reset"]:hover, input[type="button"]:hover { color: #FFFFFF;}
.mobile-search, .mobile-search .search-field, .toggle-active .main-navigation li { border-color: #dddddd;}
.mobile-search, .mobile-search .search-field, .toggle-active .main-navigation li { border-color: rgba( 221, 221, 221, 0.3 );}
.toggles, .hentry, #wp-calendar th:first-child, #wp-calendar tbody td:first-child, #wp-calendar thead, #wp-calendar th, #wp-calendar td, .widget ul ul li:first-child, .main-navigation, .widget li, .widget + .widget, .site-footer { border-color: #dddddd;}
.toggles, .hentry, #wp-calendar th:first-child, #wp-calendar tbody td:first-child, #wp-calendar thead, #wp-calendar th, #wp-calendar td, .widget ul ul li:first-child, .main-navigation, .widget li, .widget + .widget, .site-footer { border-color: rgba( 221, 221, 221, 0.3 );}
#secondary a, #secondary a:visited, #secondary a:hover, #secondary a:active, #secondary a:focus { color: #111111;}
.main-navigation a, .main-navigation a:visited { color: #111111;}
.menu-toggle .bar { background-color: #111111;}
.site-title a, .site-title a:visited { color: #111111;}
.search-toggle { color: #111111;}
.site-description { color: #111111;}
#secondary { color: #111111;}
#wp-calendar, #wp-calendar caption, #wp-calendar tfoot a, #wp-calendar tfoot a:visited { color: #111111;}
.footer-widgets #wp-calendar { color: #111111;}
.widget_recent_comments .comment-author-link, .widget_recent_comments .comment-author-link a, .widget_authors a strong { color: #111111;}
#secondary .wp-caption-text, #secondary .gallery-caption { color: #111111;}
.main-navigation ul > li + li:before { color: #6B6B6B;}
.widget_archive li, .widget_categories li, .widget_meta li, .widget_nav_menu li, .widget_pages { color: #6B6B6B;}
.widget_archive li:before, .widget_categories li:before, .widget_meta li:before, .widget_nav_menu li:before, .widget_pages li:before, .widget_authors ul ul li:before, .widget_recent_comments li:before { color: #6B6B6B;}
body, .toggles { background-color: #fafafa;}
a, a:visited, a:hover, a:active, a:focus { color: #5F3FF5;}
.comments-title span a, .comments-title span a:visited { color: #5F3FF5;}
.mobile-search .search-submit:hover { color: #5F3FF5;}
.entry-meta a, .entry-meta a:visited { color: #5F3FF5;}
.footer-widgets #wp-calendar thead th { color: #5F3FF5;}
.entry-meta a:hover, .entry-meta a:focus, .entry-meta a:active { border-bottom-color: #5F3FF5;}
.post-navigation a:hover, .post-navigation a:focus, .post-navigation a:active { color: #5F3FF5;}
.comment-navigation a:hover, .comment-navigation a:focus, .comment-navigation a:active { color: #5F3FF5;}
.search-submit:hover .genericon, .search-submit:focus .genericon, .search-submit:active .genericon { color: #5F3FF5;}
.main-navigation ul li:hover > a { color: #5F3FF5;}
#secondary #wp-calendar thead th, #wp-calendar tfoot a:hover { color: #5F3FF5;}
.search-toggle:hover, .search-toggle:active, .search-toggle:focus { color: #5F3FF5;}
.main-navigation ul > .page_item_has_children > a:after, .main-navigation ul > .menu-item-has-children > a:after { border-top-color: #5F3FF5;}
.main-navigation a:hover, .main-navigation a:active, .main-navigation a:focus { color: #5F3FF5;}
.toggle-active .main-navigation a, .toggle-active .main-navigation a:visited { color: #5F3FF5;}
.site-title a:hover, .site-title a:focus, .site-title a:active { color: #5F3FF5;}
.search-toggle.active { color: #5F3FF5;}
.menu-toggle.active .bar { background-color: #5F3FF5;}
.menu-toggle:hover .bar, .menu-toggle:active .bar, .menu-toggle:focus .bar { background-color: #5F3FF5;}
.toggle-active .main-navigation ul ul li:hover > a { color: #5F3FF5;}
.toggle-active .main-navigation ul ul a:hover, .toggle-active .main-navigation ul ul a:active, .toggle-active .main-navigation ul ul a:focus { color: #5F3FF5;}
.menu-toggle:hover .bar, .menu-toggle:active .bar, .menu-toggle:focus .bar { background-color: #5F3FF5;}
.main-navigation ul ul li:hover > a { color: #AA99F4;}
.main-navigation ul ul a:hover, .main-navigation ul ul a:active, .main-navigation ul ul a:focus { color: #AA99F4;}
.site-info a:hover, .site-info a:focus, .site-info a:active { color: #8066F4;}
.comment-author a:hover { color: #522EF4;}
.entry-content blockquote:after { color: #5F3FF5;}
.site-info .sep { color: #8066F4;}
.entry-content .page-links a:hover { background-color: #5f3ff5;}
.bypostauthor .comment-author img { background-color: #5f3ff5;}
#infinite-handle span:hover { background-color: #5f3ff5;}
.tags-links a:hover:before, .tagcloud a:hover:before { border-right-color: #5f3ff5;}
.rtl .tags-links a:hover:before, .rtl .tagcloud a:hover:before { border-left-color: #5f3ff5;}
.sticky-post { background-color: #5f3ff5;}
.sticky-post:after { border-left-color: #5f3ff5;}
.sticky-post:after { border-right-color: #5f3ff5;}
.widget-title, .jetpack-recipe-title { border-color: #5f3ff5;}
.tags-links a:hover, .tagcloud a:hover { background-color: #5f3ff5;}
.footer-widgets .tagcloud a:hover { background-color: #5f3ff5;}
.footer-widgets .tagcloud a:hover:before { border-right-color: #5f3ff5;}
.rtl .footer-widgets .tagcloud a:hover:before { border-left-color: #5f3ff5;}
.button:hover, button:hover, input[type="submit"]:hover, input[type="reset"]:hover, input[type="button"]:hover { background-color: #5f3ff5;}
</style>
<link rel="icon" href="https://blog.danslimmon.com/wp-content/uploads/2024/03/cropped-d2e_pip_transparent.png?w=32" sizes="32x32" />
<link rel="icon" href="https://blog.danslimmon.com/wp-content/uploads/2024/03/cropped-d2e_pip_transparent.png?w=192" sizes="192x192" />
<link rel="apple-touch-icon" href="https://blog.danslimmon.com/wp-content/uploads/2024/03/cropped-d2e_pip_transparent.png?w=180" />
<meta name="msapplication-TileImage" content="https://blog.danslimmon.com/wp-content/uploads/2024/03/cropped-d2e_pip_transparent.png?w=270" />
<link rel="stylesheet" id="custom-css-css" type="text/css" href="https://s2.wp.com/?custom-css=1&#038;csblog=2C2Pk&#038;cscache=6&#038;csrev=4" />
<link crossorigin='anonymous' rel='stylesheet' id='all-css-0-4' href='/_static/??-eJydj8EKwkAMRH/IGtoi9iJ+irRpKGl3N6HJuvj3VlDBm3qbGR4PBopWKMkpOcRcacgTJ4OZXHtcnh2iSIILJ4QhCC4GVlhp3aPZDr43RBlzIAPsV8lG4Q29hn+FhceJ3MBFKxXbkvkt0K+6j4d5MFxZnWWjr0zlYTvHU33sDk3TtnU33wFZQHfT&cssminify=yes' type='text/css' media='all' />
</head>
<body class="wp-singular post-template-default single single-post postid-2294 single-format-standard wp-embed-responsive wp-theme-publovecraft customizer-styles-applied jetpack-reblog-enabled custom-colors">
<div id="page" class="hfeed site">
<a class="skip-link screen-reader-text" href="#content">Skip to content</a>
<header id="masthead" class="site-header" role="banner">
<div class="site-branding">
<a href="https://blog.danslimmon.com/" class="site-logo-link" rel="home" itemprop="url"></a> <h1 class="site-title"><a href="https://blog.danslimmon.com/" rel="home">Dan Slimmon</a></h1>
<p class="site-description">Scientific incident response</p>
</div><!-- .site-branding -->
<div class="toggles clear">
<button class="menu-toggle toggle" aria-controls="primary-menu" aria-expanded="false">
<div class="bar"></div>
<div class="bar"></div>
<div class="bar"></div>
<span class="screen-reader-text">Menu</span>
</button><!-- .menu-toggle -->
<button class="search-toggle toggle">
<div class="genericon genericon-search" aria-expanded="false">
<span class="screen-reader-text">Search</span>
</div><!-- .genericon-search -->
</button><!-- .search-toggle -->
</div><!-- .toggles -->
<nav id="site-navigation" class="main-navigation" role="navigation">
<div class="menu-top-navbar-container"><ul id="primary-menu" class="menu"><li id="menu-item-378" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-home menu-item-378"><a href="https://blog.danslimmon.com">Blog</a></li>
<li id="menu-item-2836" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2836"><a href="https://d2e.engineering/">Incident Response Training</a></li>
<li id="menu-item-2837" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2837"><a href="https://www.linkedin.com/in/danslimmon/">LinkedIn</a></li>
<li id="menu-item-2838" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2838"><a href="https://twitter.com/danslimmon">Twitter</a></li>
<li id="menu-item-2839" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-2839"><a href="mailto:dan@down-to-earth.engineering">Email</a></li>
</ul></div> </nav><!-- #site-navigation -->
<div class="mobile-search">
<form role="search" method="get" class="search-form" action="https://blog.danslimmon.com/">
<label>
<span class="screen-reader-text">Search for:</span>
<input type="search" class="search-field" placeholder="Search …" value="" name="s" title="Search for:" />
</label>
<button type="submit" class="search-submit">
<span class="genericon genericon-search">
<span class="screen-reader-text">Search</span>
</span>
</button>
</form> </div>
</header><!-- #masthead -->
<a class="header-image-wrapper" href="https://blog.danslimmon.com/" rel="home">
<div class="header-image bg-image" style="background-image: url(https://blog.danslimmon.com/wp-content/uploads/2024/05/crap0.png)">
<img src="https://blog.danslimmon.com/wp-content/uploads/2024/05/crap0.png" width="1280" height="444" alt="">
</div>
</a>
<div id="content" class="site-content">
<div id="primary" class="content-area">
<main id="main" class="site-main" role="main">
<article id="post-2294" class="post-2294 post type-post status-publish format-standard hentry category-uncategorized">
<div class="entry-wrapper">
<header class="entry-header">
<h1 class="entry-title">Latency- and Throughput-Optimized Clusters Under&nbsp;Load</h1>
<div class="entry-meta">
<span class="posted-on">On <a href="https://blog.danslimmon.com/2019/03/25/latency-and-throughput-optimized-clusters-under-load/" rel="bookmark"><time class="entry-date published updated" datetime="2019-03-25T16:44:22+00:00">2019/03/25</time></a></span><span class="byline"> By <span class="author vcard"><a class="url fn n" href="https://blog.danslimmon.com/author/danslimmon/">Dan Slimmon</a></span></span><span class="cat-links">In <a href="https://blog.danslimmon.com/category/uncategorized/" rel="category tag">Uncategorized</a></span> </div><!-- .entry-meta -->
</header><!-- .entry-header -->
<div class="entry-content">
<p><span style="font-weight:400;">It&#8217;s good to have accurate and thorough metrics for our systems. But that&#8217;s only where observability starts. In order to get value out of metrics, we have to focus on the </span><i><span style="font-weight:400;">right</span></i><span style="font-weight:400;"> ones: the ones that tell us about outcomes that matter to our users.</span></p>
<p><span style="font-weight:400;">In </span><span style="color:#0000ff;"><a style="color:#0000ff;" href="https://blog.danslimmon.com/2019/02/26/the-latency-throughput-tradeoff-why-fast-services-are-slow-and-vice-versa/"><span style="font-weight:400;">The Latency/Throughput Tradeoff</span></a></span><span style="font-weight:400;">, we talked about why a given cluster can&#8217;t be optimized for low latency and high throughput at the same time. In conclusion, we decided that separate clusters should be provisioned for latency-centric use cases and throughput-centric use cases. And since these different clusters are optimized to provide different outcomes, we&#8217;ll need to interpret their metrics in accordingly different ways.</span></p>
<p><span style="font-weight:400;">Let&#8217;s consider an imaginary graph dashboard for each type of cluster. We&#8217;ll walk through the relationships between metrics in each cluster, both under normal conditions and under excessive load. And then we&#8217;ll wrap up with some ideas about evaluating the capacity of each cluster over the longer term.</span></p>
<h2><span style="font-weight:400;">Metrics for comparison</span></h2>
<p><span style="font-weight:400;">In order to contrast our two types of clusters, we&#8217;ll need some common metrics. I like to employ the </span><span style="color:#0000ff;"><a style="color:#0000ff;" href="http://brendangregg.com/usemethod.html"><span style="font-weight:400;">USE metrics</span></a></span><span style="font-weight:400;"> at the top of my graph dashboards (more on that in a future post). So let&#8217;s use those:</span></p>
<ul>
<li><b>Utilization:</b><span style="font-weight:400;"> The total CPU usage for all hosts in the cluster.</span></li>
<li><b>Saturation:<span style="font-weight:400;"> The number of queued requests.</span></b></li>
<li><b>Error rate:<span style="font-weight:400;"> The rate at which lines are being added to hosts&#8217; error logs across the cluster.</span></b></li>
</ul>
<p>In addition to these three metrics, we want to see a fourth metric. For the latency-optimized cluster, we want to see <b style="color:var(--color-neutral-600);">latency</b><span style="font-weight:400;">. And for the throughput-optimized cluster, we want to see </span><b style="color:var(--color-neutral-600);">throughput</b><span style="font-weight:400;">.</span></p>
<p><span style="font-weight:400;">One of the best things about splitting out latency-optimized and throughput-optimized clusters is that the relationships between metrics become much clearer. It&#8217;s hard to tell when something&#8217;s wrong if you&#8217;re not sure what kind of work your cluster is supposed to be doing at any given moment. But separation of concerns allows us to develop </span><a href="https://blog.danslimmon.com/tag/expertise/"><span style="font-weight:400;"><span style="color:#0000ff;">intuition</span></span></a><span style="font-weight:400;"> about our systems&#8217; behavior and stop groping around in the dark.</span></p>
<h2><span style="font-weight:400;">Latency-optimized cluster</span></h2>
<p><span style="font-weight:400;">Let&#8217;s look at the relationships between these four important metrics in a latency-optimized cluster under normal, healthy conditions:</span></p>
<p><span style="font-weight:400;"><img data-attachment-id="2295" data-permalink="https://blog.danslimmon.com/2019/03/25/latency-and-throughput-optimized-clusters-under-load/2019-03-11-blog-charts-latency-oriented-healthy/" data-orig-file="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-11-blog-charts-latency-oriented-healthy.png" data-orig-size="800,632" data-comments-opened="1" data-image-meta="{&quot;aperture&quot;:&quot;0&quot;,&quot;credit&quot;:&quot;&quot;,&quot;camera&quot;:&quot;&quot;,&quot;caption&quot;:&quot;&quot;,&quot;created_timestamp&quot;:&quot;0&quot;,&quot;copyright&quot;:&quot;&quot;,&quot;focal_length&quot;:&quot;0&quot;,&quot;iso&quot;:&quot;0&quot;,&quot;shutter_speed&quot;:&quot;0&quot;,&quot;title&quot;:&quot;&quot;,&quot;orientation&quot;:&quot;0&quot;}" data-image-title="2019-03-11 blog charts latency-oriented healthy" data-image-description="" data-image-caption="" data-large-file="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-11-blog-charts-latency-oriented-healthy.png?w=640" class="alignnone size-full wp-image-2295" src="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-11-blog-charts-latency-oriented-healthy.png?w=640" alt="2019-03-11 blog charts latency-oriented healthy" srcset="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-11-blog-charts-latency-oriented-healthy.png 800w, https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-11-blog-charts-latency-oriented-healthy.png?w=150&amp;h=119 150w, https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-11-blog-charts-latency-oriented-healthy.png?w=300&amp;h=237 300w, https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-11-blog-charts-latency-oriented-healthy.png?w=768&amp;h=607 768w" sizes="(max-width: 800px) 100vw, 800px" /></span><span style="font-weight:400;">Utilization will vary over time depending on how many requests are in flight. Saturation, however, should stay at zero. Any time a request is queued, we take a latency hit.\</span></p>
<p><span style="font-weight:400;">Error rate would </span><i><span style="font-weight:400;">ideally</span></i><span style="font-weight:400;"> be zero, but come on. We should expect it to be correlated to utilization, following the mental model that all requests carry the same probability of throwing an error.</span></p>
<p><span style="font-weight:400;">Latency, then – the metric this cluster is optimized for – should be constant. It should not be affected by utilization. If latency is correlated to utilization, then that&#8217;s a bug. There will always be some wiggle in the long tail (the high percentiles), but for a large enough workload, the median and the 10th percentile should pretty much stay put.</span></p>
<p><span style="font-weight:400;">Now let&#8217;s see what happens when the cluster is overloaded:</span></p>
<p><img data-attachment-id="2296" data-permalink="https://blog.danslimmon.com/2019/03/25/latency-and-throughput-optimized-clusters-under-load/2019-03-12-blog-charts-latency-oriented-loaded/" data-orig-file="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-12-blog-charts-latency-oriented-loaded.png" data-orig-size="800,627" data-comments-opened="1" data-image-meta="{&quot;aperture&quot;:&quot;0&quot;,&quot;credit&quot;:&quot;&quot;,&quot;camera&quot;:&quot;&quot;,&quot;caption&quot;:&quot;&quot;,&quot;created_timestamp&quot;:&quot;0&quot;,&quot;copyright&quot;:&quot;&quot;,&quot;focal_length&quot;:&quot;0&quot;,&quot;iso&quot;:&quot;0&quot;,&quot;shutter_speed&quot;:&quot;0&quot;,&quot;title&quot;:&quot;&quot;,&quot;orientation&quot;:&quot;0&quot;}" data-image-title="2019-03-12 blog charts latency-oriented loaded" data-image-description="" data-image-caption="" data-large-file="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-12-blog-charts-latency-oriented-loaded.png?w=640" class="alignnone size-full wp-image-2296" src="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-12-blog-charts-latency-oriented-loaded.png?w=640" alt="2019-03-12 blog charts latency-oriented loaded" srcset="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-12-blog-charts-latency-oriented-loaded.png 800w, https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-12-blog-charts-latency-oriented-loaded.png?w=150&amp;h=118 150w, https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-12-blog-charts-latency-oriented-loaded.png?w=300&amp;h=235 300w, https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-12-blog-charts-latency-oriented-loaded.png?w=768&amp;h=602 768w" sizes="(max-width: 800px) 100vw, 800px" /></p>
<p><span style="font-weight:400;">We start to see plateaus in utilization, or at least wider, flatter peaks. This means that the system is low on &#8220;slack&#8221;: the idle resources that it should always have available to serve new requests. When all resources are busy, saturation rises above zero.</span></p>
<p><span style="font-weight:400;">Error rate may still be correlated with utilization, or it may start to do wacky things. That depends on the specifics of our application. In a latency-optimized cluster, saturation is always pathological, so it shouldn&#8217;t be surprising to see error rates climb or spike when saturation rises above zero.</span></p>
<p><span style="font-weight:400;">Finally, we start to see consistent upward trends in latency. The higher percentiles are affected first and most dramatically. Then, as saturation rises even higher, we can see the median rise too.</span></p>
<h2><span style="font-weight:400;">Throughput-optimized cluster</span></h2>
<p><span style="font-weight:400;">The behavior of our throughput-optimized cluster, on the other hand, is pretty different. When it&#8217;s healthy, it looks like this:</span></p>
<p><span style="font-weight:400;"><img data-attachment-id="2297" data-permalink="https://blog.danslimmon.com/2019/03/25/latency-and-throughput-optimized-clusters-under-load/2019-03-22-blog-charts-throughput-oriented-healthy/" data-orig-file="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-22-blog-charts-throughput-oriented-healthy.png" data-orig-size="800,631" data-comments-opened="1" data-image-meta="{&quot;aperture&quot;:&quot;0&quot;,&quot;credit&quot;:&quot;&quot;,&quot;camera&quot;:&quot;&quot;,&quot;caption&quot;:&quot;&quot;,&quot;created_timestamp&quot;:&quot;0&quot;,&quot;copyright&quot;:&quot;&quot;,&quot;focal_length&quot;:&quot;0&quot;,&quot;iso&quot;:&quot;0&quot;,&quot;shutter_speed&quot;:&quot;0&quot;,&quot;title&quot;:&quot;&quot;,&quot;orientation&quot;:&quot;0&quot;}" data-image-title="2019-03-22 blog charts throughput-oriented healthy" data-image-description="" data-image-caption="" data-large-file="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-22-blog-charts-throughput-oriented-healthy.png?w=640" class="alignnone size-full wp-image-2297" src="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-22-blog-charts-throughput-oriented-healthy.png?w=640" alt="2019-03-22 blog charts throughput-oriented healthy.png" srcset="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-22-blog-charts-throughput-oriented-healthy.png 800w, https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-22-blog-charts-throughput-oriented-healthy.png?w=150&amp;h=118 150w, https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-22-blog-charts-throughput-oriented-healthy.png?w=300&amp;h=237 300w, https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-22-blog-charts-throughput-oriented-healthy.png?w=768&amp;h=606 768w" sizes="(max-width: 800px) 100vw, 800px" /></span></p>
<p><span style="font-weight:400;">Utilization – which, remember, we&#8217;re measuring via CPU usage – is no longer a fluffy cloud. Instead, it&#8217;s a series of trapezoids. When no job is in progress, utilization is at zero. When a job starts, utilization climbs up to 100% (or as close to 100% as reality allows it to get), and then stays there until the job is almost done. Eventually, the number of active tasks drops below the number of available processors, and utilization trails off back to zero.</span></p>
<p><span style="font-weight:400;">Saturation (the number of queued tasks) follows more of a sawtooth pattern. A client puts a ton of jobs into the system, bringing utilization up to its plateau, and then as we grind through jobs, we see saturation slowly decline. When saturation reaches zero, utilization starts to drop.</span></p>
<p><span style="font-weight:400;">Unlike that of a latency-optimized cluster, the error rate in a throughput-optimized cluster </span><i><span style="font-weight:400;">shouldn&#8217;t</span></i><span style="font-weight:400;"> be sensitive to saturation. Nonzero saturation is an expected and desired condition here. Error rate is, however, expected to be follow utilization. If it bumps around at all, it should plateau, not sawtooth.</span></p>
<p><span style="font-weight:400;">And finally, in the spot where the other cluster&#8217;s dashboard had a latency graph, we now have throughput. This we should measure in requests per second </span><i><span style="font-weight:400;">per job queued!</span></i><span style="font-weight:400;"> What our customers really care about is not how many requests per second our cluster is processing, but how many of </span><i><span style="font-weight:400;">their</span></i><span style="font-weight:400;"> requests per second. We&#8217;ll see why this matters so much in a bit, when we talk about this cluster&#8217;s behavior under excessive load.</span></p>
<p><span style="font-weight:400;">Throughput should be tightly correlated to utilization and nothing else. If it also seems to exhibit a negative correlation to saturation, then that&#8217;s worth looking into: it could mean that queue management is inappropriately coupled to job processing.</span></p>
<p><span style="font-weight:400;">Now what if our throughput-optimized cluster starts to get too loaded? What does that look like?</span></p>
<p><img data-attachment-id="2298" data-permalink="https://blog.danslimmon.com/2019/03/25/latency-and-throughput-optimized-clusters-under-load/2019-03-23-blog-charts-throughput-oriented-loaded/" data-orig-file="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-23-blog-charts-throughput-oriented-loaded.png" data-orig-size="800,631" data-comments-opened="1" data-image-meta="{&quot;aperture&quot;:&quot;0&quot;,&quot;credit&quot;:&quot;&quot;,&quot;camera&quot;:&quot;&quot;,&quot;caption&quot;:&quot;&quot;,&quot;created_timestamp&quot;:&quot;0&quot;,&quot;copyright&quot;:&quot;&quot;,&quot;focal_length&quot;:&quot;0&quot;,&quot;iso&quot;:&quot;0&quot;,&quot;shutter_speed&quot;:&quot;0&quot;,&quot;title&quot;:&quot;&quot;,&quot;orientation&quot;:&quot;0&quot;}" data-image-title="2019-03-23 blog charts throughput-oriented loaded" data-image-description="" data-image-caption="" data-large-file="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-23-blog-charts-throughput-oriented-loaded.png?w=640" loading="lazy" class="alignnone size-full wp-image-2298" src="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-23-blog-charts-throughput-oriented-loaded.png?w=640" alt="2019-03-23 blog charts throughput-oriented loaded" srcset="https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-23-blog-charts-throughput-oriented-loaded.png 800w, https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-23-blog-charts-throughput-oriented-loaded.png?w=150&amp;h=118 150w, https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-23-blog-charts-throughput-oriented-loaded.png?w=300&amp;h=237 300w, https://blog.danslimmon.com/wp-content/uploads/2019/03/2019-03-23-blog-charts-throughput-oriented-loaded.png?w=768&amp;h=606 768w" sizes="(max-width: 800px) 100vw, 800px" /></p>
<p><span style="font-weight:400;">Utilization by itself doesn&#8217;t tell us anything about the cluster&#8217;s health. Qualitatively, it looks just like before: big wide trapezoids. Sure, there a bit wider than they were before, but we probably won&#8217;t notice that – especially since it&#8217;s only on average that they&#8217;re wider.</span></p>
<p><span style="font-weight:400;">Saturation is where we really start to see the cracks. Instead of the sawtooth pattern we had before, we start to get more sawteeth-upon-sawteeth. These are jobs starting up on top of jobs that are already running. A few of these are to be expected even under healthy circumstances, but if they become more frequent, it&#8217;s an indication that there may be too much work for the cluster to handle without violating throughput SLOs.</span></p>
<p><span style="font-weight:400;">Error rate may not budge much in the face of higher load. After all, this cluster is </span><i><span style="font-weight:400;">supposed</span></i><span style="font-weight:400;"> to hold on to big queues of requests. As far as it&#8217;s concerned, nothing is wrong with that.</span></p>
<p><span style="font-weight:400;">And that&#8217;s why we need to measure throughput in requests per second per job queued. If all we looked at was requests per second, everything would look hunky dory. But when there are two jobs queued and they&#8217;re constrained to use the same pool of processors, somebody&#8217;s throughput is going to suffer. On this throughput graph, we can see that happen right before our eyes.</span></p>
<h2><span style="font-weight:400;">Longer-term metrics</span></h2>
<p><span style="font-weight:400;">Already, we&#8217;ve seen how decoupling these two clusters gives us a much clearer mental model of their expected behavior. As we get used to the relationships between the metrics we&#8217;ve surfaced, we&#8217;ll start to build intuition. That&#8217;s huge.</span></p>
<p><span style="font-weight:400;">But we don&#8217;t have to stop there! Having a theory (which is what we built above) also allows us to reason about long-term changes in the observable properties of these clusters as their workloads shift.</span></p>
<p><span style="font-weight:400;">Take the throughput-optimized cluster, for instance. As our customers place more and more demand on it, we expect to see more sawteeth-upon sawtooth and longer intervals between periods of zero saturation. This latter observation is key, since </span><span style="color:#0000ff;"><a style="color:#0000ff;" href="https://blog.danslimmon.com/2016/08/26/the-most-important-thing-to-understand-about-queues/"><span style="font-weight:400;">wait times grow asymptotically as utilization approaches 100%</span></a></span><span style="font-weight:400;">. So, if we&#8217;re going to want to do evaluate the capacity needs of our throughput-optimized cluster, we should start producing these metrics on day one and put them on a dashboard:</span></p>
<ul>
<li><b>Number of jobs started when another job was already running</b><span style="font-weight:400;">, aggregated by day. This is the &#8220;sawteeth-upon-sawteeth&#8221; number.</span></li>
<li><b>Proportion of time spent with zero saturation<span style="font-weight:400;">, aggregated by day.</span></b></li>
<li><b>Median (or 90th percentile) time-to-first-execution for jobs<span style="font-weight:400;">, aggregated by day. By this I mean: for each customer job, how much time passed between the first request being enqueued and the first request starting to be processed. This, especially compared with the previous metric, will show us how much our system is allowing jobs to interfere with one another.</span></b></li>
</ul>
<p>An analogous thought process will yield useful capacity evaluation metrics for the latency-optimized cluster.</p>
<h2><span style="font-weight:400;">TL;DR</span></h2>
<p><img data-attachment-id="2299" data-permalink="https://blog.danslimmon.com/2019/03/25/latency-and-throughput-optimized-clusters-under-load/unsteadyblueaardvark-size_restricted/" data-orig-file="https://blog.danslimmon.com/wp-content/uploads/2019/03/unsteadyblueaardvark-size_restricted.gif" data-orig-size="500,291" data-comments-opened="1" data-image-meta="{&quot;aperture&quot;:&quot;0&quot;,&quot;credit&quot;:&quot;&quot;,&quot;camera&quot;:&quot;&quot;,&quot;caption&quot;:&quot;&quot;,&quot;created_timestamp&quot;:&quot;0&quot;,&quot;copyright&quot;:&quot;&quot;,&quot;focal_length&quot;:&quot;0&quot;,&quot;iso&quot;:&quot;0&quot;,&quot;shutter_speed&quot;:&quot;0&quot;,&quot;title&quot;:&quot;&quot;,&quot;orientation&quot;:&quot;0&quot;}" data-image-title="UnsteadyBlueAardvark-size_restricted" data-image-description="" data-image-caption="" data-large-file="https://blog.danslimmon.com/wp-content/uploads/2019/03/unsteadyblueaardvark-size_restricted.gif?w=500" loading="lazy" class=" wp-image-2299 alignright" src="https://blog.danslimmon.com/wp-content/uploads/2019/03/unsteadyblueaardvark-size_restricted.gif?w=402&#038;h=234" alt="UnsteadyBlueAardvark-size_restricted" width="402" height="234" srcset="https://blog.danslimmon.com/wp-content/uploads/2019/03/unsteadyblueaardvark-size_restricted.gif?w=402&amp;h=234 402w, https://blog.danslimmon.com/wp-content/uploads/2019/03/unsteadyblueaardvark-size_restricted.gif?w=150&amp;h=87 150w, https://blog.danslimmon.com/wp-content/uploads/2019/03/unsteadyblueaardvark-size_restricted.gif?w=300&amp;h=175 300w, https://blog.danslimmon.com/wp-content/uploads/2019/03/unsteadyblueaardvark-size_restricted.gif 500w" sizes="auto, (max-width: 402px) 100vw, 402px" /></p>
<p><span style="font-weight:400;">Separating latency- and throughput-optimized workloads doesn&#8217;t just make it easier to optimize each. It carries the added benefit of making it easier to develop a theory about our system&#8217;s behavior. And when you have a theory that&#8217;s consistent with the signal and a signal that&#8217;s interpretable within your theory, you have observability.</span></p>
<div id="jp-post-flair" class="sharedaddy sd-like-enabled sd-sharing-enabled"><div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-twitter"><a rel="nofollow noopener noreferrer"
data-shared="sharing-twitter-2294"
class="share-twitter sd-button share-icon"
href="https://blog.danslimmon.com/2019/03/25/latency-and-throughput-optimized-clusters-under-load/?share=twitter"
target="_blank"
aria-labelledby="sharing-twitter-2294"
>
<span id="sharing-twitter-2294" hidden>Share on X (Opens in new window)</span>
<span>X</span>
</a></li><li class="share-facebook"><a rel="nofollow noopener noreferrer"
data-shared="sharing-facebook-2294"
class="share-facebook sd-button share-icon"
href="https://blog.danslimmon.com/2019/03/25/latency-and-throughput-optimized-clusters-under-load/?share=facebook"
target="_blank"
aria-labelledby="sharing-facebook-2294"
>
<span id="sharing-facebook-2294" hidden>Share on Facebook (Opens in new window)</span>
<span>Facebook</span>
</a></li><li class="share-end"></li></ul></div></div></div><div class='sharedaddy sd-block sd-like jetpack-likes-widget-wrapper jetpack-likes-widget-unloaded' id='like-post-wrapper-38620006-2294-6aa4e26e576e3' data-src='//widgets.wp.com/likes/index.html?ver=20260912#blog_id=38620006&amp;post_id=2294&amp;origin=danslimmon.wordpress.com&amp;obj_id=38620006-2294-6aa4e26e576e3&amp;domain=blog.danslimmon.com' data-name='like-post-frame-38620006-2294-6aa4e26e576e3' data-title='Like or Reblog'><div class='likes-widget-placeholder post-likes-widget-placeholder' style='height: 55px;'><span class='button'><span>Like</span></span> <span class='loading'>Loading...</span></div><span class='sd-text-color'></span><a class='sd-link-color'></a></div></div> </div><!-- .entry-content -->
<footer class="entry-footer">
</footer><!-- .entry-footer -->
</div>
</article><!-- #post-## -->
<nav class="navigation post-navigation" aria-label="Posts">
<h2 class="screen-reader-text">Post navigation</h2>
<div class="nav-links"><div class="nav-previous"><a href="https://blog.danslimmon.com/2019/02/26/the-latency-throughput-tradeoff-why-fast-services-are-slow-and-vice-versa/" rel="prev"><span class="meta-nav">Previous</span> <h4>The Latency/Throughput Tradeoff: Why Fast Services Are Slow And Vice&nbsp;Versa</h4></a></div><div class="nav-next"><a href="https://blog.danslimmon.com/2019/03/28/the-paradox-of-the-bloated-backlog/" rel="next"><span class="meta-nav">Next</span> <h4>The paradox of the bloated&nbsp;backlog</h4></a></div></div>
</nav>
<div id="comments" class="comments-area">
<div class="comments-wrapper">
<h2 class="comments-title">
3 thoughts on &ldquo;<span>Latency- and Throughput-Optimized Clusters Under&nbsp;Load</span>&rdquo; </h2>
<ol class="comment-list">
<li class="pingback even thread-even depth-1" id="comment-891">
<p>Pingback: <a href="https://blog.danslimmon.com/2019/02/26/the-latency-throughput-tradeoff-why-fast-services-are-slow-and-vice-versa/" class="url" rel="ugc">The Latency/Throughput Tradeoff: Why Fast Services Are Slow And Vice Versa &#8211; Dan Slimmon</a> </p>
</li><!-- #comment-## -->
<li class="pingback odd alt thread-odd thread-alt depth-1" id="comment-5727">
<p>Pingback: <a href="https://sreweekly.com/sre-weekly-issue-336/" class="url" rel="ugc external nofollow">SRE Weekly Issue #336 &#8211; SRE WEEKLY</a> </p>
</li><!-- #comment-## -->
<li class="pingback even thread-even depth-1" id="comment-5728">
<p>Pingback: <a href="https://fde.cat/index.php/2022/08/29/sre-weekly-issue-336/" class="url" rel="ugc external nofollow">SRE Weekly Issue #336 &#8211; FDE</a> </p>
</li><!-- #comment-## -->
</ol><!-- .comment-list -->
<div id="respond" class="comment-respond">
<h3 id="reply-title" class="comment-reply-title">Leave a comment <small><a rel="nofollow" id="cancel-comment-reply-link" href="/2019/03/25/latency-and-throughput-optimized-clusters-under-load/#respond" style="display:none;">Cancel reply</a></small></h3><form action="https://blog.danslimmon.com/wp-comments-post.php" method="post" id="commentform" class="comment-form">
<div class="comment-form__verbum transparent"></div><div class="verbum-form-meta"><input type='hidden' name='comment_post_ID' value='2294' id='comment_post_ID' />
<input type='hidden' name='comment_parent' id='comment_parent' value='0' />
<input type="hidden" name="highlander_comment_nonce" id="highlander_comment_nonce" value="4affefa77f" />
<input type="hidden" name="verbum_show_subscription_modal" value="hidden_disabled" /></div><p style="display: none;"><input type="hidden" id="akismet_comment_nonce" name="akismet_comment_nonce" value="7918064b51" /></p><p style="display: none !important;" class="akismet-fields-container" data-prefix="ak_"><label>&#916;<textarea name="ak_hp_textarea" cols="45" rows="8" maxlength="100"></textarea></label><input type="hidden" id="ak_js_1" name="ak_js" value="183"/><script>
document.getElementById( "ak_js_1" ).setAttribute( "value", ( new Date() ).getTime() );
</script>
</p></form> </div><!-- #respond -->
</div>
</div><!-- #comments -->
</main><!-- #main -->
</div><!-- #primary -->
<div id="secondary" class="widget-area" role="complementary">
<aside id="search-2" class="widget widget_search"><form role="search" method="get" class="search-form" action="https://blog.danslimmon.com/">
<label>
<span class="screen-reader-text">Search for:</span>
<input type="search" class="search-field" placeholder="Search …" value="" name="s" title="Search for:" />
</label>
<button type="submit" class="search-submit">
<span class="genericon genericon-search">
<span class="screen-reader-text">Search</span>
</span>
</button>
</form></aside><aside id="top-posts-2" class="widget widget_top-posts"><h1 class="widget-title">Top Posts</h1><ul><li><a href="https://blog.danslimmon.com/2019/07/15/do-nothing-scripting-the-key-to-gradual-automation/" class="bump-view" data-bump-view="tp">Do-nothing scripting: the key to gradual automation</a></li><li><a href="https://blog.danslimmon.com/2022/06/07/using-littles-law-to-scale-applications/" class="bump-view" data-bump-view="tp">Using Little&#039;s Law to scale applications</a></li><li><a href="https://blog.danslimmon.com/2019/06/24/an-incident-command-training-handbook/" class="bump-view" data-bump-view="tp">An Incident Command Training Handbook</a></li><li><a href="https://blog.danslimmon.com/2024/07/03/no-observability-without-theory-the-talk/" class="bump-view" data-bump-view="tp">No Observability Without Theory: The Talk</a></li><li><a href="https://blog.danslimmon.com/2016/03/10/start-a-paper-club/" class="bump-view" data-bump-view="tp">Start a Paper Club!</a></li></ul></aside><aside id="archives-2" class="widget widget_archive"><h1 class="widget-title">Archives</h1>
<ul>
<li><a href='https://blog.danslimmon.com/2025/06/'>June 2025</a></li>
<li><a href='https://blog.danslimmon.com/2025/03/'>March 2025</a></li>
<li><a href='https://blog.danslimmon.com/2025/02/'>February 2025</a></li>
<li><a href='https://blog.danslimmon.com/2025/01/'>January 2025</a></li>
<li><a href='https://blog.danslimmon.com/2024/08/'>August 2024</a></li>
<li><a href='https://blog.danslimmon.com/2024/07/'>July 2024</a></li>
<li><a href='https://blog.danslimmon.com/2024/06/'>June 2024</a></li>
<li><a href='https://blog.danslimmon.com/2024/05/'>May 2024</a></li>
<li><a href='https://blog.danslimmon.com/2024/04/'>April 2024</a></li>
<li><a href='https://blog.danslimmon.com/2024/03/'>March 2024</a></li>
<li><a href='https://blog.danslimmon.com/2024/02/'>February 2024</a></li>
<li><a href='https://blog.danslimmon.com/2024/01/'>January 2024</a></li>
<li><a href='https://blog.danslimmon.com/2023/12/'>December 2023</a></li>
<li><a href='https://blog.danslimmon.com/2023/08/'>August 2023</a></li>
<li><a href='https://blog.danslimmon.com/2023/05/'>May 2023</a></li>
<li><a href='https://blog.danslimmon.com/2023/04/'>April 2023</a></li>
<li><a href='https://blog.danslimmon.com/2023/03/'>March 2023</a></li>
<li><a href='https://blog.danslimmon.com/2023/02/'>February 2023</a></li>
<li><a href='https://blog.danslimmon.com/2022/12/'>December 2022</a></li>
<li><a href='https://blog.danslimmon.com/2022/11/'>November 2022</a></li>
<li><a href='https://blog.danslimmon.com/2022/10/'>October 2022</a></li>
<li><a href='https://blog.danslimmon.com/2022/07/'>July 2022</a></li>
<li><a href='https://blog.danslimmon.com/2022/06/'>June 2022</a></li>
<li><a href='https://blog.danslimmon.com/2022/05/'>May 2022</a></li>
<li><a href='https://blog.danslimmon.com/2022/02/'>February 2022</a></li>
<li><a href='https://blog.danslimmon.com/2021/12/'>December 2021</a></li>
<li><a href='https://blog.danslimmon.com/2021/03/'>March 2021</a></li>
<li><a href='https://blog.danslimmon.com/2019/08/'>August 2019</a></li>
<li><a href='https://blog.danslimmon.com/2019/07/'>July 2019</a></li>
<li><a href='https://blog.danslimmon.com/2019/06/'>June 2019</a></li>
<li><a href='https://blog.danslimmon.com/2019/05/'>May 2019</a></li>
<li><a href='https://blog.danslimmon.com/2019/03/'>March 2019</a></li>
<li><a href='https://blog.danslimmon.com/2019/02/'>February 2019</a></li>
<li><a href='https://blog.danslimmon.com/2017/10/'>October 2017</a></li>
<li><a href='https://blog.danslimmon.com/2017/07/'>July 2017</a></li>
<li><a href='https://blog.danslimmon.com/2017/06/'>June 2017</a></li>
<li><a href='https://blog.danslimmon.com/2017/04/'>April 2017</a></li>
<li><a href='https://blog.danslimmon.com/2016/09/'>September 2016</a></li>
<li><a href='https://blog.danslimmon.com/2016/08/'>August 2016</a></li>
<li><a href='https://blog.danslimmon.com/2016/06/'>June 2016</a></li>
<li><a href='https://blog.danslimmon.com/2016/03/'>March 2016</a></li>
<li><a href='https://blog.danslimmon.com/2015/12/'>December 2015</a></li>
<li><a href='https://blog.danslimmon.com/2015/10/'>October 2015</a></li>
<li><a href='https://blog.danslimmon.com/2015/07/'>July 2015</a></li>
<li><a href='https://blog.danslimmon.com/2015/06/'>June 2015</a></li>
<li><a href='https://blog.danslimmon.com/2014/09/'>September 2014</a></li>
<li><a href='https://blog.danslimmon.com/2014/08/'>August 2014</a></li>
<li><a href='https://blog.danslimmon.com/2014/07/'>July 2014</a></li>
<li><a href='https://blog.danslimmon.com/2014/05/'>May 2014</a></li>
<li><a href='https://blog.danslimmon.com/2013/10/'>October 2013</a></li>
<li><a href='https://blog.danslimmon.com/2013/09/'>September 2013</a></li>
<li><a href='https://blog.danslimmon.com/2013/06/'>June 2013</a></li>
<li><a href='https://blog.danslimmon.com/2013/05/'>May 2013</a></li>
<li><a href='https://blog.danslimmon.com/2013/04/'>April 2013</a></li>
<li><a href='https://blog.danslimmon.com/2013/03/'>March 2013</a></li>
<li><a href='https://blog.danslimmon.com/2012/12/'>December 2012</a></li>
<li><a href='https://blog.danslimmon.com/2012/11/'>November 2012</a></li>
<li><a href='https://blog.danslimmon.com/2012/08/'>August 2012</a></li>
<li><a href='https://blog.danslimmon.com/2012/07/'>July 2012</a></li>
</ul>
</aside><aside id="categories-2" class="widget widget_categories"><h1 class="widget-title">Categories</h1>
<ul>
<li class="cat-item cat-item-756099141"><a href="https://blog.danslimmon.com/category/descriptive-engineering/">Descriptive engineering</a>
</li>
<li class="cat-item cat-item-2360537"><a href="https://blog.danslimmon.com/category/incident-response/">incident-response</a>
</li>
<li class="cat-item cat-item-360969"><a href="https://blog.danslimmon.com/category/mind-hacks/">Mind hacks</a>
</li>
<li class="cat-item cat-item-111949"><a href="https://blog.danslimmon.com/category/monitoring/">Monitoring</a>
</li>
<li class="cat-item cat-item-306573"><a href="https://blog.danslimmon.com/category/post-mortems/">Post-mortems</a>
</li>
<li class="cat-item cat-item-434917"><a href="https://blog.danslimmon.com/category/postgres/">Postgres</a>
</li>
<li class="cat-item cat-item-133044"><a href="https://blog.danslimmon.com/category/problem-solving/">Problem-Solving</a>
</li>
<li class="cat-item cat-item-943791"><a href="https://blog.danslimmon.com/category/statistics-and-probability/">Statistics and probability</a>
</li>
<li class="cat-item cat-item-1"><a href="https://blog.danslimmon.com/category/uncategorized/">Uncategorized</a>
</li>
<li class="cat-item cat-item-18187"><a href="https://blog.danslimmon.com/category/workflow/">Workflow</a>
</li>
</ul>
</aside><aside id="meta-2" class="widget widget_meta"><h1 class="widget-title">Meta</h1>
<ul>
<li><a class="click-register" href="https://wordpress.com/start?ref=wplogin">Create account</a></li> <li><a href="https://danslimmon.wordpress.com/wp-login.php">Log in</a></li>
<li><a href="https://blog.danslimmon.com/feed/">Entries feed</a></li>
<li><a href="https://blog.danslimmon.com/comments/feed/">Comments feed</a></li>
<li><a href="https://wordpress.com/" title="Powered by WordPress, state-of-the-art semantic personal publishing platform.">WordPress.com</a></li>
</ul>
</aside></div><!-- #secondary -->
</div><!-- #content -->
<footer id="colophon" class="site-footer" role="contentinfo">
<div class="site-info">
<a href="https://wordpress.com/?ref=footer_blog" rel="nofollow">Blog at WordPress.com.</a>
</div><!-- .site-info -->
</footer><!-- #colophon -->
</div><!-- #page -->
<!-- -->
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/files/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/pub/lovecraft/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<div class="jetpack-subscribe-modal">
<div class="jetpack-subscribe-modal__modal-content">
<div class="wp-block-group has-border-color" style="border-color:#dddddd;border-width:1px;margin-top:0;margin-bottom:0;padding-top:32px;padding-right:32px;padding-bottom:32px;padding-left:32px"><div class="wp-block-group__inner-container is-layout-constrained wp-container-core-group-is-layout-1f4dd145 wp-block-group-is-layout-constrained">
<h2 class="wp-block-heading has-text-align-center" style="margin-top:4px;margin-bottom:10px;font-size:26px;font-style:normal;font-weight:600">Discover more from Dan Slimmon</h2>
<p class="has-text-align-center wp-block-paragraph" style='margin-top:4px;margin-bottom:1em;font-size:15px'>Subscribe now to keep reading and get access to the full archive.</p>
<div class="wp-block-jetpack-subscriptions__supports-newline is-style-compact wp-block-jetpack-subscriptions">
<div class="wp-block-jetpack-subscriptions__container is-not-subscriber">
<form
action="https://wordpress.com/email-subscriptions"
method="post"
accept-charset="utf-8"
data-blog="38620006"
data-post_access_level="everybody"
data-subscriber_email=""
id="subscribe-blog"
>
<div class="wp-block-jetpack-subscriptions__form-elements">
<p id="subscribe-email">
<label
id="subscribe-field-label"
for="subscribe-field"
class="screen-reader-text"
>
Type your email… </label>
<input
required="required"
type="email"
name="email"
autocomplete="email"
style="font-size: 16px;padding: 15px 23px 15px 23px;border-radius: 50px;border-width: 1px;"
placeholder="Type your email…"
value=""
id="subscribe-field"
title="Please fill in this field."
/> </p>
<p id="subscribe-submit"
>
<input type="hidden" name="action" value="subscribe"/>
<input type="hidden" name="blog_id" value="38620006"/>
<input type="hidden" name="source" value="https://blog.danslimmon.com/2019/03/25/latency-and-throughput-optimized-clusters-under-load/"/>
<input type="hidden" name="sub-type" value="subscribe-block"/>
<input type="hidden" name="app_source" value="subscribe-modal"/>
<input type="hidden" name="redirect_fragment" value="subscribe-blog"/>
<input type="hidden" name="lang" value="en"/>
<input type="hidden" id="_wpnonce" name="_wpnonce" value="af441365a6" /><input type="hidden" name="_wp_http_referer" value="/2019/03/25/latency-and-throughput-optimized-clusters-under-load/" /><input type="hidden" name="post_id" value="2294"/> <button type="submit"
class="wp-block-button__link"
style="font-size: 16px;padding: 15px 23px 15px 23px;margin: 0; margin-left: 10px;border-radius: 50px;border-width: 1px;"
name="jetpack_subscriptions_widget"
>
Subscribe </button>
</p>
</div>
</form>
</div>
</div>
<p class="has-text-align-center jetpack-subscribe-modal__close wp-block-paragraph" style="margin-top:20px;margin-bottom:0;font-size:14px"><a href="#">Continue reading</a></p>
</div></div>
</div>
</div>
<script id="grofiles-cards-js" src="//0.gravatar.com/js/hovercards/hovercards.min.js?ver=202637d1133547e8a1f2801609f51cef12522abe7f6647d869e9fbc974fe356a37e613"></script>
<script id="wpgroho-js-extra">
var WPGroHo = {"my_hash":""};
//# sourceURL=wpgroho-js-extra
</script>
<script crossorigin='anonymous' type='text/javascript' src='/wp-content/mu-plugins/gravatar-hovercards/wpgroho.js?m=1610363240i'></script>
<script>
// Initialize and attach hovercards to all gravatars
( function() {
function init() {
if ( typeof Gravatar === 'undefined' ) {
return;
}
if ( typeof Gravatar.init !== 'function' ) {
return;
}
Gravatar.profile_cb = function ( hash, id ) {
WPGroHo.syncProfileData( hash, id );
};
Gravatar.my_hash = WPGroHo.my_hash;
Gravatar.init(
'body',
'#wp-admin-bar-my-account',
{
i18n: {
'Edit your profile →': 'Edit your profile →',
'View profile →': 'View profile →',
'Contact': 'Contact',
'Send money': 'Send money',
'Sorry, we are unable to load this Gravatar profile.': 'Sorry, we are unable to load this Gravatar profile.',
'Gravatar not found.': 'Gravatar not found.',
'Too Many Requests.': 'Too Many Requests.',
'Internal Server Error.': 'Internal Server Error.',
'Is this you?': 'Is this you?',
'Claim your free profile.': 'Claim your free profile.',
'Email': 'Email',
'Home Phone': 'Home Phone',
'Work Phone': 'Work Phone',
'Cell Phone': 'Cell Phone',
'Contact Form': 'Contact Form',
'Calendar': 'Calendar',
},
}
);
}
if ( document.readyState !== 'loading' ) {
init();
} else {
document.addEventListener( 'DOMContentLoaded', init );
}
} )();
</script>
<div style="display:none">
</div>
<div id="actionbar" dir="ltr" style="display: none;"
class="actnbr-pub-lovecraft actnbr-has-follow actnbr-has-actions">
<ul>
<li class="actnbr-btn actnbr-hidden">
<a class="actnbr-action actnbr-actn-comment" href="https://blog.danslimmon.com/2019/03/25/latency-and-throughput-optimized-clusters-under-load/#comments">
<svg class="gridicon gridicons-comment" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M12 16l-5 5v-5H5c-1.1 0-2-.9-2-2V5c0-1.1.9-2 2-2h14c1.1 0 2 .9 2 2v9c0 1.1-.9 2-2 2h-7z"/></g></svg> <span>Comment </span>
</a>
</li>
<li class="actnbr-btn actnbr-hidden">
<a class="actnbr-action actnbr-actn-reblog" href="">
<svg class="gridicon gridicons-reblog" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M22.086 9.914L20 7.828V18c0 1.105-.895 2-2 2h-7v-2h7V7.828l-2.086 2.086L14.5 8.5 19 4l4.5 4.5-1.414 1.414zM6 16.172V6h7V4H6c-1.105 0-2 .895-2 2v10.172l-2.086-2.086L.5 15.5 5 20l4.5-4.5-1.414-1.414L6 16.172z"/></g></svg><span>Reblog</span>
</a>
</li>
<li class="actnbr-btn actnbr-hidden">
<a class="actnbr-action actnbr-actn-follow " href="">
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path clip-rule="evenodd" d="m4 4.5h12v6.5h1.5v-6.5-1.5h-1.5-12-1.5v1.5 10.5c0 1.1046.89543 2 2 2h7v-1.5h-7c-.27614 0-.5-.2239-.5-.5zm10.5 2h-9v1.5h9zm-5 3h-4v1.5h4zm3.5 1.5h-1v1h1zm-1-1.5h-1.5v1.5 1 1.5h1.5 1 1.5v-1.5-1-1.5h-1.5zm-2.5 2.5h-4v1.5h4zm6.5 1.25h1.5v2.25h2.25v1.5h-2.25v2.25h-1.5v-2.25h-2.25v-1.5h2.25z" fill-rule="evenodd"></path></svg>
<span>Subscribe</span>
</a>
<a class="actnbr-action actnbr-actn-following no-display" href="">
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M16 4.5H4V15C4 15.2761 4.22386 15.5 4.5 15.5H11.5V17H4.5C3.39543 17 2.5 16.1046 2.5 15V3H17.5V12.5H16V4.5ZM5.5 6.5H14.5V8H5.5V6.5ZM5.5 9.5H9.5V11H5.5V9.5ZM12 11H13V12H12V11ZM10.5 9.5H14.5V13.5H10.5V9.5ZM5.5 12H9.5V13.5H5.5V12Z" fill="#008A20"></path><path d="M19.5303 15.0303L15.5 19.0605L12.9697 16.5303L14.0303 15.4697L15.5 16.9395L18.4697 13.9697L19.5303 15.0303Z" fill="#008A20"></path></svg>
<span>Subscribed</span>
</a>
<div class="actnbr-popover tip tip-top-left actnbr-notice" id="follow-bubble">
<div class="tip-arrow"></div>
<div class="tip-inner actnbr-follow-bubble">
<ul>
<li class="actnbr-sitename">
<a href="https://blog.danslimmon.com">
<img loading='lazy' alt='' src='https://blog.danslimmon.com/wp-content/uploads/2024/03/cropped-d2e_pip_transparent.png?w=50' srcset='https://blog.danslimmon.com/wp-content/uploads/2024/03/cropped-d2e_pip_transparent.png?w=50 1x, https://blog.danslimmon.com/wp-content/uploads/2024/03/cropped-d2e_pip_transparent.png?w=75 1.5x, https://blog.danslimmon.com/wp-content/uploads/2024/03/cropped-d2e_pip_transparent.png?w=100 2x, https://blog.danslimmon.com/wp-content/uploads/2024/03/cropped-d2e_pip_transparent.png?w=150 3x, https://blog.danslimmon.com/wp-content/uploads/2024/03/cropped-d2e_pip_transparent.png?w=200 4x' class='avatar avatar-50' height='50' width='50' /> Dan Slimmon </a>
</li>
<div class="actnbr-message no-display"></div>
<form method="post" action="https://subscribe.wordpress.com" accept-charset="utf-8" style="display: none;">
<div class="actnbr-follow-count">Join 147 other subscribers</div>
<div>
<input type="email" name="email" placeholder="Enter your email address" class="actnbr-email-field" aria-label="Enter your email address" />
</div>
<input type="hidden" name="action" value="subscribe" />
<input type="hidden" name="blog_id" value="38620006" />
<input type="hidden" name="source" value="https://blog.danslimmon.com/2019/03/25/latency-and-throughput-optimized-clusters-under-load/" />
<input type="hidden" name="sub-type" value="actionbar-follow" />
<input type="hidden" id="_wpnonce" name="_wpnonce" value="af441365a6" /> <div class="actnbr-button-wrap">
<button type="submit" value="Sign me up">
Sign me up </button>
</div>
</form>
<li class="actnbr-login-nudge">
<div>
Already have a WordPress.com account? <a href="https://wordpress.com/log-in?redirect_to=https%3A%2F%2Fr-login.wordpress.com%2Fremote-login.php%3Faction%3Dlink%26back%3Dhttps%253A%252F%252Fblog.danslimmon.com%252F2019%252F03%252F25%252Flatency-and-throughput-optimized-clusters-under-load%252F">Log in now.</a> </div>
</li>
</ul>
</div>
</div>
</li>
<li class="actnbr-ellipsis actnbr-hidden">
<svg class="gridicon gridicons-ellipsis" height="24" width="24" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M7 12c0 1.104-.896 2-2 2s-2-.896-2-2 .896-2 2-2 2 .896 2 2zm12-2c-1.104 0-2 .896-2 2s.896 2 2 2 2-.896 2-2-.896-2-2-2zm-7 0c-1.104 0-2 .896-2 2s.896 2 2 2 2-.896 2-2-.896-2-2-2z"/></g></svg> <div class="actnbr-popover tip tip-top-left actnbr-more">
<div class="tip-arrow"></div>
<div class="tip-inner">
<ul>
<li class="actnbr-sitename">
<a href="https://blog.danslimmon.com">
<img loading='lazy' alt='' src='https://blog.danslimmon.com/wp-content/uploads/2024/03/cropped-d2e_pip_transparent.png?w=50' srcset='https://blog.danslimmon.com/wp-content/uploads/2024/03/cropped-d2e_pip_transparent.png?w=50 1x, https://blog.danslimmon.com/wp-content/uploads/2024/03/cropped-d2e_pip_transparent.png?w=75 1.5x, https://blog.danslimmon.com/wp-content/uploads/2024/03/cropped-d2e_pip_transparent.png?w=100 2x, https://blog.danslimmon.com/wp-content/uploads/2024/03/cropped-d2e_pip_transparent.png?w=150 3x, https://blog.danslimmon.com/wp-content/uploads/2024/03/cropped-d2e_pip_transparent.png?w=200 4x' class='avatar avatar-50' height='50' width='50' /> Dan Slimmon </a>
</li>
<li class="actnbr-folded-follow">
<a class="actnbr-action actnbr-actn-follow " href="">
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path clip-rule="evenodd" d="m4 4.5h12v6.5h1.5v-6.5-1.5h-1.5-12-1.5v1.5 10.5c0 1.1046.89543 2 2 2h7v-1.5h-7c-.27614 0-.5-.2239-.5-.5zm10.5 2h-9v1.5h9zm-5 3h-4v1.5h4zm3.5 1.5h-1v1h1zm-1-1.5h-1.5v1.5 1 1.5h1.5 1 1.5v-1.5-1-1.5h-1.5zm-2.5 2.5h-4v1.5h4zm6.5 1.25h1.5v2.25h2.25v1.5h-2.25v2.25h-1.5v-2.25h-2.25v-1.5h2.25z" fill-rule="evenodd"></path></svg>
<span>Subscribe</span>
</a>
<a class="actnbr-action actnbr-actn-following no-display" href="">
<svg class="gridicon" height="20" width="20" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="none"><path fill-rule="evenodd" clip-rule="evenodd" d="M16 4.5H4V15C4 15.2761 4.22386 15.5 4.5 15.5H11.5V17H4.5C3.39543 17 2.5 16.1046 2.5 15V3H17.5V12.5H16V4.5ZM5.5 6.5H14.5V8H5.5V6.5ZM5.5 9.5H9.5V11H5.5V9.5ZM12 11H13V12H12V11ZM10.5 9.5H14.5V13.5H10.5V9.5ZM5.5 12H9.5V13.5H5.5V12Z" fill="#008A20"></path><path d="M19.5303 15.0303L15.5 19.0605L12.9697 16.5303L14.0303 15.4697L15.5 16.9395L18.4697 13.9697L19.5303 15.0303Z" fill="#008A20"></path></svg>
<span>Subscribed</span>
</a>
</li>
<li class="actnbr-signup"><a href="https://wordpress.com/start/">Sign up</a></li>
<li class="actnbr-login"><a href="https://wordpress.com/log-in?redirect_to=https%3A%2F%2Fr-login.wordpress.com%2Fremote-login.php%3Faction%3Dlink%26back%3Dhttps%253A%252F%252Fblog.danslimmon.com%252F2019%252F03%252F25%252Flatency-and-throughput-optimized-clusters-under-load%252F">Log in</a></li>
<li class="actnbr-shortlink">
<a href="https://wp.me/p2C2Pk-B0">
<span class="actnbr-shortlink__text">Copy shortlink</span>
<span class="actnbr-shortlink__icon"><svg class="gridicon gridicons-checkmark" height="16" width="16" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g><path d="M9 19.414l-6.707-6.707 1.414-1.414L9 16.586 20.293 5.293l1.414 1.414"/></g></svg></span>
</a>
</li>
<li class="flb-report">
<a href="https://wordpress.com/abuse/?report_url=https://blog.danslimmon.com/2019/03/25/latency-and-throughput-optimized-clusters-under-load/" target="_blank" rel="noopener noreferrer">
Report this content </a>
</li>
<li class="actnbr-reader">
<a href="https://wordpress.com/reader/blogs/38620006/posts/2294">
View post in Reader </a>
</li>
<li class="actnbr-subs">
<a href="https://subscribe.wordpress.com/">Manage subscriptions</a>
</li>
<li class="actnbr-fold"><a href="">Collapse this bar</a></li>
</ul>
</div>
</div>
</li>
</ul>
</div>
<script>
window.addEventListener( "DOMContentLoaded", function( event ) {
var link = document.createElement( "link" );
link.href = "/wp-content/mu-plugins/actionbar/actionbar.css?v=20250116";
link.type = "text/css";
link.rel = "stylesheet";
document.head.appendChild( link );
var script = document.createElement( "script" );
script.src = "/wp-content/mu-plugins/actionbar/actionbar.js?v=20250204";
document.body.appendChild( script );
} );
</script>
<div id="jp-carousel-loading-overlay" style="display: none;">
<div id="jp-carousel-loading-wrapper">
<span id="jp-carousel-library-loading"><svg class="jetpack-spinner" width="40" height="40" viewBox="0 0 100 100" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><circle cx="50" cy="50" r="46" fill="none" stroke="#ddd" stroke-width="8"/><path d="M 50 4 A 46 46 0 0 1 96 50" fill="none" stroke="currentColor" stroke-width="8" stroke-linecap="round"><animateTransform attributeName="transform" type="rotate" dur="1.4s" from="0 50 50" to="360 50 50" repeatCount="indefinite"/></path></svg></span>
</div>
</div>
<div class="jp-carousel-overlay" style="display: none;">
<div class="jp-carousel-container">
<!-- The Carousel Swiper -->
<div
class="jp-carousel-wrap swiper jp-carousel-swiper-container jp-carousel-transitions"
itemscope
itemtype="https://schema.org/ImageGallery">
<div class="jp-carousel swiper-wrapper"></div>
<div class="jp-swiper-button-prev swiper-button-prev">
<svg width="25" height="24" viewBox="0 0 25 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<mask id="maskPrev" mask-type="alpha" maskUnits="userSpaceOnUse" x="8" y="6" width="9" height="12">
<path d="M16.2072 16.59L11.6496 12L16.2072 7.41L14.8041 6L8.8335 12L14.8041 18L16.2072 16.59Z" fill="white"/>
</mask>
<g mask="url(#maskPrev)">
<rect x="0.579102" width="23.8823" height="24" fill="#FFFFFF"/>
</g>
</svg>
</div>
<div class="jp-swiper-button-next swiper-button-next">
<svg width="25" height="24" viewBox="0 0 25 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<mask id="maskNext" mask-type="alpha" maskUnits="userSpaceOnUse" x="8" y="6" width="8" height="12">
<path d="M8.59814 16.59L13.1557 12L8.59814 7.41L10.0012 6L15.9718 12L10.0012 18L8.59814 16.59Z" fill="white"/>
</mask>
<g mask="url(#maskNext)">
<rect x="0.34375" width="23.8822" height="24" fill="#FFFFFF"/>
</g>
</svg>
</div>
</div>
<!-- The main close buton -->
<div class="jp-carousel-close-hint">
<svg width="25" height="24" viewBox="0 0 25 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<mask id="maskClose" mask-type="alpha" maskUnits="userSpaceOnUse" x="5" y="5" width="15" height="14">
<path d="M19.3166 6.41L17.9135 5L12.3509 10.59L6.78834 5L5.38525 6.41L10.9478 12L5.38525 17.59L6.78834 19L12.3509 13.41L17.9135 19L19.3166 17.59L13.754 12L19.3166 6.41Z" fill="white"/>
</mask>
<g mask="url(#maskClose)">
<rect x="0.409668" width="23.8823" height="24" fill="#FFFFFF"/>
</g>
</svg>
</div>
<!-- Image info, comments and meta -->
<div class="jp-carousel-info">
<div class="jp-carousel-info-footer">
<div class="jp-carousel-pagination-container">
<div class="jp-swiper-pagination swiper-pagination"></div>
<div class="jp-carousel-pagination"></div>
</div>
<div class="jp-carousel-photo-title-container">
<div class="jp-carousel-photo-caption"></div>
</div>
<div class="jp-carousel-photo-icons-container">
<a href="#" class="jp-carousel-icon-btn jp-carousel-icon-info" aria-label="Toggle photo metadata visibility">
<span class="jp-carousel-icon">
<svg width="25" height="24" viewBox="0 0 25 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<mask id="maskInfo" mask-type="alpha" maskUnits="userSpaceOnUse" x="2" y="2" width="21" height="20">
<path fill-rule="evenodd" clip-rule="evenodd" d="M12.7537 2C7.26076 2 2.80273 6.48 2.80273 12C2.80273 17.52 7.26076 22 12.7537 22C18.2466 22 22.7046 17.52 22.7046 12C22.7046 6.48 18.2466 2 12.7537 2ZM11.7586 7V9H13.7488V7H11.7586ZM11.7586 11V17H13.7488V11H11.7586ZM4.79292 12C4.79292 16.41 8.36531 20 12.7537 20C17.142 20 20.7144 16.41 20.7144 12C20.7144 7.59 17.142 4 12.7537 4C8.36531 4 4.79292 7.59 4.79292 12Z" fill="white"/>
</mask>
<g mask="url(#maskInfo)">
<rect x="0.8125" width="23.8823" height="24" fill="#FFFFFF"/>
</g>
</svg>
</span>
</a>
<a href="#" class="jp-carousel-icon-btn jp-carousel-icon-comments" aria-label="Toggle photo comments visibility">
<span class="jp-carousel-icon">
<svg width="25" height="24" viewBox="0 0 25 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<mask id="maskComments" mask-type="alpha" maskUnits="userSpaceOnUse" x="2" y="2" width="21" height="20">
<path fill-rule="evenodd" clip-rule="evenodd" d="M4.3271 2H20.2486C21.3432 2 22.2388 2.9 22.2388 4V16C22.2388 17.1 21.3432 18 20.2486 18H6.31729L2.33691 22V4C2.33691 2.9 3.2325 2 4.3271 2ZM6.31729 16H20.2486V4H4.3271V18L6.31729 16Z" fill="white"/>
</mask>
<g mask="url(#maskComments)">
<rect x="0.34668" width="23.8823" height="24" fill="#FFFFFF"/>
</g>
</svg>
<span class="jp-carousel-has-comments-indicator" aria-label="This image has comments."></span>
</span>
</a>
</div>
</div>
<div class="jp-carousel-info-extra">
<div class="jp-carousel-info-content-wrapper">
<div class="jp-carousel-photo-title-container">
<div class="jp-carousel-photo-title"></div>
</div>
<div class="jp-carousel-comments-wrapper">
<div id="jp-carousel-comments-loading">
<span>Loading Comments...</span>
</div>
<div class="jp-carousel-comments"></div>
<div id="jp-carousel-comment-form-container">
<span id="jp-carousel-comment-form-spinner"><svg class="jetpack-spinner" width="20" height="20" viewBox="0 0 100 100" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><circle cx="50" cy="50" r="46" fill="none" stroke="#ddd" stroke-width="8"/><path d="M 50 4 A 46 46 0 0 1 96 50" fill="none" stroke="currentColor" stroke-width="8" stroke-linecap="round"><animateTransform attributeName="transform" type="rotate" dur="1.4s" from="0 50 50" to="360 50 50" repeatCount="indefinite"/></path></svg></span>
<div id="jp-carousel-comment-post-results"></div>
<form id="jp-carousel-comment-form">
<label for="jp-carousel-comment-form-comment-field" class="screen-reader-text">Write a Comment...</label>
<textarea
name="comment"
class="jp-carousel-comment-form-field jp-carousel-comment-form-textarea"
id="jp-carousel-comment-form-comment-field"
placeholder="Write a Comment..."
></textarea>
<div id="jp-carousel-comment-form-submit-and-info-wrapper">
<div id="jp-carousel-comment-form-commenting-as">
<fieldset>
<label for="jp-carousel-comment-form-email-field">Email (Required)</label>
<input type="text" name="email" class="jp-carousel-comment-form-field jp-carousel-comment-form-text-field" id="jp-carousel-comment-form-email-field" />
</fieldset>
<fieldset>
<label for="jp-carousel-comment-form-author-field">Name (Required)</label>
<input type="text" name="author" class="jp-carousel-comment-form-field jp-carousel-comment-form-text-field" id="jp-carousel-comment-form-author-field" />
</fieldset>
<fieldset>
<label for="jp-carousel-comment-form-url-field">Website</label>
<input type="text" name="url" class="jp-carousel-comment-form-field jp-carousel-comment-form-text-field" id="jp-carousel-comment-form-url-field" />
</fieldset>
</div>
<input
type="submit"
name="submit"
class="jp-carousel-comment-form-button"
id="jp-carousel-comment-form-button-submit"
value="Post Comment" />
</div>
</form>
</div>
</div>
<div class="jp-carousel-image-meta">
<div class="jp-carousel-title-and-caption">
<div class="jp-carousel-photo-info">
<div class="jp-carousel-caption" itemprop="caption description"></div>
</div>
<div class="jp-carousel-photo-description"></div>
</div>
<a class="jp-carousel-image-download" href="#" aria-label="Download image" target="_blank" style="display: none;">
<svg width="25" height="24" viewBox="0 0 25 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<mask id="mask0" mask-type="alpha" maskUnits="userSpaceOnUse" x="3" y="3" width="19" height="18">
<path fill-rule="evenodd" clip-rule="evenodd" d="M5.84615 5V19H19.7775V12H21.7677V19C21.7677 20.1 20.8721 21 19.7775 21H5.84615C4.74159 21 3.85596 20.1 3.85596 19V5C3.85596 3.9 4.74159 3 5.84615 3H12.8118V5H5.84615ZM14.802 5V3H21.7677V10H19.7775V6.41L9.99569 16.24L8.59261 14.83L18.3744 5H14.802Z" fill="white"/>
</mask>
<g mask="url(#mask0)">
<rect x="0.870605" width="23.8823" height="24" fill="#FFFFFF"/>
</g>
</svg>
<span class="jp-carousel-download-text"></span>
</a>
<div class="jp-carousel-image-map" style="display: none;"></div>
</div>
</div>
</div>
</div>
</div>
</div>
<script type="text/javascript">
window.WPCOM_sharing_counts = {"https://blog.danslimmon.com/2019/03/25/latency-and-throughput-optimized-clusters-under-load/":2294};
</script>
<script type="text/javascript">
(function () {
var wpcom_reblog = {
source: 'toolbar',
toggle_reblog_box_flair: function (obj_id, post_id) {
// Go to site selector. This will redirect to their blog if they only have one.
const postEndpoint = `https://wordpress.com/post`;
// Ideally we would use the permalink here, but fortunately this will be replaced with the
// post permalink in the editor.
const originalURL = `${ document.location.href }?page_id=${ post_id }`;
const url =
postEndpoint +
'?url=' +
encodeURIComponent( originalURL ) +
'&is_post_share=true' +
'&v=5';
const redirect = function () {
if (
! window.open( url, '_blank' )
) {
location.href = url;
}
};
if ( /Firefox/.test( navigator.userAgent ) ) {
setTimeout( redirect, 0 );
} else {
redirect();
}
},
};
window.wpcom_reblog = wpcom_reblog;
})();
</script>
<script crossorigin='anonymous' type='text/javascript' src='/wp-content/mu-plugins/likes/queuehandler.js?m=1741961244i'></script>
<script id="wp-dom-ready-js" src="/wp-content/plugins/gutenberg-core/v23.9.1/build/scripts/dom-ready/index.min.js?m=1788785546i&#038;ver=3fe927cab37bf38d6a23"></script>
<script id="subscribe-modal-js-js-extra">
var Jetpack_Subscriptions = {"modalLoadTime":"60000","modalScrollThreshold":"50","modalInterval":"86400000"};
//# sourceURL=subscribe-modal-js-js-extra
</script>
<script crossorigin='anonymous' type='text/javascript' src='/wp-content/mu-plugins/jetpack-plugin/moon/modules/subscriptions/subscribe-modal/subscribe-modal.js?m=1729082728i'></script>
<script id="wp-hooks-js" src="/wp-content/plugins/gutenberg-core/v23.9.1/build/scripts/hooks/index.min.js?m=1788785546i&#038;ver=f0f188028580e8dc1255"></script>
<script id="wp-i18n-js" src="/wp-content/plugins/gutenberg-core/v23.9.1/build/scripts/i18n/index.min.js?m=1788785546i&#038;ver=1dfe7db3940c23ea9216"></script>
<script id="wp-i18n-js-after">
wp.i18n.setLocaleData( { 'text direction\u0004ltr': [ 'ltr' ] } );
//# sourceURL=wp-i18n-js-after
</script>
<script id="verbum-settings-js-before">
window.VerbumComments = {"Log in or provide your name and email to leave a reply.":"Log in or provide your name and email to leave a reply.","Log in or provide your name and email to leave a comment.":"Log in or provide your name and email to leave a comment.","Receive web and mobile notifications for posts on this site.":"Receive web and mobile notifications for posts on this site.","Name":"Name","Email (address never made public)":"Email (address never made public)","Website (optional)":"Website (optional)","Leave a reply. (log in optional)":"Leave a reply. (log in optional)","Leave a comment. (log in optional)":"Leave a comment. (log in optional)","Log in to leave a reply.":"Log in to leave a reply.","Log in to leave a comment.":"Log in to leave a comment.","Logged in via %s":"Logged in via %s","Log out":"Log out","Your browser is blocking the cookies needed to log in and comment here. Allow cookies in your privacy settings, then reload the page.":"Your browser is blocking the cookies needed to log in and comment here. Allow cookies in your privacy settings, then reload the page.","Email":"Email","(Address never made public)":"(Address never made public)","Instantly":"Instantly","Daily":"Daily","Reply":"Reply","Comment":"Comment","WordPress":"WordPress","Weekly":"Weekly","Notify me of new posts":"Notify me of new posts","Email me new posts":"Email me new posts","Email me new comments":"Email me new comments","Cancel":"Cancel","Write a comment...":"Write a comment...","Write a reply...":"Write a reply...","Website":"Website","Optional":"Optional","We'll keep you in the loop!":"We'll keep you in the loop!","Loading your comment...":"Loading your comment...","Discover more from":"Discover more from Dan Slimmon","Subscribe now to keep reading and get access to the full archive.":"Subscribe now to keep reading and get access to the full archive.","Continue reading":"Continue reading","Never miss a beat!":"Never miss a beat!","Interested in getting blog post updates? Simply click the button below to stay in the loop!":"Interested in getting blog post updates? Simply click the button below to stay in the loop!","Enter your email address":"Enter your email address","Subscribe":"Subscribe","Comment sent successfully":"Comment sent successfully","Save my name, email, and website in this browser for the next time I comment.":"Save my name, email, and website in this browser for the next time I comment.","hovercardi18n":{"Edit your profile \u2192":"Edit your profile \u2192","View profile \u2192":"View profile \u2192","Contact":"Contact","Send money":"Send money","Gravatar not found.":"Gravatar not found.","This profile is private.":"This profile is private.","Too Many Requests.":"Too Many Requests.","Internal Server Error.":"Internal Server Error.","Sorry, we are unable to load this Gravatar profile.":"Sorry, we are unable to load this Gravatar profile."},"siteId":38620006,"postId":2294,"mustLogIn":false,"requireNameEmail":true,"commentRegistration":false,"connectURL":"https://danslimmon.wordpress.com/public.api/connect/?action=request\u0026domain=blog.danslimmon.com","logoutURL":"https://danslimmon.wordpress.com/wp-login.php?action=logout\u0026_wpnonce=b263a00deb","homeURL":"https://blog.danslimmon.com/","subscribeToBlog":true,"subscribeToComment":true,"isJetpackCommentsLoggedIn":false,"jetpackUsername":"","jetpackUserId":0,"jetpackSignature":"","jetpackAvatar":"https://0.gravatar.com/avatar/?s=96\u0026amp;d=identicon\u0026amp;r=G","enableBlocks":true,"enableSubscriptionModal":false,"currentLocale":"en","isJetpackComments":false,"allowedBlocks":["core/paragraph","core/list","core/code","core/list-item","core/quote","core/image","core/embed","core/quote","core/code"],"embedNonce":"2db44e3fcc","verbumBundleUrl":"/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/moon/jetpack_vendor/automattic/jetpack-mu-wpcom/src/features/verbum-comments/dist/index.js","isRTL":false,"vbeCacheBuster":1771405207,"iframeUniqueId":0,"colorScheme":false}
//# sourceURL=verbum-settings-js-before
</script>
<script data-wp-strategy="defer" defer id="verbum-dynamic-loader-js" src="/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/moon/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/verbum-comments/assets/dynamic-loader.js?m=1772518873i&#038;minify=false&#038;ver=8b5296f9195db92c42e4"></script>
<script id="jetpack-carousel-js-extra">
var jetpackSwiperLibraryPath = {"url":"/wp-content/mu-plugins/jetpack-plugin/moon/_inc/blocks/swiper.js"};
var jetpackCarouselStrings = {"widths":[370,700,1000,1200,1400,2000],"is_logged_in":"","lang":"en","ajaxurl":"https://blog.danslimmon.com/wp-admin/admin-ajax.php","nonce":"fd1855488a","display_exif":"1","display_comments":"1","single_image_gallery":"1","single_image_gallery_media_file":"","background_color":"black","comment":"Comment","post_comment":"Post Comment","write_comment":"Write a Comment...","loading_comments":"Loading Comments...","image_label":"Open image in full-screen.","download_original":"View full size \u003Cspan class=\"photo-size\"\u003E{0}\u003Cspan class=\"photo-size-times\"\u003E\u00d7\u003C/span\u003E{1}\u003C/span\u003E","no_comment_text":"Please be sure to submit some text with your comment.","no_comment_email":"Please provide an email address to comment.","no_comment_author":"Please provide your name to comment.","comment_post_error":"Sorry, but there was an error posting your comment. Please try again later.","comment_approved":"Your comment was approved.","comment_unapproved":"Your comment is in moderation.","camera":"Camera","aperture":"Aperture","shutter_speed":"Shutter Speed","focal_length":"Focal Length","copyright":"Copyright","comment_registration":"0","require_name_email":"1","login_url":"https://danslimmon.wordpress.com/wp-login.php?redirect_to=https%3A%2F%2Fblog.danslimmon.com%2F2019%2F03%2F25%2Flatency-and-throughput-optimized-clusters-under-load%2F","blog_id":"38620006","meta_data":["camera","aperture","shutter_speed","focal_length","copyright"],"stats_query_args":"blog=38620006&v=wpcom&tz=0&user_id=0&subd=danslimmon","is_public":"1"};
//# sourceURL=jetpack-carousel-js-extra
</script>
<script id="jetpack-script-data-js-before">
window.JetpackScriptData = {"site":{"host":"wpcom","is_wpcom_platform":true}};
//# sourceURL=jetpack-script-data-js-before
</script>
<script id="jetpack-blocks-assets-base-url-js-before">
var Jetpack_Block_Assets_Base_Url="https://s0.wp.com/wp-content/mu-plugins/jetpack-plugin/moon/_inc/blocks/";
//# sourceURL=jetpack-blocks-assets-base-url-js-before
</script>
<script crossorigin='anonymous' type='text/javascript' src='/_static/??-eJyNUNtOwzAM/SHSDG1ie0F8yuQmaXHrxFHsdOzviWCtkECir+fqY3vLxnHSkNTqe4hBbK69JV6CKzConcQmWHAERU7dJE92h2Mk7oH2qmXGbAjTbAZ2VcyAH6sVk6Pqm6nJHMfYYkwJme5dxF/XxGoy1RFTkwfN4GbToFtuxgdhI3NayesSkudioSpHUEW32UAkqNi+IvkNFFcwq/GgsKP5Z+G1zXiEOShcJdCmW4F/BsGMEoOaY3f4jluBoXyJ/V8f8yhqHysblZnuA9Ja9RZfn8+Xl9PpcLycp0/8osLv'></script>
<script data-wp-strategy="defer" defer id="jetpack-block-subscriptions-js" src="/wp-content/mu-plugins/jetpack-plugin/moon/_inc/blocks/subscriptions/view.js?m=1782389597i&#038;minify=false&#038;ver=16.3-a.0"></script>
<script id="sharing-js-js-extra">
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":"1"};
//# sourceURL=sharing-js-js-extra
</script>
<script crossorigin='anonymous' type='text/javascript' src='/_static/??/wp-content/mu-plugins/carousel-wpcom/carousel-wpcom.js,/wp-content/mu-plugins/jetpack-plugin/moon/_inc/build/sharedaddy/sharing.min.js?m=1787950866j'></script>
<script id="sharing-js-js-after">
window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {};
window.WPCOM_sharing_popups[ "twitter" ] = "menubar=1,resizable=1,width=600,height=350";
window.WPCOM_sharing_popups = window.WPCOM_sharing_popups || {};
window.WPCOM_sharing_popups[ "facebook" ] = "menubar=1,resizable=1,width=600,height=400";
//# sourceURL=sharing-js-js-after
</script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s0.wp.com/wp-content/mu-plugins/wpcom-smileys/twemoji/2/72x72/","ext":".png","svgUrl":"https://s0.wp.com/wp-content/mu-plugins/wpcom-smileys/twemoji/2/svg/","svgExt":".svg","source":{"concatemoji":"/wp-includes/js/wp-emoji-release.min.js?m=1783090810i&ver=7.1.1-alpha-63555"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=/wp-includes/js/wp-emoji-loader.min.js
</script>
<iframe src='https://widgets.wp.com/likes/master.html?ver=20260912#ver=20260912' scrolling='no' id='likes-master' name='likes-master' style='display:none;'></iframe>
<div id='likes-other-gravatars' role="dialog" aria-hidden="true" tabindex="-1"><div class="likes-text"><span>%d</span></div><ul class="wpl-avatars sd-like-gravatars"></ul></div>
<script src="//stats.wp.com/w.js?69" defer></script> <script type="text/javascript">
_tkq = window._tkq || [];
_stq = window._stq || [];
_tkq.push(['storeContext', {'blog_id':'38620006','blog_tz':'0','user_lang':'en','blog_lang':'en','user_id':'0'}]);
// Prevent sending pageview tracking from WP-Admin pages.
_stq.push(['view', {'blog':'38620006','v':'wpcom','tz':'0','user_id':'0','post':'2294','subd':'danslimmon'}]);
_stq.push(['extra', {'crypt':'UE5tW3cvZGRkOERlal9Yd291RzZXcmdOZVVpJVVkJTNPRktUU0lrQW9PVUpPZzRnc2JsLnZITS0taTNaV3FRWjFXOS09YT9yOTFpK0NFK0pfUFRhVDA0Mm9DMmlaREgzY0FiLS1xUzQzWCVpRnl1a2l6dEhDfGc0OEZneERzSThfblY0NltYfHJ8M35uL0czfCZ5aFdTQjI4LyUmNjA4TH4zTkY9RiY4RExLblBaPXNtVWN4V2o9Nkh4d3FtXXlySTcldGIlZUNvXzFldHVQeVpmMz1+T11dMUYwU1E1Vkc2aG0/TVdlR3d6fnlRanlvR0lnW3o='}]);
_stq.push([ 'clickTrackerInit', '38620006', '2294' ]);
</script>
<noscript><img src="https://pixel.wp.com/b.gif?v=noscript" style="height:1px;width:1px;overflow:hidden;position:absolute;bottom:1px;" alt="" /></noscript>
<meta id="bilmur" property="bilmur:data" content="" data-provider="wordpress.com" data-service="simple" data-site-tz="UTC" data-custom-props="{&quot;enq_wp-polyfill&quot;:&quot;1&quot;,&quot;logged_in&quot;:&quot;0&quot;,&quot;wptheme&quot;:&quot;pub\/lovecraft&quot;,&quot;wptheme_is_block&quot;:&quot;0&quot;}" >
<script defer src="/wp-content/js/bilmur.min.js?i=17&amp;m=202637"></script>
<script>
(function() {
'use strict';
const fetches = {};
const promises = {};
const urls = {
'verbum': '/wp-content/mu-plugins/jetpack-mu-wpcom-plugin/moon/jetpack_vendor/automattic/jetpack-mu-wpcom/src/build/verbum-comments/verbum-comments.js?m=1788898305i&minify=false&ver=3d6130a57c3104d5bc3b'
};
const loaders = {
'verbum': () => {
fetchExternalScript('verbum');
promises['verbum'] = promises['verbum'] || loadWPScript('verbum');
return promises['verbum'];
},
};
const scriptExtras = {
};
window.WP_Enqueue_Dynamic_Script = {
loadScript: (handle) => {
if (!loaders[handle]) {
console.error('WP_Enqueue_Dynamic_Script: unregistered script `' + handle + '`.');
}
return loaders[handle]();
}
};
function fetchExternalScript(handle) {
if (!urls[handle]) {
return Promise.resolve();
}
fetches[handle] = fetches[handle] || fetch(urls[handle], { mode: 'no-cors' });
return fetches[handle];
}
function runExtraScript(handle, type, index) {
const id = 'wp-enqueue-dynamic-script:' + handle + ':' + type + ':' + (index + 1);
const template = document.getElementById(id);
if (!template) {
return Promise.reject();
}
const script = document.createElement( 'script' );
script.innerHTML = template.innerHTML;
document.body.appendChild( script );
return Promise.resolve();
}
function loadExternalScript(handle) {
if (!urls[handle]) {
return Promise.resolve();
}
return fetches[handle].then(() => {
return new Promise((resolve, reject) => {
const script = document.createElement('script');
script.onload = () => resolve();
script.onerror = (e) => reject(e);
script.src = urls[handle];
document.body.appendChild(script);
});
});
}
function loadExtra(handle, pos) {
const count = (scriptExtras[handle] && scriptExtras[handle][pos]) || 0;
let promise = Promise.resolve();
for (let i = 0; i < count; i++) {
promise = promise.then(() => runExtraScript(handle, pos, i));
}
return promise;
}
function loadWPScript(handle) {
// Core loads scripts in this order. See: https://github.com/WordPress/WordPress/blob/a59eb9d39c4fcba834b70c9e8dfd64feeec10ba6/wp-includes/class-wp-scripts.php#L428.
return loadExtra(handle, 'translations')
.then(() => loadExtra(handle, 'before'))
.then(() => loadExternalScript(handle))
.then(() => loadExtra(handle, 'after'));
}
} )();
</script>
</body>
</html>