Files
nexus/sreweekly/articles/401/04-a-spooky-performance-regression-in-aws-ebs-volumes.html
2026-09-12 17:23:01 +08:00

227 lines
31 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html lang="en" data-astro-cid-z4jru4n3><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><!-- Polyfill for Node.js process object used by some dependencies --><script>
if (typeof process === "undefined") {
window.process = { env: {}, argv: [], platform: "", version: "" };
}
</script><title>A Spooky Performance Regression in AWS EBS Volumes | DoltHub Blog</title><meta name="title" property="og:title" content="A Spooky Performance Regression in AWS EBS Volumes"><meta name="description" property="og:description" content="Blog for DoltHub, a website hosting databases made with Dolt, an open-source version-controlled SQL database with Git-like semantics."><meta name="image" property="og:image" content="https://static.dolthub.com/blogimages/spooky_computer.jpg/fca1e68dc21581e075bc474baf67f4c12a671e6f278df290bc5448b535068a2b.webp"><meta property="og:type" content="article"><meta name="twitter:image" content="https://static.dolthub.com/blogimages/spooky_computer.jpg/fca1e68dc21581e075bc474baf67f4c12a671e6f278df290bc5448b535068a2b.webp"><meta name="twitter:card" content="summary_large_image"><meta name="twitter:creator" content="DoltHub"><meta name="twitter:title" content="A Spooky Performance Regression in AWS EBS Volumes"><meta name="twitter:description" content="Blog for DoltHub, a website hosting databases made with Dolt, an open-source version-controlled SQL database with Git-like semantics."><meta name="twitter:site" content="@dolthub"><meta property="article:published_time" content="2023-11-22"><meta name="author" content="Dustin Brown"><script type="application/ld+json">{"@context":"https://schema.org","publisher":{"@type":"Organization","name":"DoltHub","logo":{"@type":"ImageObject","url":"https://static.dolthub.com/blogimages/dolthub-logo-new.png/74c964f28625513f156fe18b9ff84b606e67c3515676dac5dbf752f0540ce0f2.webp"}},"@type":"Article","headline":"A Spooky Performance Regression in AWS EBS Volumes","image":"https://static.dolthub.com/blogimages/spooky_computer.jpg/fca1e68dc21581e075bc474baf67f4c12a671e6f278df290bc5448b535068a2b.webp","description":"Blog for DoltHub, a website hosting databases made with Dolt, an open-source version-controlled SQL database with Git-like semantics.","datePublished":"2023-11-22","author":{"@type":"Person","name":"Dustin Brown","url":"https://www.dolthub.com/team#dustin"},"mainEntityOfPage":{"@type":"WebPage","@id":"https://www.dolthub.com/blog/"}}</script><link rel="icon" type="image/png" href="/blog/favicon.png"><meta name="generator" content="Astro v7.2.8"><!-- Preconnect to external domains for faster loading --><link rel="preconnect" href="https://fonts.googleapis.com" crossorigin><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link rel="preconnect" href="https://www.googletagmanager.com" crossorigin><link rel="dns-prefetch" href="https://www.googletagmanager.com"><link rel="preconnect" href="https://js.hs-scripts.com" crossorigin><link rel="dns-prefetch" href="https://js.hs-scripts.com"><!-- Google Fonts --><link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Source+Sans+Pro:ital,wght@0,400;0,600;0,700;1,400;1,600;1,700&family=Space+Mono:ital,wght@0,400;0,700;1,400;1,700&display=swap"><!-- Google Analytics --><script src="https://www.googletagmanager.com/gtag/js?id=G-BF3PNBH479" async></script><script>window.dataLayer = window.dataLayer || [];
function gtag() {
dataLayer.push(arguments);
}
gtag("js", new Date());
gtag("config", "G-BF3PNBH479", {
page_path: window.location.pathname,
});</script><script id="mcjs">!function(c,h,i,m,p){m=c.createElement(h),p=c.getElementsByTagName(h)[0],m.async=1,m.src=i,p.parentNode.insertBefore(m,p)}(document,"script","https://chimpstatic.com/mcjs-connected/js/users/362cc10f3158476f3baf3a9d6/cd79a048c8d6d0d81be9fcb7d.js");</script><!-- Smooth scroll for anchor links --><script type="module">function e(e,t=!0){if(!e||e===`#`||e===``)return;let n=document.querySelector(e);if(n){let e=n.getBoundingClientRect().top+window.pageYOffset-80;window.scrollTo({top:e,behavior:t?`smooth`:`auto`})}}document.addEventListener(`DOMContentLoaded`,()=>{document.querySelectorAll(`a[href^="#"]`).forEach(t=>{t.addEventListener(`click`,function(t){let n=this.getAttribute(`href`);n!==`#`&&n!==``&&document.querySelector(n)&&(t.preventDefault(),e(n,!0),window.history.pushState(null,null,n))})}),window.location.hash&&setTimeout(()=>{e(window.location.hash,!1)},100)});</script><link rel="stylesheet" href="/blog/_astro/BaseLayout.CcQdOFRi.css"><link rel="stylesheet" href="/blog/_astro/TagNav.CWEevpuq.css"><link rel="stylesheet" href="/blog/_astro/_..BZ4GUMVV.css"><style>._shareWidget_1ibbk_1{justify-content:flex-start;align-items:center;display:flex;&>p{--tw-text-opacity:1;color:rgb(122 150 171/var(--tw-text-opacity,1));margin-right:.75rem}}._rssContainer_1ibbk_9{margin-bottom:.25rem;padding:.5rem;display:inline-block}._rssIcon_1ibbk_13{--tw-text-opacity:1;color:rgb(12 26 50/var(--tw-text-opacity,1));font-size:1.125rem;line-height:1.75rem}
._container_aro5d_1{margin:1rem auto}._inner_aro5d_5{--tw-bg-opacity:1;background-color:rgb(255 255 255/var(--tw-bg-opacity,1));border-radius:.5rem;flex-direction:column;align-items:center;margin-left:1rem;margin-right:1rem;padding:1.5rem;display:flex}@media (width>=1024px){._inner_aro5d_5{flex-direction:row;justify-content:space-between;height:7rem;padding:0 2rem}}._inner_aro5d_5{& p{letter-spacing:.025em;--tw-text-opacity:1;color:rgb(156 159 177/var(--tw-text-opacity,1));margin-top:.25rem}@media (width>=1024px){& p{font-size:1rem;line-height:1.5rem}}}._left_aro5d_14{flex-direction:column;margin-right:1rem;display:flex}@media (width>=640px){._left_aro5d_14{flex-direction:row;align-items:center}}._left_aro5d_14{& h3{font-size:1.125rem;line-height:1.75rem}@media (width>=1024px){& h3{font-size:1.25rem;line-height:1.75rem}}& img{width:3rem;height:3rem;margin-bottom:.5rem;margin-right:1rem}@media (width>=640px){& img{margin-bottom:0}}@media (width>=1024px){& img{width:4rem;height:4rem;margin-right:2.5rem}}& a{--tw-text-opacity:1;color:rgb(39 105 180/var(--tw-text-opacity,1));font-weight:600;text-decoration-line:underline}& a:hover{--tw-text-opacity:1;color:rgb(61 145 240/var(--tw-text-opacity,1))}}._right_aro5d_30{padding-top:1rem}
</style></head><body data-astro-cid-z4jru4n3><style>astro-island,astro-slot,astro-static-slot{display:contents}</style><script>(()=>{var e=async t=>{await(await t())()};(self.Astro||(self.Astro={})).load=e;window.dispatchEvent(new Event("astro:load"));})();</script><script>(()=>{var g=Object.defineProperty;var w=(a,s,c)=>s in a?g(a,s,{enumerable:!0,configurable:!0,writable:!0,value:c}):a[s]=c;var l=(a,s,c)=>w(a,typeof s!="symbol"?s+"":s,c);var E=new Set(["__proto__","constructor","prototype"]);{let a={0:t=>y(t),1:t=>c(t),2:t=>new RegExp(t),3:t=>new Date(t),4:t=>new Map(c(t)),5:t=>new Set(c(t)),6:t=>BigInt(t),7:t=>new URL(t),8:t=>new Uint8Array(t),9:t=>new Uint16Array(t),10:t=>new Uint32Array(t),11:t=>Number.POSITIVE_INFINITY*t},s=t=>{let[p,e]=t;return p in a?a[p](e):void 0},c=t=>t.map(s),y=t=>typeof t!="object"||t===null?t:Object.fromEntries(Object.entries(t).map(([p,e])=>[p,s(e)]));class f extends HTMLElement{constructor(){super(...arguments);l(this,"Component");l(this,"hydrator");l(this,"hydrate",async()=>{var b;if(!this.hydrator||!this.isConnected)return;let e=(b=this.parentElement)==null?void 0:b.closest("astro-island[ssr]");if(e){e.addEventListener("astro:hydrate",this.hydrate,{once:!0});return}let r=this.querySelectorAll("astro-slot"),n={},d=this.querySelectorAll("template[data-astro-template]");for(let o of d){let i=o.closest(this.tagName);i!=null&&i.isSameNode(this)&&(n[o.getAttribute("data-astro-template")||"default"]=o.innerHTML,o.remove())}for(let o of r){let i=o.closest(this.tagName);i!=null&&i.isSameNode(this)&&(n[o.getAttribute("name")||"default"]=o.innerHTML)}let u;try{u=this.hasAttribute("props")?y(JSON.parse(this.getAttribute("props"))):{}}catch(o){let i=this.getAttribute("component-url")||"<unknown>",v=this.getAttribute("component-export");throw v&&(i+=` (export ${v})`),console.error(`[hydrate] Error parsing props for component ${i}`,this.getAttribute("props"),o),o}let h;await this.hydrator(this)(this.Component,u,n,{client:this.getAttribute("client")}),this.removeAttribute("ssr"),this.dispatchEvent(new CustomEvent("astro:hydrate"))});l(this,"unmount",()=>{this.isConnected||this.dispatchEvent(new CustomEvent("astro:unmount"))})}disconnectedCallback(){document.removeEventListener("astro:after-swap",this.unmount),document.addEventListener("astro:after-swap",this.unmount,{once:!0})}connectedCallback(){if(!this.hasAttribute("await-children")||document.readyState==="interactive"||document.readyState==="complete")this.childrenConnectedCallback();else{let e=()=>{document.removeEventListener("DOMContentLoaded",e),r.disconnect(),this.childrenConnectedCallback()},r=new MutationObserver(()=>{var n;((n=this.lastChild)==null?void 0:n.nodeType)===Node.COMMENT_NODE&&this.lastChild.nodeValue==="astro:end"&&(this.lastChild.remove(),e())});r.observe(this,{childList:!0}),document.addEventListener("DOMContentLoaded",e)}}async childrenConnectedCallback(){let e=this.getAttribute("before-hydration-url");e&&await import(e),this.start()}getRetryImportUrl(e){let r=new URL(e,document.baseURI);return r.searchParams.set("astro-retry",Date.now().toString()),r.toString()}async importWithRetry(e){try{return await import(e)}catch(r){return await new Promise(n=>setTimeout(n,1e3)),import(this.getRetryImportUrl(e))}}handleHydrationError(e){let r=this.getAttribute("component-url"),n=new CustomEvent("astro:hydration-error",{cancelable:!0,bubbles:!0,composed:!0,detail:{error:e,componentUrl:r}});this.dispatchEvent(n)&&console.error(`[astro-island] Error hydrating ${r}`,e)}async start(){let e=JSON.parse(this.getAttribute("opts")),r=this.getAttribute("client");if(Astro[r]===void 0){window.addEventListener(`astro:${r}`,()=>this.start(),{once:!0});return}try{await Astro[r](async()=>{let n=this.getAttribute("renderer-url");try{let[d,{default:u}]=await Promise.all([this.importWithRetry(this.getAttribute("component-url")),n?this.importWithRetry(n):Promise.resolve({default:()=>()=>{}})]),h=this.getAttribute("component-export")||"default";if(h.includes(".")){this.Component=d;for(let m of h.split(".")){if(E.has(m)||!this.Component||typeof this.Component!="object"&&typeof this.Component!="function"||!Object.hasOwn(this.Component,m))throw new Error(`Invalid component export path: ${h}`);this.Component=this.Component[m]}}else{if(E.has(h))throw new Error(`Invalid component export path: ${h}`);this.Component=d[h]}return this.hydrator=u,this.hydrate}catch(d){return this.handleHydrationError(d),()=>{}}},e,this)}catch(n){this.handleHydrationError(n)}}attributeChangedCallback(){this.hydrate()}}l(f,"observedAttributes",["props"]),customElements.get("astro-island")||customElements.define("astro-island",f)}})();</script><astro-island uid="Z1N0E8q" prefix="r8" component-url="/blog/_astro/ThemeInit.BPM5YMoI.js" component-export="default" renderer-url="/blog/_astro/client.Cu7VUUOY.js" props="{&quot;data-astro-cid-z4jru4n3&quot;:[0,true]}" ssr client="load" opts="{&quot;name&quot;:&quot;ThemeInit&quot;,&quot;value&quot;:true}" await-children><span></span><!--astro:end--></astro-island><div data-astro-cid-z4jru4n3><script>(()=>{var e=async t=>{await(await t())()};(self.Astro||(self.Astro={})).only=e;window.dispatchEvent(new Event("astro:only"));})();</script><astro-island uid="Z1Ng7Qf" component-url="/blog/_astro/Navbar.D8sLApy3.js" component-export="default" renderer-url="/blog/_astro/client.Cu7VUUOY.js" props="{&quot;data-astro-cid-z4jru4n3&quot;:[0,true]}" ssr client="only" opts="{&quot;name&quot;:&quot;Navbar&quot;,&quot;value&quot;:&quot;react&quot;}"></astro-island><main class="siteMain" data-astro-cid-z4jru4n3><div class="container" data-astro-cid-n67f4zmd><astro-island uid="Z22EsbD" prefix="r11" component-url="/blog/_astro/TagNav.DNZ68-Wm.js" component-export="default" renderer-url="/blog/_astro/client.Cu7VUUOY.js" props="{&quot;data-astro-cid-n67f4zmd&quot;:[0,true]}" ssr client="load" opts="{&quot;name&quot;:&quot;TagNav&quot;,&quot;value&quot;:true}" await-children><div class="_container_13ry7_1" data-cy="tag-nav"><div class="_top_13ry7_9"><astro-slot><div class="wrapper" data-astro-cid-mfsbv3p2><a class="link" href="/blog" data-cy="blog-home-back-link" data-astro-cid-mfsbv3p2><svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 1024 1024" data-astro-cid-mfsbv3p2="true" height="1em" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M724 218.3V141c0-6.7-7.7-10.4-12.9-6.3L260.3 486.8a31.86 31.86 0 0 0 0 50.3l450.8 352.1c5.3 4.1 12.9.4 12.9-6.3v-77.3c0-4.9-2.3-9.6-6.1-12.6l-360-281 360-281.1c3.8-3 6.1-7.7 6.1-12.6z"></path></svg>Blog</a></div></astro-slot></div><div><h3>PRODUCTS</h3><ul><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-dolt"><span class="_tagName_13ry7_20">Dolt</span></button></li><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-doltgres"><span class="_tagName_13ry7_20">Doltgres</span></button></li><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-doltlite"><span class="_tagName_13ry7_20">DoltLite</span></button></li><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-dumbo"><span class="_tagName_13ry7_20">Dumbo</span></button></li><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-hosted"><span class="_tagName_13ry7_20">Hosted Dolt</span></button></li><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-dolthub"><span class="_tagName_13ry7_20">DoltHub</span></button></li><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-doltlab"><span class="_tagName_13ry7_20">DoltLab</span></button></li><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-workbench"><span class="_tagName_13ry7_20">Dolt Workbench</span></button></li></ul></div><div><h3>KEYWORDS</h3><ul><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-ai"><span class="_tagName_13ry7_20">AI</span></button></li><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-usecase"><span class="_tagName_13ry7_20">Use Case</span></button></li><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-reference"><span class="_tagName_13ry7_20">Reference</span></button></li><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-featurerelease"><span class="_tagName_13ry7_20">Feature Release</span></button></li><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-golang"><span class="_tagName_13ry7_20">Golang</span></button></li><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-integration"><span class="_tagName_13ry7_20">Integration</span></button></li><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-performance"><span class="_tagName_13ry7_20">Performance</span></button></li><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-technical"><span class="_tagName_13ry7_20">Technical</span></button></li><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-web"><span class="_tagName_13ry7_20">Web</span></button></li><li><button class="_tagButton_13ry7_13" type="button" data-cy="nav-tag-dataset"><span class="_tagName_13ry7_20">Dataset</span></button></li></ul></div></div><!--astro:end--></astro-island><div class="main" data-astro-cid-n67f4zmd><article class="blog-post" data-cy="blog-post" data-astro-cid-n67f4zmd><div class="inner" data-astro-cid-n67f4zmd><header data-astro-cid-n67f4zmd><div class="inner-header" data-astro-cid-n67f4zmd><astro-island uid="1F1cfk" prefix="r6" component-url="/blog/_astro/Tags.BJDnvT4Z.js" component-export="default" renderer-url="/blog/_astro/client.Cu7VUUOY.js" props="{&quot;tags&quot;:[0,&quot;dolt, performance&quot;],&quot;data-astro-cid-n67f4zmd&quot;:[0,true]}" ssr client="load" opts="{&quot;name&quot;:&quot;Tags&quot;,&quot;value&quot;:true}" await-children><div class="_top_w734n_1" data-cy="blog-tags"><div class="_tagList_w734n_5"><button type="button" class="_label_w734n_9 _dolt_w734n_32" aria-label="Search for Dolt posts">Dolt</button><button type="button" class="_label_w734n_9 _performance_w734n_59" aria-label="Search for Performance posts">Performance</button></div></div><!--astro:end--></astro-island><h1 data-astro-cid-n67f4zmd>A Spooky Performance Regression in AWS EBS Volumes</h1><div class="post-metadata" data-astro-cid-n67f4zmd><div class="_metadata_va01l_1"><img class="_authorImg_va01l_17" alt="Dustin Brown" src="data:image/webp;base64,UklGRiwGAABXRUJQVlA4WAoAAAAQAAAAXwAAXwAAQUxQSFkCAAABkEbbtrEt97kfsm3btm3btm3btm3btt1n29Zzct2zb/jViIgJEP+QrTJX6jpq8qQRnSuklxqyyNfv4Mdgk4goLujN9vbpNROvxhZnk344+tnYbDopuTuQfuHr/ol1Yd3PiX5tzIG8ekiyKIJ++dNKOki8Oo4U2tTAs5wdS0rfl4HrHEaKb2cGy/OOlK+yhDIWkvrA2lC57BnQfmuk3iYHj8JAlgeIo9kbKON7FrQOqHwQj8sJcNrG8niTDmck8XTOhTObiU9xnJVMAsrgrGcSVA5nE5NA7QWUwdnIxK8kzjomXoVh5GomzjlBjM57HzLxXtNBQuSxJ75uRSB6m4xoEIKxmThvMQAS3WB1JQFAqmesHiXT3ePkAIlvsbqXBEDuZnUuHoAYxmqDQKwSwmkIRBYbRkGVIBJeZ2STCcLYxOh6IgjRJJTPBIFpPfaNH4vYiCPpQISRegKLPbVSCtx6EQzM7gI5rxuD4EpQSW4xeJsBSixjsN8Sq3W0uoECO5uNMq+iYHKzshPxwUQvZSMEentTVVe0FLtJ9Z6UYP1MZTQRK94JUv88LVR+Vwbh9aD6mAxoJpBR4hZx/NArM0q+Je7EM/bNrOKWALmm2hFjj90t0vCShWd/JOaRT+dWS8EmUa1NroQYfH9hw4xSnZG9z4Vggo14u7VDdgslqRuus4kj7Bjb7e2zGr8oSZW5j8JJhzEf1tRK/HMy37CrQaTPkIvd0v5YwpobnUizMQ8HZvheohangknDcU8HpflKVj8RTpqOu91QitSzfUnjflMLnyS9x7yi/z0EAFZQOCCsAwAAMBYAnQEqYABgAD6NPJZHpSOiITKz2zCgEYlAGgmSenMA597b38703tcdmPNuLQ728V9UoHdzSJEiQfqzMXDJOKGEx/uCrVAu+NcVH44BUjztQX5AWymL/R2k//Z63N+oFjV7/epArFrNPoJSvL17NiRcVNjhiFMYcJLkhPefDVMqOGjBxC55a5XbwgFvv4tOjVGE2v8Dq5uAIvRNfO4aI3RW6fL0SQz5hM0hmqGFmoYmeG1++8NiBpaAAP78rRf/m98pI8+I0xYnCHMitYyXOmBdGksiSlRVfO9702BhTrYM7dMMOI1zqFFWwvaRIGV4V16NX5FiXd3F4oTPSjmvPcfyXC8m4dwQx0s+1dqyZi5qf/Hx7ingo+diF6jmTsBMNHkycgV9SKhO/fxRufrsZIZlsKtJlJUbRhvrPZXJth2BUKYkBDbrZ0UQphPnWB7Lw6nOC4iBfEkbkghvn0mSF2c8i96x233ZgcUZDI+rlcdaoAFsfvw8j5VVK6uVvQHwLHYr7v9XJGXQ/GYZ/37Z1fZC15shP4ufxf1G+qzQOo8wdMg7ThIC0mzCz00RQ771twhESFDoKoMNvW2YDBZ0mbZj2LobhEoR2MqZFvWg3ugNb3rLqD1h+UzA46mAB5CaeVX1cgP9yXsqs9nvKtK27YGsAvJYl+WpNBqk58kHzv8Fb+Z3MFZ+QFeOu3pJBgtnMEmSyrF9nJvZOQ7mAOM6PDEC1/rWgooG6drdwGXnVXNzB3kcufcSD7P2h8kFg46HdHXuBp0sVNpc/gJHUq85OQDoPlQd88X16S+RPU9gGxTuwssovsDyVXbmS+WObBm0aYWAb0neKW7yfr4SxBCyDfU0BtEFTOvfuDfafZdxy6WNT4KSMHIqZookUCjenTWvXq/aEniMyUEyXOvo5kbh4nwC8kS2SZmoON1cjUa7C7I2AOaB3FXSDJtbQWGzlzNmA7N4jCU+8AAbxCO2PN6+t2+DC6bpIdicKnWUPSvedmnR76Wx8jpdIjl2QmXSqqGBJYEHkKp49YBybgJ8FJ92em/BagtlWzfUvV23+BpLE1gQ2F1nbEFzw1KPyeCPNS65eLw2zx/E7sEYQCs5cRdt2NOX5NvRd52YVUCU+Yx74e7RITSpO/FeI6IJgRABDLAT0XmWPDHvhYfkG6XtuJefrHRXbOeBmMXd21UVTsUC5scctxwicrDwpjF3Gm4X2R9NrMV5pHtPGGK/czenuBjbaj6tTYyb9k2QJMFgAA==" loading="lazy"/><div class="_authorInfo_va01l_21"><a href="https://www.dolthub.com/team#dustin">Dustin Brown</a></div><time>November 22, 2023</time></div><div class="time-to-read" data-astro-cid-n67f4zmd>5 min read</div></div></div></header><div class="middle" data-astro-cid-n67f4zmd><div class="post-text" data-cy="blog-post-text"><p>For every <a href="https://www.doltdb.com">Dolt</a> release, we run a suite of <a href="https://github.com/akopytov/sysbench">Sysbench</a> tests that measure the median latency of Dolt’s reads and writes.</p>
<p>Since Dolt is a drop-in replacement for <a href="https://www.mysql.com/">MySQL</a>, and soon to be a drop-in replacement for <a href="https://www.postgresql.org/">PostgreSQL</a>, we compare Dolt’s results to MySQL’s to understand how much slower Dolt is than its counterpart.</p>
<p>We publish these benchmarks on <a href="https://dolthub.com/docs/sql-reference/benchmarks/latency">our documentation site</a> and also post them in Dolt’s release notes. Internally, to catch any significant changes in latency that might have been committed during the day, we run these tests against nightly builds of Dolt as well.</p>
<p>Curiously, on October 31st 2023, All Hallows’ Eve, in our nightly run of Sysbench against Dolt and MySQL we noticed the latency of writes against both MySQL and Dolt had increased, meaning writes got slower.</p>
<p><img src="https://static.dolthub.com/blogimages/spooky_computer_2.jpg/21739a398ce04980adadb5bf48a58628b561b8a6674a4401cac73944c2427b81.webp" alt="Spooky computer"></p>
<p>Even more curious, still, was that MySQL’s latency for certain write tests had gotten much worse than Dolt’s, and surprisingly, Dolt was reporting as <em>faster</em> than MySQL on writes, where it used to be slower.</p>
<p>Here’s what we saw. The nightly write benchmarks on Oct 31th looked like this:</p>
<table><thead><tr><th>write_tests</th><th>server_name</th><th>server_version</th><th>latency_median</th><th>server_name</th><th>server_version</th><th>latency_median</th><th>multiplier</th></tr></thead><tbody><tr><td>oltp_delete_insert</td><td>mysql</td><td>8.0.35</td><td>4.49</td><td>dolt</td><td>c1790e8c</td><td>5.57</td><td>1.2</td></tr><tr><td>oltp_insert</td><td>mysql</td><td>8.0.35</td><td>2.26</td><td>dolt</td><td>c1790e8c</td><td>2.66</td><td>1.2</td></tr><tr><td>oltp_read_write</td><td>mysql</td><td>8.0.35</td><td>6.79</td><td>dolt</td><td>c1790e8c</td><td>13.95</td><td>2.1</td></tr><tr><td>oltp_update_index</td><td>mysql</td><td>8.0.35</td><td>2.3</td><td>dolt</td><td>c1790e8c</td><td>2.76</td><td>1.2</td></tr><tr><td>oltp_update_non_index</td><td>mysql</td><td>8.0.35</td><td>2.26</td><td>dolt</td><td>c1790e8c</td><td>2.66</td><td>1.2</td></tr><tr><td>oltp_write_only</td><td>mysql</td><td>8.0.35</td><td>3.43</td><td>dolt</td><td>c1790e8c</td><td>6.91</td><td>2.0</td></tr><tr><td>types_delete_insert</td><td>mysql</td><td>8.0.35</td><td>4.49</td><td>dolt</td><td>c1790e8c</td><td>5.88</td><td>1.3</td></tr></tbody></table>
<p>The <code>latency_median</code> columns show the median latencies in milliseconds for each server, and the <code>multiplier</code> columns show how many multiples slower Dolt is than MySQL.</p>
<p>Then, on Halloween, our Sysbench report showed the following:</p>
<table><thead><tr><th>write_tests</th><th>server_name</th><th>server_version</th><th>latency_median</th><th>server_name</th><th>server_version</th><th>latency_median</th><th>multiplier</th></tr></thead><tbody><tr><td>oltp_delete_insert</td><td>mysql</td><td>8.0.35</td><td>7.98</td><td>dolt</td><td>c7b85e8c</td><td>6.79</td><td>0.9</td></tr><tr><td>oltp_insert</td><td>mysql</td><td>8.0.35</td><td>3.75</td><td>dolt</td><td>c7b85e8c</td><td>3.36</td><td>0.9</td></tr><tr><td>oltp_read_write</td><td>mysql</td><td>8.0.35</td><td>8.43</td><td>dolt</td><td>c7b85e8c</td><td>14.46</td><td>2.1</td></tr><tr><td>oltp_update_index</td><td>mysql</td><td>8.0.35</td><td>3.82</td><td>dolt</td><td>c7b85e8c</td><td>3.36</td><td>0.9</td></tr><tr><td>oltp_update_non_index</td><td>mysql</td><td>8.0.35</td><td>3.82</td><td>dolt</td><td>c7b85e8c</td><td>3.3</td><td>0.9</td></tr><tr><td>oltp_write_only</td><td>mysql</td><td>8.0.35</td><td>5.37</td><td>dolt</td><td>c7b85e8c</td><td>7.56</td><td>1.4</td></tr><tr><td>types_delete_insert</td><td>mysql</td><td>8.0.35</td><td>7.7</td><td>dolt</td><td>c7b85e8c</td><td>7.3</td><td>1.0</td></tr></tbody></table>
<p>MySQL and Dolt had gotten slower on all benchmarks, but MySQL had gotten much slower than Dolt had, compared to the previous night’s run.</p>
<p>Additionally, Dolt was also reporting <em>faster</em> than MySQL on four benchmarks, with a multiplier of only <code>0.9</code>, where previously it had been slower with a multiplier of <code>1.2</code>.</p>
<p>Once we saw this anomaly, we started an investigation to find out why our benchmark results changed so significantly overnight.</p>
<p>We started by looking at changes in Dolt to see if we’d altered anything in the Dolt code that might explain the regression. But, because both Dolt <em>and</em> MySQL were showing a performance regression, we knew pretty quickly that something in our benchmarking infrastructure had changed, resulting in the slower results for both databases.</p>
<p>Coincidentally, on October 31st during the day, we <em>had</em> made a change that affected our benchmarking infrastructure.</p>
<p>We upgraded the <a href="https://aws.amazon.com/eks/">AWS EKS</a> cluster where our benchmarks run from version <code>1.27</code> to <code>1.28</code>. This also required us to upgrade the <a href="https://aws.amazon.com/pm/ec2">EC2</a> <a href="https://docs.aws.amazon.com/AWSEC2/latest/WindowsGuide/AMIs.html">AMIs</a> used for the benchmarking hosts, and upgrade some <a href="https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html">EKS Add-ons</a> as well. Benchmarking hosts are provisioned when a benchmarking Job is scheduled, and are deprovisioned when it’s complete.</p>
<p>This upgrade process is nothing new for us, and until now, we’ve never seen it affect our benchmarking results. This made this performance regression all the more odd, but we went ahead and started downgrading the AMIs and Add-ons to see if the regression went away.</p>
<p>It did not!</p>
<p>At this point we were confident that upgrading the EKS cluster did not actually change write performance on the benchmarking hosts. We were also sure that upgrading the host’s AMIs were also not responsible for the slow down, since we downgraded to an AMI released in May of 2023 and <em>still</em> saw the regression.</p>
<p>Finally, we decided to strip our benchmarking down to the bare essentials and run the Sysbench tests on an EC2 host outside of our EKS Cluster to try and isolate and confirm the source of regression.</p>
<p>We set up the host and ran the Sysbench tests by hand from the terminal. When we did this, we still observed the regression in write tests, and this led us to our hypothesis that something must have changed with EC2’s EBS volumes.</p>
<p><img src="https://static.dolthub.com/blogimages/ec2_ebs_in_graveyard.jpg/0b82d2a8e9f6f6609a7371a59353d5c523fc8ae9ff89c8163d0ba7f15d3c1976.webp" alt="Spooky EBS"></p>
<p>We’ve always benchmarked MySQL and Dolt on hosts using EBS volumes since modern applications often use similar cloud architecture, and bare metal hosts with disks attached are becoming less and less common in application stacks.</p>
<p>To test our hypothesis that the regression might be a result of the disk being used, we created an in-memory filesystem on the EC2 and benchmarked MySQL using this instead.</p>
<p>As expected, we saw no regression in these in-memory results, and most of the write test median latencies were sub-millisecond.</p>
<p>My coworker Aaron then suggested we run the benchmarks again against the EBS volume, only this time, with <code>fsync</code> disabled in MySQL.</p>
<p>Disabling <code>fsync</code> in MySQL changes the flush behavior of writes, allowing the operating system to do its best to buffer, order, and delay writes, which can improve performance, but also can corrupt the database if the system crashes and transactional writes are not completely written to disk.</p>
<p>And boom. With <code>fsync</code> disabled, Sysbench was once again reporting the numbers we’d seen for MySQL before October 31st!</p>
<p><img src="https://static.dolthub.com/blogimages/surprise_zach.png/07b14858391e782b452fbef890720ef4a1d600135d5e375cf5558de11b112222.webp" alt="Shocked face"></p>
<p>As far as we can tell, AWS changed something with their EBS <code>gp2</code> volumes that causes MySQL’s (and Dolt’s, to a lesser degree) write performance with <code>fsync</code> enabled, to be slower than it used to be.</p>
<p>We aren’t sure what exactly they’ve changed, but it has been consistent in all of our benchmarking tests… ever… since… that… night…</p>
<h1 id="conclusion">Conclusion<a class="anchor-link" aria-label="Link to heading" href="#conclusion">#</a></h1>
<p>In the end, we decided to keep running and publishing benchmarks with <code>fsync</code> enabled. As far as we can tell, this is real-world performance and Dolt now compares favorably on writes to MySQL with the current EBS network attached storage.</p>
<p>However, this saga shows the danger of publishing benchmark comparisons of databases. Small changes in the set up can create large differences when comparing results.</p>
<p>It’s safe to say that Dolt is approximately 2X slower than MySQL. Making a comparison with more precision requires further specifying the problem. Are we using network attached storage or local SSDs? Is <code>fsync</code> on or off? As you can see, these choices really matter.</p>
<p>Have you noticed any difference in EBS performance recently? Let us know by hopping into our <a href="https://discord.gg/gqr7K4VNKe">Discord</a>.</p>
<p>Don’t forget to check out each of our different product offerings below, to find which ones are right for you:</p>
<ul>
<li><a href="http://www.doltdb.com">Dolt</a>—it’s Git for data.</li>
<li><a href="https://www.dolthub.com">DoltHub</a>—it’s GitHub for Dolt.</li>
<li><a href="https://www.doltlab.com">DoltLab</a>—it’s GitLab for data.</li>
<li><a href="https://hosted.doltdb.com/">Hosted Dolt</a>—it’s RDS for Dolt databases.</li>
</ul></div></div><div class="bottom" data-astro-cid-n67f4zmd><div class="wrapper" data-astro-cid-mfsbv3p2><a class="link" href="/blog" data-cy="blog-home-back-link" data-astro-cid-mfsbv3p2><svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 1024 1024" data-astro-cid-mfsbv3p2="true" height="1em" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M724 218.3V141c0-6.7-7.7-10.4-12.9-6.3L260.3 486.8a31.86 31.86 0 0 0 0 50.3l450.8 352.1c5.3 4.1 12.9.4 12.9-6.3v-77.3c0-4.9-2.3-9.6-6.1-12.6l-360-281 360-281.1c3.8-3 6.1-7.7 6.1-12.6z"></path></svg>Blog</a></div><astro-island uid="EEObE" component-url="/blog/_astro/ShareWidget.4Cb1sV4F.js" component-export="default" renderer-url="/blog/_astro/client.Cu7VUUOY.js" props="{&quot;url&quot;:[0,&quot;https://www.dolthub.com/blog/2023-11-22-spooky-performance-regression-aws-ebs&quot;],&quot;data-astro-cid-n67f4zmd&quot;:[0,true]}" ssr client="only" opts="{&quot;name&quot;:&quot;ShareWidget&quot;,&quot;value&quot;:&quot;react&quot;}"></astro-island></div></div></article><div class="tim-update" data-astro-cid-n67f4zmd><astro-island uid="23NI33" component-url="/blog/_astro/TimWeeklyUpdate.B0ltDdcN.js" component-export="default" renderer-url="/blog/_astro/client.Cu7VUUOY.js" props="{&quot;data-astro-cid-n67f4zmd&quot;:[0,true]}" ssr client="only" opts="{&quot;name&quot;:&quot;TimWeeklyUpdate&quot;,&quot;value&quot;:&quot;react&quot;}"></astro-island></div></div></div></main><div class="bottomOuter" data-astro-cid-z4jru4n3><div class="bottom" data-astro-cid-z4jru4n3><astro-island uid="Su3vr" component-url="/blog/_astro/Footer.Bj3EZl7C.js" component-export="default" renderer-url="/blog/_astro/client.Cu7VUUOY.js" props="{&quot;data-astro-cid-z4jru4n3&quot;:[0,true]}" ssr client="only" opts="{&quot;name&quot;:&quot;Footer&quot;,&quot;value&quot;:&quot;react&quot;}"></astro-island></div></div></div></body></html>