499 lines
35 KiB
HTML
499 lines
35 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
|
|
<title>To Auto Scale or not to Auto Scale, that is the question</title>
|
|
<meta charset="utf-8" />
|
|
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
|
|
<meta name="HandheldFriendly" content="True" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
|
|
<link rel="preload" as="style" href="/assets/built/screen.css?v=S0DSiw6jVHJaYsG0" />
|
|
<link rel="preload" as="script" href="/assets/built/casper.js?v=YtPhrez3GuGLVDYs" />
|
|
|
|
<link rel="stylesheet" type="text/css" href="/assets/built/screen.css?v=S0DSiw6jVHJaYsG0" />
|
|
|
|
<meta name="description" content="The challenges of Auto Scaling, from cold start impact, tech debt, and cost realities. Prioritising scaling as code and shared responsibility for optimal performance in cloud efficiency.">
|
|
<link rel="canonical" href="https://karlstoney.com/to-auto-scale-or-not-to-auto-scale-that-is-the-question/">
|
|
<meta name="referrer" content="no-referrer-when-downgrade">
|
|
|
|
<meta property="og:site_name" content="karlstoney.com">
|
|
<meta property="og:type" content="article">
|
|
<meta property="og:title" content="To Auto Scale or not to Auto Scale, that is the question">
|
|
<meta property="og:description" content="The challenges of Auto Scaling, from cold start impact, tech debt, and cost realities. Prioritising scaling as code and shared responsibility for optimal performance in cloud efficiency.">
|
|
<meta property="og:url" content="https://karlstoney.com/to-auto-scale-or-not-to-auto-scale-that-is-the-question/">
|
|
<meta property="article:published_time" content="2023-10-05T08:41:57.000Z">
|
|
<meta property="article:modified_time" content="2024-07-01T08:11:46.000Z">
|
|
<meta property="article:tag" content="Kubernetes">
|
|
|
|
<meta name="twitter:card" content="summary">
|
|
<meta name="twitter:title" content="To Auto Scale or not to Auto Scale, that is the question">
|
|
<meta name="twitter:description" content="The challenges of Auto Scaling, from cold start impact, tech debt, and cost realities. Prioritising scaling as code and shared responsibility for optimal performance in cloud efficiency.">
|
|
<meta name="twitter:url" content="https://karlstoney.com/to-auto-scale-or-not-to-auto-scale-that-is-the-question/">
|
|
<meta name="twitter:label1" content="Written by">
|
|
<meta name="twitter:data1" content="Karl Stoney">
|
|
<meta name="twitter:label2" content="Filed under">
|
|
<meta name="twitter:data2" content="Kubernetes">
|
|
<meta name="twitter:site" content="@karlstoney">
|
|
<meta name="twitter:creator" content="@karlstoney">
|
|
|
|
<script type="application/ld+json">
|
|
{
|
|
"@context": "https://schema.org",
|
|
"@type": "Article",
|
|
"publisher": {
|
|
"@type": "Organization",
|
|
"name": "karlstoney.com",
|
|
"url": "https://karlstoney.com/",
|
|
"logo": {
|
|
"@type": "ImageObject",
|
|
"url": "https://karlstoney.com/favicon.ico",
|
|
"width": 48,
|
|
"height": 48
|
|
}
|
|
},
|
|
"author": {
|
|
"@type": "Person",
|
|
"name": "Karl Stoney",
|
|
"image": {
|
|
"@type": "ImageObject",
|
|
"url": "https://karlstoney.com/content/images/2020/02/0.jpeg",
|
|
"width": 200,
|
|
"height": 200
|
|
},
|
|
"url": "https://karlstoney.com/author/karl/",
|
|
"sameAs": [
|
|
"https://karlstoney.com",
|
|
"https://x.com/karlstoney"
|
|
]
|
|
},
|
|
"headline": "To Auto Scale or not to Auto Scale, that is the question",
|
|
"url": "https://karlstoney.com/to-auto-scale-or-not-to-auto-scale-that-is-the-question/",
|
|
"datePublished": "2023-10-05T08:41:57.000Z",
|
|
"dateModified": "2024-07-01T08:11:46.000Z",
|
|
"keywords": "Kubernetes",
|
|
"description": "The challenges of Auto Scaling, from cold start impact, tech debt, and cost realities. Prioritising scaling as code and shared responsibility for optimal performance in cloud efficiency.",
|
|
"mainEntityOfPage": "https://karlstoney.com/to-auto-scale-or-not-to-auto-scale-that-is-the-question/"
|
|
}
|
|
</script>
|
|
|
|
<meta name="generator" content="Ghost 6.62">
|
|
<link rel="alternate" type="application/rss+xml" title="karlstoney.com" href="https://karlstoney.com/rss/">
|
|
<script defer src="https://cdn.jsdelivr.net/ghost/portal@~2.71/umd/portal.min.js" data-i18n="true" data-ghost="https://karlstoney.com/" data-key="1fcef56573b2a5779958db5e8c" data-api="https://karlstoney.com/ghost/api/content/" data-locale="en" crossorigin="anonymous"></script><style id="gh-members-styles">.gh-post-upgrade-cta-content,
|
|
.gh-post-upgrade-cta {
|
|
display: flex;
|
|
flex-direction: column;
|
|
align-items: center;
|
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;
|
|
text-align: center;
|
|
width: 100%;
|
|
color: #ffffff;
|
|
font-size: 16px;
|
|
}
|
|
|
|
.gh-post-upgrade-cta-content {
|
|
border-radius: 8px;
|
|
padding: 40px 4vw;
|
|
}
|
|
|
|
.gh-post-upgrade-cta h2 {
|
|
color: #ffffff;
|
|
font-size: 28px;
|
|
letter-spacing: -0.2px;
|
|
margin: 0;
|
|
padding: 0;
|
|
}
|
|
|
|
.gh-post-upgrade-cta p {
|
|
margin: 20px 0 0;
|
|
padding: 0;
|
|
}
|
|
|
|
.gh-post-upgrade-cta small {
|
|
font-size: 16px;
|
|
letter-spacing: -0.2px;
|
|
}
|
|
|
|
.gh-post-upgrade-cta a {
|
|
color: #ffffff;
|
|
cursor: pointer;
|
|
font-weight: 500;
|
|
box-shadow: none;
|
|
text-decoration: underline;
|
|
}
|
|
|
|
.gh-post-upgrade-cta a:hover {
|
|
color: #ffffff;
|
|
opacity: 0.8;
|
|
box-shadow: none;
|
|
text-decoration: underline;
|
|
}
|
|
|
|
.gh-post-upgrade-cta a.gh-btn {
|
|
display: block;
|
|
background: #ffffff;
|
|
text-decoration: none;
|
|
margin: 28px 0 0;
|
|
padding: 8px 18px;
|
|
border-radius: 4px;
|
|
font-size: 16px;
|
|
font-weight: 600;
|
|
}
|
|
|
|
.gh-post-upgrade-cta a.gh-btn:hover {
|
|
opacity: 0.92;
|
|
}</style><script async src="https://js.stripe.com/v3/"></script>
|
|
<script defer src="https://cdn.jsdelivr.net/ghost/sodo-search@~1.8/umd/sodo-search.min.js" data-key="1fcef56573b2a5779958db5e8c" data-styles="https://cdn.jsdelivr.net/ghost/sodo-search@~1.8/umd/main.css" data-sodo-search="https://karlstoney.com/" data-locale="en" crossorigin="anonymous"></script>
|
|
|
|
<link href="https://karlstoney.com/webmentions/receive/" rel="webmention">
|
|
<script defer src="/public/cards.min.js?v=ShRHxgy4po8zN-Wf"></script>
|
|
<link rel="stylesheet" type="text/css" href="/public/cards.min.css?v=WwnU9jw5ancNC8Gc">
|
|
<script defer src="/public/comment-counts.min.js?v=oFYkaGLdiMqB8VN9" data-ghost-comments-counts-api="https://karlstoney.com/members/api/comments/counts/"></script>
|
|
<script defer src="/public/member-attribution.min.js?v=AKG4hWena9j3yX3I"></script><style>:root {--ghost-accent-color: #15171A;}</style>
|
|
<meta name="msvalidate.01" content="6D7C1C7A2BF9A2CF366B3881023DC3BE" />
|
|
|
|
<style type='text/css'>
|
|
|
|
.gh-powered-by {
|
|
display: none;
|
|
}
|
|
|
|
|
|
</style>
|
|
|
|
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.5.1/css/brands.min.css" crossorigin="anonymous" referrerpolicy="no-referrer" />
|
|
|
|
<style>
|
|
.gh-head-menu .nav-linkedin a,
|
|
.gh-head-menu .nav-mastodon a,
|
|
.gh-head-menu .nav-twitter a,
|
|
.gh-head-menu .nav-x a,
|
|
.gh-head-menu .nav-slack a,
|
|
.gh-head-menu .nav-github a {
|
|
font-size: 0 !important;
|
|
}
|
|
|
|
.gh-head-menu .nav-linkedin a::before,
|
|
.gh-head-menu .nav-mastodon a::before,
|
|
.gh-head-menu .nav-twitter a::before,
|
|
.gh-head-menu .nav-x a::before,
|
|
.gh-head-menu .nav-slack a::before,
|
|
.gh-head-menu .nav-github a::before {
|
|
font-family: "Font Awesome 6 Brands";
|
|
display: inline-block;
|
|
font-size: 20px;
|
|
font-style: normal;
|
|
font-weight: normal;
|
|
font-variant: normal;
|
|
text-rendering: auto;
|
|
-webkit-font-smoothing: antialiased;
|
|
}
|
|
|
|
.gh-head-menu .nav-linkedin a::before {content: "\f08c"}
|
|
.gh-head-menu .nav-mastodon a::before {content: "\f4f6"}
|
|
.gh-head-menu .nav-twitter a::before {content: "\f099"}
|
|
.gh-head-menu .nav-x a::before {content: "\e61b"}
|
|
.gh-head-menu .nav-slack a::before {content: "\f198"}
|
|
.gh-head-menu .nav-github a::before {content: "\f09b"}
|
|
</style>
|
|
|
|
</head>
|
|
<body class="post-template tag-kubernetes is-head-left-logo has-cover">
|
|
<div class="viewport">
|
|
|
|
<header id="gh-head" class="gh-head outer">
|
|
<div class="gh-head-inner inner">
|
|
<div class="gh-head-brand">
|
|
<a class="gh-head-logo no-image" href="https://karlstoney.com">
|
|
karlstoney.com
|
|
</a>
|
|
<button class="gh-search gh-icon-btn" aria-label="Search this site" data-ghost-search><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2" width="20" height="20"><path stroke-linecap="round" stroke-linejoin="round" d="M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z"></path></svg></button>
|
|
<button class="gh-burger" aria-label="Main Menu"></button>
|
|
</div>
|
|
|
|
<nav class="gh-head-menu">
|
|
<ul class="nav">
|
|
<li class="nav-home"><a href="https://karlstoney.com/">Home</a></li>
|
|
<li class="nav-about"><a href="https://karlstoney.com/about/">About</a></li>
|
|
<li class="nav-email"><a href="https://karlstoney.com/contact/">Email</a></li>
|
|
<li class="nav-github"><a href="https://github.com/Stono">Github</a></li>
|
|
<li class="nav-twitter"><a href="https://twitter.com/karlstoney">Twitter</a></li>
|
|
<li class="nav-linkedin"><a href="https://www.linkedin.com/in/karl-stoney-94995548/">LinkedIn</a></li>
|
|
</ul>
|
|
|
|
</nav>
|
|
|
|
<div class="gh-head-actions">
|
|
<button class="gh-search gh-icon-btn" aria-label="Search this site" data-ghost-search><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2" width="20" height="20"><path stroke-linecap="round" stroke-linejoin="round" d="M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z"></path></svg></button>
|
|
<div class="gh-head-members">
|
|
<a class="gh-head-link" href="#/portal/signin" data-portal="signin">Sign in</a>
|
|
<a class="gh-head-button" href="#/portal/signup" data-portal="signup">Subscribe</a>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</header>
|
|
|
|
<div class="site-content">
|
|
|
|
|
|
|
|
|
|
<main id="site-main" class="site-main">
|
|
<article class="article post tag-kubernetes no-image ">
|
|
|
|
<header class="article-header gh-canvas">
|
|
|
|
<div class="article-tag post-card-tags">
|
|
<span class="post-card-primary-tag">
|
|
<a href="/tag/kubernetes/">Kubernetes</a>
|
|
</span>
|
|
</div>
|
|
|
|
<h1 class="article-title">To Auto Scale or not to Auto Scale, that is the question</h1>
|
|
|
|
<p class="article-excerpt">The challenges of Auto Scaling, from cold start impact, tech debt, and cost realities. Prioritising scaling as code and shared responsibility for optimal performance in cloud efficiency.</p>
|
|
|
|
<div class="article-byline">
|
|
<section class="article-byline-content">
|
|
|
|
<ul class="author-list instapaper_ignore">
|
|
<li class="author-list-item">
|
|
<a href="/author/karl/" class="author-avatar" aria-label="Read more of Karl Stoney">
|
|
<img class="author-profile-image" src="/content/images/size/w100/2020/02/0.jpeg" alt="Karl Stoney" loading="eager" />
|
|
</a>
|
|
</li>
|
|
</ul>
|
|
|
|
<div class="article-byline-meta">
|
|
<h4 class="author-name"><a href="/author/karl/">Karl Stoney</a></h4>
|
|
<div class="byline-meta-content">
|
|
<time class="byline-meta-date" datetime="2023-10-05">05 Oct 2023</time>
|
|
<span class="byline-reading-time"><span class="bull">•</span> 6 min read</span>
|
|
</div>
|
|
</div>
|
|
|
|
</section>
|
|
<a href="#/share" class="gh-button gh-button-share">Share</a>
|
|
</div>
|
|
|
|
|
|
</header>
|
|
|
|
<section class="gh-content gh-canvas">
|
|
<p>If you've read my more recent blog post around <a href="https://karlstoney.com/scaling-and-sizing-the-sidecar/">Scaling the Sidecar</a>, you'll have picked up on the fact that I actually don't like Auto Scaling our workloads. Today I'm going to dig into a little bit more why that's the case, based on the types of workloads that we run at Auto Trader. </p><p>To be clear, I'm not sat here saying that you shouldn't Auto Scale either. Make your own decisions on what's right for your problem space. You could also read this blog from an inverse perspective of "Challenges to tackle if you want to Auto Scale" too. </p><h3 id="cold-starts-hurt-latency">Cold Starts hurt Latency</h3><p>I admit this one is our own doing, but we have a lot of workloads that rely on Just in Time compilation (e.g JVM). That means when instances of those services come up, there's a reasonable period (some seconds or minutes) where latencies are higher, because the code is being compiled on the fly. That means Pod Churn negatively impacts our users. There are ways to minimise the impact of this (some more extreme than others), for example:</p><ul><li>Using warmupDurationSeconds on a <a href="https://istio.io/latest/docs/reference/config/networking/destination-rule/?ref=karlstoney.com#LoadBalancerSettings">DestinationRule</a>, to gradually ramp up traffic.</li><li>Using <a href="https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/?ref=karlstoney.com">Kubernetes Start-up</a> Probes to send some requests to known hot paths, before the pod becomes Ready.</li><li>If you're a heavy JVM user like us, look at native compilation like <a href="https://www.graalvm.org/?ref=karlstoney.com">graal</a>, however our experience tells us whilst this helps with startup, JIT performs better once fully warmed.</li><li>There are enterprise JVM offerings like <a href="https://www.azul.com/products/prime/?ref=karlstoney.com#features">Azul prime</a>, which offer effectively a distributed cache for JIT compilation. But they're quite costly i've found (negating the cost savings of Auto Scaling).</li><li>Pivot to languages that are more aligned with Auto Scaling (native compilation and fast startups - golang, rust, etc).</li></ul><p>You'll need to do a lot of these anyway to thrive in a highly ephemeral environment such as kubernetes. There are many reasons pods restart (evictions, deployments, etc). We've invested a lot in ensuring our workloads do this but even so, we still prefer to minimise that pod churn during our peak consumer hours when load is at its highest.</p><h3 id="scaling-masks-problems">Scaling Masks Problems</h3><p>Performance regressions get introduced all the time, some subtle and compound over time, others are obvious and immediate. Without a doubt the most common response I see to performance problems is "lets scale it". It's quick, it's easy, and it generally works (up to a point). </p><p>I however see this as Tech Debt, but even worse it's Tech Debt that you're not actively tracking. That will compound over time and eventually you'll be at a breaking point. It might be service performance, or simply cost of scaling up constantly, but you'll be unpicking potentially years of regressions.</p><p>I always encourage scaling <em>after</em> you've understood the performance of your service, not before. More often than not this simple mentality switch results in the issue being resolved without scaling. </p><p>Here's a real world example for you. One of our Vehicle Metrics team had deployed an ML API (in Python) and started routing traffic to it. Response times weren't quite where they wanted to be and their initial instinct was to scale. Without any data there was a belief that ML APIs are just inherently CPU intensive so we'd need to scale. I challenged them to show that with Data.</p><p>Python is relatively immature here in terms of our runtime debug capability, so getting that data wasn't actually an option for them. This is where the Platform Team fit into our organisation. We worked with them to build a capability to generate flame graphs (with <a href="https://github.com/benfred/py-spy?ref=karlstoney.com">py-spy</a>) from running Python pods. This requires running a privileged <a href="https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/?ref=karlstoney.com">debug container</a>, which we exposed behind our developer dashboard:</p><figure class="kg-card kg-image-card kg-card-hascaption"><img src="https://karlstoney.com/content/images/2023/10/Screenshot-2023-10-05-at-09.07.08.png" class="kg-image" alt="" loading="lazy" width="1460" height="578" srcset="https://karlstoney.com/content/images/size/w600/2023/10/Screenshot-2023-10-05-at-09.07.08.png 600w, https://karlstoney.com/content/images/size/w1000/2023/10/Screenshot-2023-10-05-at-09.07.08.png 1000w, https://karlstoney.com/content/images/2023/10/Screenshot-2023-10-05-at-09.07.08.png 1460w" sizes="(min-width: 720px) 720px"><figcaption><span style="white-space: pre-wrap;">Ability to grab py-spy samples for production workloads</span></figcaption></figure><p>Upon digging into the flame graphs, it became obvious a significant amount of time was spent doing in-memory sorts. They realised that in their model output, they'd missed an index.</p><figure class="kg-card kg-image-card kg-card-hascaption"><img src="https://karlstoney.com/content/images/2023/10/Screenshot-2023-10-05-at-08.59.35.png" class="kg-image" alt="" loading="lazy" width="1714" height="612" srcset="https://karlstoney.com/content/images/size/w600/2023/10/Screenshot-2023-10-05-at-08.59.35.png 600w, https://karlstoney.com/content/images/size/w1000/2023/10/Screenshot-2023-10-05-at-08.59.35.png 1000w, https://karlstoney.com/content/images/size/w1600/2023/10/Screenshot-2023-10-05-at-08.59.35.png 1600w, https://karlstoney.com/content/images/2023/10/Screenshot-2023-10-05-at-08.59.35.png 1714w" sizes="(min-width: 720px) 720px"><figcaption><span style="white-space: pre-wrap;">Response times halved</span></figcaption></figure><figure class="kg-card kg-image-card kg-card-hascaption"><img src="https://karlstoney.com/content/images/2023/10/Screenshot-2023-10-05-at-08.59.43.png" class="kg-image" alt="" loading="lazy" width="1708" height="616" srcset="https://karlstoney.com/content/images/size/w600/2023/10/Screenshot-2023-10-05-at-08.59.43.png 600w, https://karlstoney.com/content/images/size/w1000/2023/10/Screenshot-2023-10-05-at-08.59.43.png 1000w, https://karlstoney.com/content/images/size/w1600/2023/10/Screenshot-2023-10-05-at-08.59.43.png 1600w, https://karlstoney.com/content/images/2023/10/Screenshot-2023-10-05-at-08.59.43.png 1708w" sizes="(min-width: 720px) 720px"><figcaption><span style="white-space: pre-wrap;">CPU time halved</span></figcaption></figure><p>A simple one line fix in their code halved response time, and cpu time. And the problem was solved, <strong>without scaling</strong>. But on top of that, we'd added a capability to the platform that meant future python application developers could now self serve data that enables them to debug performance problems too. They talked about this in a brown bag session which helps to grow a culture of valuing performance profiling.</p><figure class="kg-card kg-image-card kg-card-hascaption"><img src="https://karlstoney.com/content/images/2023/10/Screenshot-2023-10-05-at-09.31.22.png" class="kg-image" alt="" loading="lazy" width="1868" height="874" srcset="https://karlstoney.com/content/images/size/w600/2023/10/Screenshot-2023-10-05-at-09.31.22.png 600w, https://karlstoney.com/content/images/size/w1000/2023/10/Screenshot-2023-10-05-at-09.31.22.png 1000w, https://karlstoney.com/content/images/size/w1600/2023/10/Screenshot-2023-10-05-at-09.31.22.png 1600w, https://karlstoney.com/content/images/2023/10/Screenshot-2023-10-05-at-09.31.22.png 1868w" sizes="(min-width: 720px) 720px"><figcaption><span style="white-space: pre-wrap;">Debugging capabilities provided by Auto Trader platform</span></figcaption></figure><p>So as you can see here for me, scaling something should generally be a conscious decision. Therefore Auto Scaling is automating a decision that I personally don't believe should be automated. Obviously the natural workaround to this would be to set an upper limit on your Auto Scaler but I find this just reinforces a culture of scale first, whereas I prefer to see debug first, scale later.</p><p>Because of this, scaling at Auto Trader is done as code, it's committed into the Git repository, goes through CI, runs through any tests, and is deployed to multiple environments before prod. The reason for scaling is cemented in the Git message, and we have automation that shares in slack that the scale happened. That slack message tags all the owners of the service - to highlight the fact it happened, and if needs be trigger a "why" type conversation.</p><figure class="kg-card kg-image-card kg-card-hascaption"><img src="https://karlstoney.com/content/images/2023/10/Screenshot-2023-10-05-at-07.29.08.png" class="kg-image" alt="" loading="lazy" width="1102" height="378" srcset="https://karlstoney.com/content/images/size/w600/2023/10/Screenshot-2023-10-05-at-07.29.08.png 600w, https://karlstoney.com/content/images/size/w1000/2023/10/Screenshot-2023-10-05-at-07.29.08.png 1000w, https://karlstoney.com/content/images/2023/10/Screenshot-2023-10-05-at-07.29.08.png 1102w" sizes="(min-width: 720px) 720px"><figcaption><span style="white-space: pre-wrap;">Slack message showing a scale event</span></figcaption></figure><h3 id="youre-at-the-mercy-of-infrastructure-capacity">You're at the mercy of Infrastructure Capacity</h3><figure class="kg-card kg-image-card kg-card-hascaption"><img src="https://karlstoney.com/content/images/2023/10/Screenshot-2023-10-05-at-08.35.05.png" class="kg-image" alt="" loading="lazy" width="1716" height="618" srcset="https://karlstoney.com/content/images/size/w600/2023/10/Screenshot-2023-10-05-at-08.35.05.png 600w, https://karlstoney.com/content/images/size/w1000/2023/10/Screenshot-2023-10-05-at-08.35.05.png 1000w, https://karlstoney.com/content/images/size/w1600/2023/10/Screenshot-2023-10-05-at-08.35.05.png 1600w, https://karlstoney.com/content/images/2023/10/Screenshot-2023-10-05-at-08.35.05.png 1716w" sizes="(min-width: 720px) 720px"><figcaption><span style="white-space: pre-wrap;">Seasonality of Consumer Services at Auto Trader</span></figcaption></figure><p>Above shows you the traffic pattern of one of our services. You can see we exhibit a significant seasonality in our traffic patterns, with our peak being in the evening and it being a good 80% lower by 2am. On paper this type of traffic pattern is perfect for auto scaling, it gradually ramps up and down.</p><p>In the real world however, if you're auto scaling your Deployment, and your underlying Kubernetes cluster nodes, then your ability to add new nodes to your cluster is dependent on their being availability of the compute machine types you're requesting from your cloud provider. This happens, on Google Cloud Platform at least, more than you'd believe. There's even a blog from a <a href="https://www.googlecloudcommunity.com/gc/Community-Blogs/Managing-Capacity-Quota-and-Stockouts-in-the-Cloud-Concepts-and/ba-p/464770?ref=karlstoney.com">Google TAM</a> here talking about it and how to plan for it. </p><p>Whilst experimenting with Auto Scaling we experienced situations where we were unable to scale up to meet customer demand gradually ramping up, despite running in 3 availability zones.</p><p>Stock-outs in GCP happen at the Cell level (groups of infra within a zone), then the Zone, then the Region. There's a significant lack of transparency around Capacity and Availability. GCP Support are instructed to provide no timelines for resolution and simply tell you to spin up more capacity in another region for example - as if it's that simple to do.</p><p>The ultimate solution proposed by Google is to buy<a href="https://cloud.google.com/compute/docs/instances/reservations-overview?ref=karlstoney.com"> Compute Reservations</a>, the incredible irony here is that you're then pre-buying compute you expect to use to guarantee it's availability. At full price. Immediately negative a any cost savings of auto scaling. </p><h3 id="you-dont-actually-save-that-much-money">You don't actually save that much money!</h3><p>I know, I know. At least on Google for us when we add up:</p><ul><li><a href="https://cloud.google.com/compute/docs/instances/signing-up-committed-use-discounts?ref=karlstoney.com">Committed use discounts</a> - where you commit to buying resources up front</li><li><a href="https://cloud.google.com/compute/docs/sustained-use-discounts?ref=karlstoney.com">Sustained use discounts</a> - where you get a discount for using compute constantly</li><li>Committed spend discounts - where you negotiate a discount based on spend over longer term contracts</li></ul><p>You can achieve sufficiently high levels of discount on committed use that in our case, we almost entirely offset the savings from a more opex/ephemeral usage pattern. </p><p>If you follow the <a href="https://www.finops.org/framework/principles/?ref=karlstoney.com">FinOps princples</a> you'll be familiar with the concept that cloud usage is a shared responsibility model. In our organisation, we don't internally charge teams for their cloud usage, we want them to focus on building products. The way the shared responsibility model manifests for us is that the teams right-size their workloads, care about performance, scaling after investigation, not before.</p><h3 id="summary">Summary</h3><p>Auto-scale or not, that's up to you. But nothing in life is free, and auto scaling is not a silver bullet.</p>
|
|
</section>
|
|
|
|
<section class="article-comments gh-canvas">
|
|
|
|
<script defer src="https://cdn.jsdelivr.net/ghost/comments-ui@~1.6/umd/comments-ui.min.js" data-locale="en" data-ghost-comments="https://karlstoney.com/" data-api="https://karlstoney.com/ghost/api/content/" data-admin="https://karlstoney.com/ghost/" data-key="1fcef56573b2a5779958db5e8c" data-title="null" data-count="true" data-post-id="651e634b66e4200012a256c4" data-color-scheme="auto" data-avatar-saturation="60" data-accent-color="#15171A" data-comments-enabled="all" data-publication="karlstoney.com" crossorigin="anonymous"></script>
|
|
|
|
</section>
|
|
|
|
</article>
|
|
</main>
|
|
|
|
|
|
|
|
|
|
<aside class="read-more-wrap outer">
|
|
<div class="read-more inner">
|
|
|
|
<article class="post-card post featured no-image keep-ratio">
|
|
|
|
|
|
<div class="post-card-content">
|
|
|
|
<a class="post-card-content-link" href="/from-pr-review-bot-atai-cli-to-an-autonomous-work-queue/">
|
|
<header class="post-card-header">
|
|
<div class="post-card-tags">
|
|
<span class="post-card-featured"><svg width="16" height="17" viewBox="0 0 16 17" fill="none" xmlns="http://www.w3.org/2000/svg">
|
|
<path d="M4.49365 4.58752C3.53115 6.03752 2.74365 7.70002 2.74365 9.25002C2.74365 10.6424 3.29678 11.9778 4.28134 12.9623C5.26591 13.9469 6.60127 14.5 7.99365 14.5C9.38604 14.5 10.7214 13.9469 11.706 12.9623C12.6905 11.9778 13.2437 10.6424 13.2437 9.25002C13.2437 6.00002 10.9937 3.50002 9.16865 1.68127L6.99365 6.25002L4.49365 4.58752Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path>
|
|
</svg> Featured</span>
|
|
</div>
|
|
<h2 class="post-card-title">
|
|
From PR Review Bot + ATAI CLI to an Autonomous Work Queue
|
|
</h2>
|
|
</header>
|
|
<div class="post-card-excerpt">Taking our CLI, a Kubernetes Cluster and an Orchestrator to build an autonomous AI developer agent.</div>
|
|
</a>
|
|
|
|
<footer class="post-card-meta">
|
|
<time class="post-card-meta-date" datetime="2026-04-17">17 Apr 2026</time>
|
|
<span class="post-card-meta-length">8 min read</span>
|
|
<script
|
|
data-ghost-comment-count="69de00ffd764770001f6c8dd"
|
|
data-ghost-comment-count-empty=""
|
|
data-ghost-comment-count-singular="comment"
|
|
data-ghost-comment-count-plural="comments"
|
|
data-ghost-comment-count-tag="span"
|
|
data-ghost-comment-count-class-name=""
|
|
data-ghost-comment-count-autowrap="true"
|
|
>
|
|
</script>
|
|
</footer>
|
|
|
|
</div>
|
|
|
|
</article>
|
|
|
|
<article class="post-card post featured no-image keep-ratio">
|
|
|
|
|
|
<div class="post-card-content">
|
|
|
|
<a class="post-card-content-link" href="/building-atai-for-copilot/">
|
|
<header class="post-card-header">
|
|
<div class="post-card-tags">
|
|
<span class="post-card-featured"><svg width="16" height="17" viewBox="0 0 16 17" fill="none" xmlns="http://www.w3.org/2000/svg">
|
|
<path d="M4.49365 4.58752C3.53115 6.03752 2.74365 7.70002 2.74365 9.25002C2.74365 10.6424 3.29678 11.9778 4.28134 12.9623C5.26591 13.9469 6.60127 14.5 7.99365 14.5C9.38604 14.5 10.7214 13.9469 11.706 12.9623C12.6905 11.9778 13.2437 10.6424 13.2437 9.25002C13.2437 6.00002 10.9937 3.50002 9.16865 1.68127L6.99365 6.25002L4.49365 4.58752Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path>
|
|
</svg> Featured</span>
|
|
</div>
|
|
<h2 class="post-card-title">
|
|
Building ATAI: Our Opinionated CLI for Copilot at Scale
|
|
</h2>
|
|
</header>
|
|
<div class="post-card-excerpt">How we wrapped Copilot, to accelerate the use of AI engineering tools at Autotrader</div>
|
|
</a>
|
|
|
|
<footer class="post-card-meta">
|
|
<time class="post-card-meta-date" datetime="2026-04-14">14 Apr 2026</time>
|
|
<span class="post-card-meta-length">8 min read</span>
|
|
<script
|
|
data-ghost-comment-count="69de1228d764770001f6c8e7"
|
|
data-ghost-comment-count-empty=""
|
|
data-ghost-comment-count-singular="comment"
|
|
data-ghost-comment-count-plural="comments"
|
|
data-ghost-comment-count-tag="span"
|
|
data-ghost-comment-count-class-name=""
|
|
data-ghost-comment-count-autowrap="true"
|
|
>
|
|
</script>
|
|
</footer>
|
|
|
|
</div>
|
|
|
|
</article>
|
|
|
|
<article class="post-card post no-image keep-ratio">
|
|
|
|
|
|
<div class="post-card-content">
|
|
|
|
<a class="post-card-content-link" href="/building-a-pr-review-agent/">
|
|
<header class="post-card-header">
|
|
<div class="post-card-tags">
|
|
</div>
|
|
<h2 class="post-card-title">
|
|
Building a PR Review Agent
|
|
</h2>
|
|
</header>
|
|
<div class="post-card-excerpt">Building a PR review agent using Gemini CLI on Kubernetes</div>
|
|
</a>
|
|
|
|
<footer class="post-card-meta">
|
|
<time class="post-card-meta-date" datetime="2025-10-08">08 Oct 2025</time>
|
|
<span class="post-card-meta-length">11 min read</span>
|
|
<script
|
|
data-ghost-comment-count="68c29ebee76f0800016b2076"
|
|
data-ghost-comment-count-empty=""
|
|
data-ghost-comment-count-singular="comment"
|
|
data-ghost-comment-count-plural="comments"
|
|
data-ghost-comment-count-tag="span"
|
|
data-ghost-comment-count-class-name=""
|
|
data-ghost-comment-count-autowrap="true"
|
|
>
|
|
</script>
|
|
</footer>
|
|
|
|
</div>
|
|
|
|
</article>
|
|
</div>
|
|
</aside>
|
|
|
|
|
|
|
|
</div>
|
|
|
|
<footer class="site-footer outer">
|
|
<div class="inner">
|
|
<section class="copyright"><a href="https://karlstoney.com">karlstoney.com</a> © 2026</section>
|
|
<div class="site-footer-center">
|
|
<div class="site-footer-social-links">
|
|
<a href="https://x.com/karlstoney" target="_blank" rel="noopener" aria-label="X">
|
|
<svg class="icon" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
|
|
<path d="M18.2439 2.25H21.5519L14.3249 10.51L22.8269 21.75H16.1699L10.9559 14.933L4.98991 21.75H1.67991L9.40991 12.915L1.25391 2.25H8.07991L12.7929 8.481L18.2439 2.25ZM17.0829 19.77H18.9159L7.08391 4.126H5.11691L17.0829 19.77Z" fill="currentColor"/>
|
|
</svg> </a>
|
|
</div>
|
|
<nav class="site-footer-nav">
|
|
<ul class="nav">
|
|
<li class="nav-twitter"><a href="https://twitter.com/karlstoney">Twitter</a></li>
|
|
<li class="nav-github"><a href="https://github.com/Stono">Github</a></li>
|
|
<li class="nav-linkedin"><a href="https://www.linkedin.com/in/karl-stoney-94995548/">LinkedIn</a></li>
|
|
</ul>
|
|
|
|
</nav>
|
|
</div>
|
|
<div class="gh-powered-by"><a href="https://ghost.org/" target="_blank" rel="noopener">Powered by Ghost</a></div>
|
|
</div>
|
|
</footer>
|
|
|
|
</div>
|
|
|
|
<div class="pswp" tabindex="-1" role="dialog" aria-hidden="true">
|
|
<div class="pswp__bg"></div>
|
|
|
|
<div class="pswp__scroll-wrap">
|
|
<div class="pswp__container">
|
|
<div class="pswp__item"></div>
|
|
<div class="pswp__item"></div>
|
|
<div class="pswp__item"></div>
|
|
</div>
|
|
|
|
<div class="pswp__ui pswp__ui--hidden">
|
|
<div class="pswp__top-bar">
|
|
<div class="pswp__counter"></div>
|
|
|
|
<button class="pswp__button pswp__button--close" title="Close (Esc)"></button>
|
|
<button class="pswp__button pswp__button--share" title="Share"></button>
|
|
<button class="pswp__button pswp__button--fs" title="Toggle fullscreen"></button>
|
|
<button class="pswp__button pswp__button--zoom" title="Zoom in/out"></button>
|
|
|
|
<div class="pswp__preloader">
|
|
<div class="pswp__preloader__icn">
|
|
<div class="pswp__preloader__cut">
|
|
<div class="pswp__preloader__donut"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="pswp__share-modal pswp__share-modal--hidden pswp__single-tap">
|
|
<div class="pswp__share-tooltip"></div>
|
|
</div>
|
|
|
|
<button class="pswp__button pswp__button--arrow--left" title="Previous (arrow left)"></button>
|
|
<button class="pswp__button pswp__button--arrow--right" title="Next (arrow right)"></button>
|
|
|
|
<div class="pswp__caption">
|
|
<div class="pswp__caption__center"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<script src="/assets/built/casper.js?v=YtPhrez3GuGLVDYs" defer></script>
|
|
|
|
<script>
|
|
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
|
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
|
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
|
})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
|
|
|
|
ga('create', 'UA-37161145-1', 'auto');
|
|
ga('send', 'pageview');
|
|
</script>
|
|
|
|
<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" data-cf-beacon='{"version":"2024.11.0","token":"5b355c7ee12445ab8f857265175fa35d","r":1,"spa":2}' crossorigin="anonymous"></script>
|
|
</body>
|
|
</html>
|