584 lines
28 KiB
HTML
584 lines
28 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en-us" prefix="og: http://ogp.me/ns#"><head>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
|
|
|
<link rel="icon" type="image/png" href="/favicon.ico">
|
|
|
|
|
|
<title>Serverless: Cold Start War | Mikhail Shilkov</title>
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.13.1/css/all.min.css" crossorigin="anonymous">
|
|
<link href="https://fonts.googleapis.com/css?family=Righteous%7CMerriweather:300,300i,400,400i,700,700i" rel="stylesheet">
|
|
|
|
<link rel="stylesheet" href="/sass/main.css">
|
|
|
|
|
|
|
|
<meta property="og:title" content="Serverless: Cold Start War" />
|
|
<meta property="og:description" content="Comparison of cold start statistics for FaaS across AWS, Azure and GCP" />
|
|
<meta property="og:type" content="article" />
|
|
<meta property="og:url" content="https://mikhail.io/2018/08/serverless-cold-start-war/" />
|
|
|
|
|
|
|
|
|
|
|
|
<meta property="og:image" content="https://mikhail.io/2018/08/serverless-cold-start-war/teaser.jpg" />
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<meta property="article:tag" content="Azure" />
|
|
|
|
<meta property="article:tag" content="Azure Functions" />
|
|
|
|
<meta property="article:tag" content="Serverless" />
|
|
|
|
<meta property="article:tag" content="Performance" />
|
|
|
|
<meta property="article:tag" content="Cold Starts" />
|
|
|
|
<meta property="article:tag" content="AWS" />
|
|
|
|
|
|
|
|
|
|
<meta name="twitter:card" content="summary_large_image"/>
|
|
|
|
<meta name="twitter:image" content="https://mikhail.io/2018/08/serverless-cold-start-war/teaser.jpg" />
|
|
|
|
|
|
|
|
<meta name="twitter:title" content="Serverless: Cold Start War"/>
|
|
<meta name="twitter:description" content="Comparison of cold start statistics for FaaS across AWS, Azure and GCP"/>
|
|
<meta name="twitter:creator" content="@MikhailShilkov"></meta>
|
|
</head>
|
|
<body><script type="text/javascript" src="https://www.gstatic.com/charts/loader.js"></script>
|
|
<script type="text/javascript">
|
|
google.charts.load("current", {packages:["corechart"]});
|
|
function addChart(make) {
|
|
google.charts.setOnLoadCallback(drawChart);
|
|
function drawChart() {
|
|
const data = new google.visualization.DataTable();
|
|
|
|
const options = {
|
|
height: 420,
|
|
chartArea: { width: '85%', height: '70%' },
|
|
legend: 'none',
|
|
hAxis: { minValue: 0 },
|
|
vAxis: {},
|
|
series: {
|
|
0: { tooltip : false}
|
|
}
|
|
};
|
|
const chart = make(data, options);
|
|
options.hAxis.textStyle = options.hAxis.titleTextStyle
|
|
= options.vAxis.textStyle = options.vAxis.titleTextStyle = { fontName: 'Merriweather' };
|
|
|
|
chart.draw(data, options);
|
|
}
|
|
}
|
|
</script>
|
|
<nav class="navbar navbar-expand-lg navbar-light bg-light">
|
|
<div class="container pr-0">
|
|
<a class="navbar-brand" href="/">
|
|
<img class="author-thumb" src="/images/author.jpg" alt="Mikhail Shilkov">
|
|
<span class="text-primary">Mikhail Shilkov</span>
|
|
</a>
|
|
|
|
|
|
|
|
|
|
|
|
<button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarMediumish" aria-controls="navbarSupportedContent"
|
|
aria-expanded="false" aria-label="Toggle navigation">
|
|
<span class="navbar-toggler-icon"></span>
|
|
</button>
|
|
|
|
|
|
<div class="collapse navbar-collapse" id="navbarMediumish">
|
|
|
|
<ul class="navbar-nav ml-auto">
|
|
|
|
<li class="nav-item ">
|
|
<a class="nav-link" href="/tags/">TOPICS</a>
|
|
</li>
|
|
|
|
<li class="nav-item ">
|
|
<a class="nav-link" href="/archives/">ARCHIVES</a>
|
|
</li>
|
|
|
|
<li class="nav-item ">
|
|
<a class="nav-link" href="/talks/">TALKS</a>
|
|
</li>
|
|
|
|
<li class="nav-item ">
|
|
<a class="nav-link" href="/about/">ABOUT</a>
|
|
</li>
|
|
<li class="nav-item">
|
|
<a class="nav-link" href="https://mikhail.io/feed/"><i class="fas fa-rss social-icon" aria-hidden="true"></i></a>
|
|
</li>
|
|
|
|
<li class="nav-item">
|
|
<a class="nav-link" href="https://x.com/MikhailShilkov"><i class="fab fa-twitter social-icon" aria-hidden="true"></i></a>
|
|
</li>
|
|
|
|
|
|
<li class="nav-item">
|
|
<a class="nav-link" href="https://dev.to/mikhailshilkov"><i class="fab fa-dev social-icon" aria-hidden="true"></i></a>
|
|
</li>
|
|
|
|
|
|
<li class="nav-item">
|
|
<a class="nav-link" href="https://medium.com/@MikhailShilkov"><i class="fab fa-medium social-icon" aria-hidden="true"></i></a>
|
|
</li>
|
|
|
|
|
|
<li class="nav-item">
|
|
<a class="nav-link" href="https://github.com/MikhailShilkov"><i class="fab fa-github social-icon" aria-hidden="true"></i></a>
|
|
</li>
|
|
|
|
|
|
<li class="nav-item">
|
|
<a class="nav-link" href="https://www.linkedin.com/in/MikhailShilkov"><i class="fab fa-linkedin social-icon" aria-hidden="true"></i></a>
|
|
</li>
|
|
|
|
</ul>
|
|
</div>
|
|
|
|
</div>
|
|
</nav>
|
|
|
|
|
|
<div class="container">
|
|
|
|
<div class="main-content">
|
|
|
|
<div class="container">
|
|
<div class="row">
|
|
|
|
<div class="col-lg-2 pl-0"><div class="share">
|
|
<ul>
|
|
<li class="ml-1 mr-1" title="Say 'Thank You' for this article">
|
|
<a href="#" onclick="heart();return false;">
|
|
<i class="fas fa-heart" style="color: red"></i>
|
|
</a>
|
|
<div class="count" style="float: right; padding-left: .5em; padding-top: .25em" id="heartcount"></div>
|
|
</li>
|
|
|
|
<li class="ml-1 mr-1" title="Tweet this article">
|
|
<a target="_blank"
|
|
href="https://x.com/intent/tweet?text=Serverless%3a%20Cold%20Start%20War%20by%20@MikhailShilkov&url=https%3a%2f%2fmikhail.io%2f2018%2f08%2fserverless-cold-start-war%2f"
|
|
onclick="window.open(this.href, 'twitter-share', 'width=550,height=435');return false;">
|
|
<i class="fab fa-twitter"></i>
|
|
</a>
|
|
</li>
|
|
|
|
<li class="ml-1 mr-1" title="See the responses or write a response">
|
|
<a href="#comments">
|
|
<i class="fas fa-comment"></i>
|
|
</a>
|
|
</li>
|
|
</ul>
|
|
</div></div>
|
|
|
|
<div class="col-lg-9 flex-first flex-lg-unordered">
|
|
|
|
<div class="mainheading">
|
|
|
|
<h1 class="posttitle">Serverless: Cold Start War</h1>
|
|
|
|
<div class="post-top-meta">
|
|
<div>
|
|
|
|
|
|
<span class="post-date">Published on Aug 30, 2018
|
|
|
|
|
|
|
|
· 10 min read</span></span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
<div class="article-post">
|
|
<p><em>The newer and much more detailed cold start comparison is now available: <a href="/serverless/coldstarts/">Cold Starts in Serverless Functions</a></em></p>
|
|
<p>Serverless cloud services are hot. Except when they are not :)</p>
|
|
<p>AWS Lambda, Azure Functions, Google Cloud Functions are all similar in their attempt
|
|
to enable rapid development of cloud-native serverless applications.</p>
|
|
<p>Auto-provisioning and auto-scalability are the killer features of those Function-as-a-Service
|
|
cloud offerings. No management required, cloud providers will deliver infrastructure for the user
|
|
based on the actual incoming load.</p>
|
|
<p>One drawback of such dynamic provisioning is a phenomenon called “cold start”. Basically,
|
|
applications that haven’t been used for a while take longer to startup and to handle the
|
|
first request.</p>
|
|
<p>Cloud providers keep a bunch of generic unspecialized workers in stock. Whenever a serverless
|
|
application needs to scale up, be it from 0 to 1 instances, or from N to N+1 likewise, the runtime
|
|
will pick one of the spare workers and will configure it to serve the named application:</p>
|
|
<p><img src="coldstart.png" alt="Cold Start"></p>
|
|
<p>This procedure takes time, so the latency of the application event handling increases. To avoid
|
|
doing this for every event, the specialized worker will be kept intact for some period of time.
|
|
When another event comes in, this worker will stand available to process it as soon as possible.
|
|
This is a “warm start”:</p>
|
|
<p><img src="warmstart.png" alt="Warm Start"></p>
|
|
<p>The problem of cold start latency was described multiple times, here are the notable links:</p>
|
|
<ul>
|
|
<li><a href="https://blogs.msdn.microsoft.com/appserviceteam/2018/02/07/understanding-serverless-cold-start/">Understanding Serverless Cold Start</a></li>
|
|
<li><a href="https://hackernoon.com/cold-starts-in-aws-lambda-f9e3432adbf0">Everything you need to know about cold starts in AWS Lambda</a></li>
|
|
<li><a href="https://serverless.com/blog/keep-your-lambdas-warm/">Keeping Functions Warm</a></li>
|
|
<li><a href="https://theburningmonk.com/2018/01/im-afraid-youre-thinking-about-aws-lambda-cold-starts-all-wrong/">I’m afraid you’re thinking about AWS Lambda cold starts all wrong</a></li>
|
|
</ul>
|
|
<p>The goal of my article today is to explore how cold starts compare:</p>
|
|
<ul>
|
|
<li>Across Big-3 cloud providers (Amazon, Microsoft, Google)</li>
|
|
<li>For different languages and runtimes</li>
|
|
<li>For smaller vs larger applications (including dependencies)</li>
|
|
<li>How often cold starts happen</li>
|
|
<li>What can be done to optimize the cold starts</li>
|
|
</ul>
|
|
<p>Let’s see how I did that and what the outcome was.</p>
|
|
<p><em>DISCLAIMER. Performance testing is hard. I might be missing some important factors and parameters that
|
|
influence the outcome. My interpretation might be wrong. The results might change over time. If you happen
|
|
to know a way to improve my tests, please let me know and I will re-run them and re-publish the results.</em></p>
|
|
<h2 id="methodology">Methodology</h2>
|
|
<p>All tests were run against HTTP Functions because that’s where cold start matters the most.</p>
|
|
<p>All the functions were returning a simple JSON reporting their current instance ID, language etc.
|
|
Some functions were also loading extra dependencies, see below.</p>
|
|
<p>I did not rely on execution time reported by a cloud provider. Instead, I measured end-to-end duration from
|
|
the client perspective. This means that durations of HTTP gateway (e.g. API Gateway in case of AWS) are included
|
|
into the total duration. However, all calls were made from within the same region, so network latency should
|
|
have minimal impact:</p>
|
|
<p><img src="test-setup.png" alt="Test Setup"></p>
|
|
<p>Important note: I ran all my tests on GA (generally available) versions of services/languages, so e.g.
|
|
Azure tests were done with version 1 of Functions runtime (.NET Framework), and GCP tests were only made for
|
|
Javascript runtime.</p>
|
|
<h2 id="when-does-cold-start-happen">When Does Cold Start Happen?</h2>
|
|
<p>Obviously, cold start happens when the very first request comes in. After that request is processed,
|
|
the instance is kept alive in case subsequent requests arrive. But for how long?</p>
|
|
<p>The answer differs between cloud providers.</p>
|
|
<p>To help you read the charts in this section, I’ve marked cold starts with blue color dots, and warm starts
|
|
with orange color dots.</p>
|
|
<h3 id="azure">Azure</h3>
|
|
<p>Here is the chart for Azure. It shows the values of normalized request durations across
|
|
different languages and runtime versions (Y-axis) depending on the time since the previous
|
|
request in minutes (X-axis):</p>
|
|
<p><img src="azure-coldstart-threshold.png" alt="Azure Cold Start Threshold"></p>
|
|
<p>Clearly, an idle instance lives for 20 minutes and then gets recycled. All requests after 20 minutes
|
|
threshold hit another cold start.</p>
|
|
<h3 id="aws">AWS</h3>
|
|
<p>AWS is more tricky. Here is the same kind of chart, relative durations vs time since the last request,
|
|
measured for AWS Lambda:</p>
|
|
<p><img src="aws-coldstart-threshold.png" alt="AWS Cold Start vs Warm Start"></p>
|
|
<p>There’s no clear threshold here… For this sample, no cold starts happened within 28 minutes after the previous
|
|
invocation. Afterward, the frequency of cold starts slowly rises. But even after 1 hour of inactivity, there’s still a
|
|
good chance that your instance is alive and ready to take requests.</p>
|
|
<p>This doesn’t match the official information that AWS Lambdas stay alive for just 5 minutes after the last
|
|
invocation. I reached out to Chris Munns, and he confirmed:</p>
|
|
<blockquote class="twitter-tweet" data-conversation="none" data-lang="en"><p lang="en" dir="ltr">
|
|
So what you are seeing is very much possible as the team plays with certain knobs/levers for execution environment lifecycle.
|
|
let me know if you have concerns about it, but it should be just fine</p>— chrismunns (@chrismunns)
|
|
<a href="https://twitter.com/chrismunns/status/1021452964630851585?ref_src=twsrc%5Etfw">July 23, 2018</a>
|
|
</blockquote>
|
|
<script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>
|
|
<p>A couple learning points here:</p>
|
|
<ul>
|
|
<li>AWS is working on improving cold start experience (and probably Azure/GCP do too)</li>
|
|
<li>My results might not be reliably reproducible in your application since it’s affected by recent adjustments</li>
|
|
</ul>
|
|
<h3 id="gcp">GCP</h3>
|
|
<p>Google Cloud Functions left me completely puzzled. Here is the same chart for GCP cold starts (again,
|
|
orange dots are warm and blue ones are cold):</p>
|
|
<p><img src="gcp-coldstart-threshold.png" alt="GCP Cold Start vs Warm Start"></p>
|
|
<p>This looks totally random to me. A cold start can happen in 3 minutes after the previous request, or an instance
|
|
can be kept alive for the whole hour. The probability of a cold start doesn’t seem to depend on the interval,
|
|
at least just by looking at this chart.</p>
|
|
<p>Any ideas about what’s going on are welcome!</p>
|
|
<h3 id="parallel-requests">Parallel requests</h3>
|
|
<p>Cold starts happen not only when the first instance of an application is provisioned. The same issue will happen whenever
|
|
all the provisioned instances are busy handling incoming events, and yet another event comes in (at scale out).</p>
|
|
<p>As far as I’m aware, this behavior is common to all 3 providers, so I haven’t prepared any comparison charts
|
|
for N+1 cold starts. Yet, be aware of them!</p>
|
|
<h2 id="reading-candle-charts">Reading Candle Charts</h2>
|
|
<p>In the following sections, you will see charts that represent statistical distribution of cold start time as
|
|
measured during my experiments. I repeated experiments multiple times and then grouped the metric values, e.g.
|
|
by the cloud provider or by language.</p>
|
|
<p>Each group will be represented by a “candle” on the chart. This is how you should read each candle:</p>
|
|
<p><img src="sample-coldstart-chart.png" alt="How to Read Cold Start Charts"></p>
|
|
<h2 id="memory-allocation">Memory Allocation</h2>
|
|
<p>AWS Lambda and Google Cloud Functions have a setting to define the memory size that gets allocated to a single
|
|
instance of a function. A user can select a value from 128MB to 2GB and above at creation time.</p>
|
|
<p>More importantly, the virtual CPU cycles get allocated proportionally to this provisioned memory size. This means
|
|
that an instance of 512 MB will have twice as much CPU speed as an instance of 256MB.</p>
|
|
<p>Does this affect the cold start time?</p>
|
|
<p>I’ve run a series of tests to compare cold start latency across the board of memory/CPU sizes. The results are
|
|
somewhat mixed.</p>
|
|
<p>AWS Lambda Javascript doesn’t seem to have significant differences. This probably means that not so much CPU load
|
|
is required to start a Node.js “Hello World” application:</p>
|
|
<p><img src="aws-coldstart-js-by-memory.png" alt="AWS Javascript Cold Start by Memory"></p>
|
|
<p>AWS Lambda .NET Core runtime does depend on memory size though. Cold start time drops dramatically with every increase
|
|
in allocated memory and CPU:</p>
|
|
<p><img src="aws-coldstart-csharp-by-memory.png" alt="AWS C# Cold Start by Memory"></p>
|
|
<p>GCP Cloud Functions expose a similar effect even for Javascript runtime:</p>
|
|
<p><img src="gcp-coldstart-js-by-memory.png" alt="GCP Javascript Cold Start by Memory"></p>
|
|
<p>In contrast to Amazon and Google, Microsoft doesn’t ask to select a memory limit. Azure will charge Functions based
|
|
on the actual memory usage. More importantly, it will always dedicate a full vCore for a given Function execution.</p>
|
|
<p>It’s not exactly apples-to-apples, but I chose to fix the memory allocations of AWS Lambda and GCF to 1024 MB.
|
|
This feels the closest to Azure’s vCore capacity, although I haven’t tried a formal CPU performance comparison.</p>
|
|
<p>Given that, let’s see how the 3 cloud providers compare in cold start time.</p>
|
|
<h2 id="javascript-baseline">Javascript Baseline</h2>
|
|
<p>Node.js is the only runtime supported in production by Google Cloud Functions right now. Javascript is also
|
|
probably by far the most popular language for serverless applications across the board.</p>
|
|
<p>Thus, it makes sense to compare the 3 cloud providers on how they perform in Javascript. The
|
|
base test measures the cold starts of “Hello World” type of functions. Functions have no
|
|
dependencies, so deployment package is really small.</p>
|
|
<p>Here are the numbers for cold starts:</p>
|
|
<p><img src="coldstart-js-baseline.png" alt="Cold Start for Basic Javascript Functions"></p>
|
|
<p>AWS is clearly doing the best job here. GCP takes the second place, and Azure is the slowest. The rivals are
|
|
sort of close though, seemingly playing in the same league so the exact disposition might change over time.</p>
|
|
<h2 id="how-do-languages-compare">How Do Languages Compare?</h2>
|
|
<p>I’ve written Hello World HTTP function in all supported languages of the cloud platforms:</p>
|
|
<ul>
|
|
<li>AWS: Javascript, Python, Java, Go and C# (.NET Core)</li>
|
|
<li>Azure: Javascript and C# (precompiled .NET assembly)</li>
|
|
<li>GCP: Javascript</li>
|
|
</ul>
|
|
<p>Azure kind of supports much more languages, including Python and Java, but they are still considered
|
|
experimental / preview, so the cold starts are not fully optimized. See
|
|
<a href="https://mikhail.io/2018/04/azure-functions-cold-starts-in-numbers/">my previous article</a> for exact numbers.</p>
|
|
<p>Same applies to Python on GCP.</p>
|
|
<p>The following chart shows some intuition about the cold start duration per language. The languages
|
|
are ordered based on mean response time, from lowest to highest:</p>
|
|
<p><img src="coldstart-per-language.png" alt="Cold Start per Language per Cloud and Language"></p>
|
|
<p>AWS provides the richest selection of runtimes, and 4 out of 5 are faster than the other two cloud providers.
|
|
C# / .NET seems to be the least optimized (Amazon, why is that?).</p>
|
|
<h2 id="does-size-matter">Does Size Matter?</h2>
|
|
<p>OK, enough of Hello World. A real-life function might be more heavy, mainly because it would
|
|
depend on other third-party libraries.</p>
|
|
<p>To simulate such scenario, I’ve measured cold starts for functions with extra dependencies:</p>
|
|
<ul>
|
|
<li>Javascript referencing 3 NPM packages - 5MB zipped</li>
|
|
<li>Javascript referencing 38 NPM packages - 35 MB zipped</li>
|
|
<li>C# function referencing 5 NuGet packages - 2 MB zipped</li>
|
|
<li>Java function referencing 5 Maven packages - 15 MB zipped</li>
|
|
</ul>
|
|
<p>Here are the results:</p>
|
|
<p><img src="coldstart-dependencies.png" alt="Cold Start Dependencies"></p>
|
|
<p>As expected, the dependencies slow the loading down. You should keep your Functions lean,
|
|
otherwise, you will pay in seconds for every cold start.</p>
|
|
<p>However, the increase in cold start seems quite low, especially for precompiled languages.</p>
|
|
<p>A very cool feature of GCP Cloud Functions is that you don’t have to include NPM packages into
|
|
the deployment archive. You just add <code>package.json</code> file and the runtime will restore them for you.
|
|
This makes the deployment artifact ridiculously small, but doesn’t seem to slow down the cold
|
|
starts either. Obviously, Google pre-restores the packages in advance, before the actual request
|
|
comes in.</p>
|
|
<h2 id="avoiding-cold-starts">Avoiding Cold Starts</h2>
|
|
<p>The overall impression is that cold start delays aren’t that high, so most applications can tolerate
|
|
them just fine.</p>
|
|
<p>If that’s not the case, some tricks can be implemented to keep function instances warm.
|
|
The approach is universal for all 3 providers: once in X minutes, make an artificial call to
|
|
the function to prevent it from expiring.</p>
|
|
<p>Implementation details will differ since the expiration policies are different, as we explored
|
|
above.</p>
|
|
<p>For applications with higher load profile, you might want to fire several parallel “warming”
|
|
requests in order to make sure that enough instances are kept in warm stock.</p>
|
|
<p>For further reading, have a look at my
|
|
<a href="https://mikhail.io/2018/05/azure-functions-cold-starts-beyond-first-load/">Cold Starts Beyond First Request in Azure Functions</a>
|
|
and <a href="https://mikhail.io/2018/08/aws-lambda-warmer-as-pulumi-component/">AWS Lambda Warmer as Pulumi Component</a>.</p>
|
|
<h2 id="conclusions">Conclusions</h2>
|
|
<p>Here are some lessons learned from all the experiments above:</p>
|
|
<ul>
|
|
<li>Be prepared for 1-3 seconds cold starts even for the smallest Functions</li>
|
|
<li>Different languages and runtimes have roughly comparable cold start time within the same platform</li>
|
|
<li>Minimize the number of dependencies, only bring what’s needed</li>
|
|
<li>AWS keeps cold starts below 1 second most of the time, which is pretty amazing</li>
|
|
<li>All cloud providers are aware of the problem and are actively optimizing the cold start experience</li>
|
|
<li>It’s likely that in middle term these optimizations will make cold starts a non-issue for the
|
|
vast majority of applications</li>
|
|
</ul>
|
|
<p>Do you see anything weird or unexpected in my results? Do you need me to dig deeper into other aspects?
|
|
Please leave a comment below or ping me on <a href="https://twitter.com/MikhailShilkov">twitter</a>, and let’s
|
|
sort it all out.</p>
|
|
<p>Stay tuned for more serverless perf goodness!</p>
|
|
|
|
</div>
|
|
|
|
|
|
<div class="after-post-tags">
|
|
<ul class="tags">
|
|
|
|
<li>
|
|
|
|
|
|
<a href="/tags/azure">Azure</a>
|
|
|
|
</li>
|
|
|
|
<li>
|
|
|
|
|
|
<a href="/tags/azure-functions">Azure Functions</a>
|
|
|
|
</li>
|
|
|
|
<li>
|
|
|
|
|
|
<a href="/tags/serverless">Serverless</a>
|
|
|
|
</li>
|
|
|
|
<li>
|
|
|
|
|
|
<a href="/tags/performance">Performance</a>
|
|
|
|
</li>
|
|
|
|
<li>
|
|
|
|
<a href="/serverless/coldstarts/">Cold Starts</a>
|
|
|
|
</li>
|
|
|
|
<li>
|
|
|
|
|
|
<a href="/tags/aws">AWS</a>
|
|
|
|
</li>
|
|
|
|
<li>
|
|
|
|
|
|
<a href="/tags/aws-lambda">AWS Lambda</a>
|
|
|
|
</li>
|
|
|
|
<li>
|
|
|
|
|
|
<a href="/tags/gcp">GCP</a>
|
|
|
|
</li>
|
|
|
|
<li>
|
|
|
|
|
|
<a href="/tags/google-cloud-functions">Google Cloud Functions</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</div>
|
|
|
|
|
|
<hr/>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
<div class="container">
|
|
<div id="sharepane" class="row justify-content-center">
|
|
<div class="col-md-2">
|
|
|
|
</div>
|
|
<div class="contact-intro col-md-2 text-center">
|
|
<img class="profile-small d-inline-block mx-auto rounded-circle mb-3" height="100px"
|
|
src="/images/author.jpg" alt="">
|
|
</div>
|
|
<div class="col-md-8">
|
|
<section class="article-open_author">
|
|
<div class="article-open_author_bio">
|
|
<h5 itemprop="author" itemscope="" itemtype="http://schema.org/Person">Author: Mikhail Shilkov</h5>
|
|
<p>
|
|
<div>Cloud and AI developer and researcher.</div>
|
|
</p>
|
|
</div>
|
|
</section>
|
|
<a href="https://x.com/MikhailShilkov?ref_src=twsrc%5Etfw" class="twitter-follow-button" data-show-count="false">Follow @MikhailShilkov</a><script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="container">
|
|
<div id="comments" class="row justify-content-center mb-5">
|
|
<div class="col-md-8">
|
|
|
|
|
|
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
<footer class="footer">
|
|
<div class="container">
|
|
<div class="row">
|
|
<div class="col-md-6 col-sm-6 text-center text-lg-left">
|
|
© Copyright Mikhail Shilkov - All rights reserved
|
|
</div>
|
|
<div class="col-md-6 col-sm-6 text-center text-lg-right">
|
|
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</footer>
|
|
|
|
|
|
<script>
|
|
document.addEventListener('DOMContentLoaded', function() {
|
|
document.querySelectorAll('code.language-diff span span').forEach(function(span) {
|
|
const text = span.textContent;
|
|
if (text.startsWith('-')) {
|
|
span.style.backgroundColor = '#ffdddd';
|
|
span.style.color = '#d73a49';
|
|
span.style.display = 'block';
|
|
} else if (text.startsWith('+')) {
|
|
span.style.backgroundColor = '#ddffdd';
|
|
span.style.color = '#28a745';
|
|
span.style.display = 'block';
|
|
}
|
|
});
|
|
});
|
|
</script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<script src="/js/bundle.min.js"></script>
|
|
<script async src="https://www.googletagmanager.com/gtag/js?id=G-NL7F82LX48"></script>
|
|
<script>
|
|
var doNotTrack = false;
|
|
if ( false ) {
|
|
var dnt = (navigator.doNotTrack || window.doNotTrack || navigator.msDoNotTrack);
|
|
var doNotTrack = (dnt == "1" || dnt == "yes");
|
|
}
|
|
if (!doNotTrack) {
|
|
window.dataLayer = window.dataLayer || [];
|
|
function gtag(){dataLayer.push(arguments);}
|
|
gtag('js', new Date());
|
|
gtag('config', 'G-NL7F82LX48');
|
|
}
|
|
</script>
|
|
<script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" data-cf-beacon='{"version":"2024.11.0","token":"aa9d069cdfb549c49190f335e8ca12c6","r":1,"spa":2}' crossorigin="anonymous"></script>
|
|
</body>
|
|
</html>
|