Files
nexus/sreweekly/articles/453/04-go-profiling-in-production.html
2026-09-12 17:23:01 +08:00

858 lines
47 KiB
HTML

<!DOCTYPE html>
<html lang="en">
<head>
<title>Go Profiling in Production: A Practical pprof Guide</title>
<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="HandheldFriendly" content="True" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="preload" as="style" href="https://blog.oodle.ai/assets/built/screen.css?v=S0DSiw6jVHJaYsG0" />
<link rel="preload" as="script" href="https://blog.oodle.ai/assets/built/casper.js?v=YtPhrez3GuGLVDYs" />
<link rel="stylesheet" type="text/css" href="https://blog.oodle.ai/assets/built/screen.css?v=S0DSiw6jVHJaYsG0" />
<meta name="description" content="Profiling Go in production: net/http/pprof setup, CPU and heap profiles, pprof labels, goroutine leak detection, and Go 1.25&#x27;s Flight Recorder.">
<link rel="icon" href="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w256h256/2024/09/oodle_logo_1x.png" type="image/png">
<link rel="canonical" href="https://blog.oodle.ai/go-profiling-in-production/">
<meta name="referrer" content="no-referrer-when-downgrade">
<meta property="og:site_name" content="Oodle AI">
<meta property="og:type" content="article">
<meta property="og:title" content="Go Profiling in Production">
<meta property="og:description" content="Profiling Go in production: net/http/pprof setup, CPU and heap profiles, pprof labels, goroutine leak detection, and Go 1.25&#x27;s Flight Recorder.">
<meta property="og:url" content="https://blog.oodle.ai/go-profiling-in-production/">
<meta property="og:image" content="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/2026/07/oodle-go-profiling-og-1.png">
<meta property="article:published_time" content="2024-11-20T04:39:14.000Z">
<meta property="article:modified_time" content="2026-07-31T13:49:17.000Z">
<meta name="twitter:card" content="summary_large_image">
<meta name="twitter:title" content="Go Profiling in Production">
<meta name="twitter:description" content="Profiling Go in production: net/http/pprof setup, CPU and heap profiles, pprof labels, goroutine leak detection, and Go 1.25&#x27;s Flight Recorder.">
<meta name="twitter:url" content="https://blog.oodle.ai/go-profiling-in-production/">
<meta name="twitter:image" content="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/2026/07/oodle-go-profiling-og.png">
<meta name="twitter:label1" content="Written by">
<meta name="twitter:data1" content="Gaurav Maheshwari">
<meta name="twitter:label2" content="Filed under">
<meta name="twitter:data2" content="">
<meta name="twitter:site" content="@oodleai">
<meta name="twitter:creator" content="@gaurav25d">
<meta property="og:image:width" content="1200">
<meta property="og:image:height" content="630">
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "Article",
"publisher": {
"@type": "Organization",
"name": "Oodle AI",
"url": "https://blog.oodle.ai/",
"logo": {
"@type": "ImageObject",
"url": "https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/2025/04/logo_blue.png"
}
},
"author": {
"@type": "Person",
"name": "Gaurav Maheshwari",
"image": {
"@type": "ImageObject",
"url": "https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w1200/2024/11/_N2A0018-3.jpg",
"width": 1200,
"height": 1879
},
"url": "https://blog.oodle.ai/author/gaurav/",
"sameAs": [
"https://mgaurav.github.io/",
"https://x.com/gaurav25d"
]
},
"headline": "Go Profiling in Production: A Practical pprof Guide",
"url": "https://blog.oodle.ai/go-profiling-in-production/",
"datePublished": "2024-11-20T04:39:14.000Z",
"dateModified": "2026-07-31T13:49:17.000Z",
"image": {
"@type": "ImageObject",
"url": "https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w1200/2024/11/gotools-1.jpg",
"width": 1200,
"height": 393
},
"description": "TL;DR\n\n\n * Go's built-in profiler, pprof, exposes CPU, heap, goroutine, block, and mutex profiles through net/http/pprof; run it on a dedicated localhost-only port.\n * go tool pprof -http=:8080 \"http://localhost:6060/debug/pprof/profile?seconds=30\" collects a 30-second CPU profile and opens it as a flame graph.\n * Comparing heap profiles with -diff_base isolates memory growth between two points in time.\n * pprof.Do profiler labels attribute CPU time to individual tenants in a multi-tenant servic",
"mainEntityOfPage": "https://blog.oodle.ai/go-profiling-in-production/"
}
</script>
<meta name="generator" content="Ghost 6.64">
<link rel="alternate" type="application/rss+xml" title="Oodle AI" href="https://blog.oodle.ai/rss/">
<script defer src="https://cdn.jsdelivr.net/ghost/portal@~2.71/umd/portal.min.js" data-i18n="true" data-ghost="https://blog.oodle.ai/" data-key="5d821cde2a2f12de6e45a55016" data-api="https://oodle-ai.ghost.io/ghost/api/content/" data-locale="en" crossorigin="anonymous"></script><style id="gh-members-styles">.gh-post-upgrade-cta-content,
.gh-post-upgrade-cta {
display: flex;
flex-direction: column;
align-items: center;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;
text-align: center;
width: 100%;
color: #ffffff;
font-size: 16px;
}
.gh-post-upgrade-cta-content {
border-radius: 8px;
padding: 40px 4vw;
}
.gh-post-upgrade-cta h2 {
color: #ffffff;
font-size: 28px;
letter-spacing: -0.2px;
margin: 0;
padding: 0;
}
.gh-post-upgrade-cta p {
margin: 20px 0 0;
padding: 0;
}
.gh-post-upgrade-cta small {
font-size: 16px;
letter-spacing: -0.2px;
}
.gh-post-upgrade-cta a {
color: #ffffff;
cursor: pointer;
font-weight: 500;
box-shadow: none;
text-decoration: underline;
}
.gh-post-upgrade-cta a:hover {
color: #ffffff;
opacity: 0.8;
box-shadow: none;
text-decoration: underline;
}
.gh-post-upgrade-cta a.gh-btn {
display: block;
background: #ffffff;
text-decoration: none;
margin: 28px 0 0;
padding: 8px 18px;
border-radius: 4px;
font-size: 16px;
font-weight: 600;
}
.gh-post-upgrade-cta a.gh-btn:hover {
opacity: 0.92;
}</style>
<script defer src="https://cdn.jsdelivr.net/ghost/sodo-search@~1.8/umd/sodo-search.min.js" data-key="5d821cde2a2f12de6e45a55016" data-styles="https://cdn.jsdelivr.net/ghost/sodo-search@~1.8/umd/main.css" data-sodo-search="https://oodle-ai.ghost.io/" data-locale="en" crossorigin="anonymous"></script>
<link href="https://blog.oodle.ai/webmentions/receive/" rel="webmention">
<script defer src="/public/cards.min.js?v=ShRHxgy4po8zN-Wf"></script>
<link rel="stylesheet" type="text/css" href="/public/cards.min.css?v=WwnU9jw5ancNC8Gc">
<script defer src="/public/member-attribution.min.js?v=AKG4hWena9j3yX3I"></script>
<script defer src="/public/ghost-stats.min.js?v=vFcCUf6ZQ0Hyhc8h" data-stringify-payload="false" data-datasource="analytics_events" data-storage="localStorage" data-host="https://blog.oodle.ai/.ghost/analytics/api/v1/page_hit" tb_site_uuid="b2485d36-0dcd-4e46-b7bd-418ab6e38e18" tb_post_uuid="c0266fd0-ce25-485d-9250-49a2c5b84d47" tb_post_type="post" tb_member_uuid="undefined" tb_member_status="undefined" tb_gift_link=""></script><style>:root {--ghost-accent-color: #0091cf;}</style>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.10.0/styles/atom-one-dark.min.css" integrity="sha512-Jk4AqjWsdSzSWCSuQTfYRIF84Rq/eV0G2+tu07byYwHcbTGfdmLrHjUSwvzp5HvbiqK4ibmNwdcG49Y5RGYPTg==" crossorigin="anonymous" referrerpolicy="no-referrer" />
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.10.0/highlight.min.js" integrity="sha512-6yoqbrcLAHDWAdQmiRlHG4+m0g/CT/V9AGyxabG8j7Jk8j3r3K6due7oqpiRMZqcYe9WM2gPcaNNxnl2ux+3tA==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<!-- and it's easy to individually load additional languages -->
<script src="https://cdn.jsdelivr.net/npm/prismjs/prism.min.js" defer></script>
<script src="https://cdn.jsdelivr.net/npm/prismjs/plugins/autoloader/prism-autoloader.min.js" defer></script>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/prismjs/themes/prism.min.css">
<script>
document.addEventListener('DOMContentLoaded', function () {
Prism.languages.d2 = {
comment: [
{ pattern: /"""[\s\S]*?"""/, greedy: true },
{ pattern: /#.*/ }
],
string: [
{ pattern: /"(?:\\.|[^"\\])*"/, greedy: true },
{ pattern: /'(?:\\.|[^'\\])*'/, greedy: true }
],
keyword:
/\b(?:style|shape|label|icon|link|tooltip|near|class|classes|vars|layers|steps|scenarios|direction|fill|stroke|opacity|font-size|font-color|shadow|multiple|animated|bold|italic|underline|border-radius|stroke-width|stroke-dash|double-border|3d|filled|width|height|top|left|grid-rows|grid-columns|grid-gap|vertical-gap|horizontal-gap|text-transform|fill-pattern|constraint|font)\b/,
boolean: /\b(?:true|false)\b/,
null: { pattern: /\bnull\b/, alias: 'keyword' },
arrow: {
pattern: /<?-+>?/,
alias: 'operator'
},
number: /\b\d+(?:\.\d+)?\b/,
punctuation: /[{}[\];:.]/
};
});
</script>
<!-- Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=G-M0CVSZP7TP"></script>
<script id="gtag-init">
window.dataLayer = window.dataLayer || [];
function gtag(){ window.dataLayer.push(arguments); };
gtag("js", new Date());
gtag("config", "G-M0CVSZP7TP");
</script>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/tocbot/4.32.2/tocbot.css" integrity="sha512-Di7Va5KC5NtXyMi+aEyVe2pUnniyhFoxfCrdCAOj8aSA42Te/bWKTz0iumaj5v1sN1nZdsaX8QTCn0k1nN4aLA==" crossorigin="anonymous" referrerpolicy="no-referrer" />
<style>
img{border-radius:10px;}.gh-content{position:relative}.gh-toc>.toc-list{position:relative}.toc-list{overflow:hidden;list-style:none}.gh-toc .is-active-link::before{background-color:var(--ghost-accent-color)}a.toc-link{display:inline-flex;font-weight:400;height:100%;line-height:1.2em;padding:6px 0;text-decoration:none;transition:.4s ease;font-size:100%!important}li.toc-list-item{color:#738a94!important}a.toc-link:hover{color:#15171a!important}.is-collapsible a.is-active-link,a.is-active-link{color:#15171a!important;font-weight:500}@media (max-width:1400px){.gh-toc{background:#fff;border-radius:1em;box-shadow:0 10px 50px rgba(25,37,52,.14),0 2px 5px rgba(25,37,52,.03);padding:30px;width:100%}}
.gh-sidebar .gh-toc .toc-list{padding-inline-start: 16px; margin-inline-start: 8px;}
@media (min-width:1300px){.gh-sidebar{position:absolute;top:0;bottom:0;margin-top:4vmin;grid-column:wide-start / main-start}.gh-toc{position:sticky;top:4vmin}li.toc-list-item{font-size:1.4rem}}
.blockquote {
position: relative;
align-self: center;
}
/* The Quote */
.blockquote .quote {
position: relative; /* for pseudos */
border-color: var(--ghost-accent-color);
color: var(--ghost-accent-color);
border-style: solid;
font-weight: normal;
font-size: 2rem;
font-weight: 400;
font-style: italic;
line-height: 1.6em;
margin: 0;
border-width: 2px;
border-radius:10px;
padding: 25px;
background: linear-gradient(180deg, rgba(186, 233, 255, 0.5) 0%, #fff 100%, #fff 100%);
}
/* Blockquote right double quotes */
.blockquote .quote:after {
content:"";
position: absolute;
border-color: var(--ghost-accent-color);
border-width: 2px;
border-style: solid;
border-radius: 0 100% 0 0;
width: 60px;
height: 60px;
bottom: -60px;
left: 50px;
border-bottom: none;
border-left: none;
z-index: 3;
}
.blockquote .quote:before {
content:"";
position: absolute;
width: 80px;
border: 6px solid #fff;
bottom: -3px;
left: 50px;
z-index: 2;
}
.blockquote .name {
margin-left: 130px;
margin-top: 10px;
font-weight: bold;
}
.blockquote .title {
margin-left: 130px;
}
</style>
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "TechArticle",
"headline": "Go Profiling in Production",
"datePublished": "2024-11-20",
"dateModified": "<set on publish>",
"author": {"@type": "Person", "name": "Gaurav Maheshwari"},
"publisher": {"@type": "Organization", "name": "Oodle AI", "url": "https://oodle.ai"},
"about": ["pprof", "Go (programming language)", "profiling"],
"proficiencyLevel": "Expert",
"url": "https://blog.oodle.ai/go-profiling-in-production/"
}
</script>
<!-- Table fit: the theme forces nowrap + horizontal scroll on wide tables; let this post's
comparison table wrap and fit the content column instead; also hide the
blockquote accent bar (it hangs into the left gutter and collides with the TOC) -->
<style>
.gh-content table:not(.gist table){display:table;width:100%;white-space:normal}
.gh-content table:not(.gist table) td,.gh-content table:not(.gist table) th{white-space:normal;vertical-align:top}
.gh-content table:not(.gist table) code{white-space:normal;overflow-wrap:anywhere}
.gh-content blockquote::before{display:none!important}
</style>
</head>
<body class="post-template tag-hash-golang tag-hash-profile tag-hash-performance is-head-left-logo has-sans-body">
<div class="viewport">
<header id="gh-head" class="gh-head outer is-header-hidden">
<div class="gh-head-inner inner">
<div class="gh-head-brand">
<a class="gh-head-logo" href="https://blog.oodle.ai">
<img src="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/2025/04/logo_blue.png" alt="Oodle AI">
</a>
<button class="gh-search gh-icon-btn" aria-label="Search this site" data-ghost-search><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2" width="20" height="20"><path stroke-linecap="round" stroke-linejoin="round" d="M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z"></path></svg></button>
<button class="gh-burger" aria-label="Main Menu"></button>
</div>
<nav class="gh-head-menu">
<ul class="nav">
<li class="nav-home"><a href="https://blog.oodle.ai/">Home</a></li>
<li class="nav-discord"><a href="https://discord.gg/VdEuk9G3Gn">Discord</a></li>
<li class="nav-playground"><a href="https://play.oodle.ai/">Playground</a></li>
<li class="nav-sign-up"><a href="https://us1.oodle.ai/signup">Sign up</a></li>
<li class="nav-docs"><a href="https://docs.oodle.ai/">Docs</a></li>
<li class="nav-about"><a href="https://blog.oodle.ai/about/">About</a></li>
</ul>
</nav>
<div class="gh-head-actions">
<button class="gh-search gh-icon-btn" aria-label="Search this site" data-ghost-search><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2" width="20" height="20"><path stroke-linecap="round" stroke-linejoin="round" d="M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z"></path></svg></button>
<div class="gh-head-members">
<a class="gh-head-button" href="#/portal/signin" data-portal="signin">Sign in</a>
</div>
</div>
</div>
</header>
<div class="site-content">
<main id="site-main" class="site-main">
<article class="article post tag-hash-golang tag-hash-profile tag-hash-performance ">
<header class="article-header gh-canvas">
<div class="article-tag post-card-tags">
</div>
<h1 class="article-title">Go Profiling in Production</h1>
<div class="article-byline">
<section class="article-byline-content">
<ul class="author-list instapaper_ignore">
<li class="author-list-item">
<a href="/author/gaurav/" class="author-avatar" aria-label="Read more of Gaurav Maheshwari">
<img class="author-profile-image" src="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w100/2024/11/_N2A0018-3.jpg" alt="Gaurav Maheshwari" loading="eager" />
</a>
</li>
</ul>
<div class="article-byline-meta">
<h4 class="author-name"><a href="/author/gaurav/">Gaurav Maheshwari</a></h4>
<div class="byline-meta-content">
<time class="byline-meta-date" datetime="2024-11-20">20 Nov 2024</time>
<span class="byline-reading-time"><span class="bull">&bull;</span> 8 min read</span>
</div>
</div>
</section>
<a href="#/share" class="gh-button gh-button-share">Share</a>
</div>
<figure class="article-image">
<img
class="gh-feature-image"
srcset="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w300/2024/11/gotools-1.jpg 300w,
https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w600/2024/11/gotools-1.jpg 600w,
https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w1000/2024/11/gotools-1.jpg 1000w,
https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w2000/2024/11/gotools-1.jpg 2000w"
sizes="(min-width: 1400px) 1400px, 92vw"
src="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w2000/2024/11/gotools-1.jpg"
alt="Go Profiling in Production"
fetchpriority="high"
loading="eager"
/>
<figcaption><span style="white-space:pre-wrap">An engineer debugging performance problems</span></figcaption>
</figure>
</header>
<section class="gh-content gh-canvas">
<p><strong>TL;DR</strong></p>
<ul>
<li>Go's built-in profiler, <strong>pprof</strong>, exposes CPU, heap, goroutine, block, and mutex profiles through <code>net/http/pprof</code>; run it on a dedicated localhost-only port.</li>
<li><code>go tool pprof -http=:8080 "http://localhost:6060/debug/pprof/profile?seconds=30"</code> collects a 30-second CPU profile and opens it as a flame graph.</li>
<li>Comparing heap profiles with <code>-diff_base</code> isolates memory growth between two points in time.</li>
<li><code>pprof.Do</code> profiler labels attribute CPU time to individual tenants in a multi-tenant service.</li>
<li>Go 1.25's Flight Recorder keeps the last few seconds of execution trace in memory, so you can capture an incident after it happens.</li>
</ul>
<p><strong>Profiling a Go application in production</strong> is how we find CPU, memory, and goroutine bottlenecks at Oodle before they turn into incidents.</p>
<p>At Oodle, we use Golang for our backend services. In an earlier blog post (<a href="https://blog.oodle.ai/high-performance-in-golang/">Go faster!</a>), we discussed optimization techniques for performance and scale, with a specific focus on reducing memory allocations. In this post, we'll explore the profiling tools we use. Golang provides a rich set of <a href="https://go.dev/doc/diagnostics?ref=blog.oodle.ai">profiling tools</a> to help uncover code bottlenecks.</p>
<h2 id="our-setup">Our Setup</h2>
<p>We use the standard <a href="https://pkg.go.dev/net/http/pprof?ref=blog.oodle.ai">net/http/pprof</a> package to register handlers for profiling endpoints. As part of our service startup routine, we run a profiler server on a dedicated port.</p>
<pre><code class="language-go">import (
_ "net/http/pprof"
)
...
var profilerPort = flaggy.GetEnvStringVar(
"PROFILER_PORT",
"profiler_port",
"Profiler port",
"6060",
)
func StartProfilerServer(ctx context.Context) {
go func() {
http.ListenAndServe(fmt.Sprintf("localhost:%s", *profilerPort), nil)
}()
}
</code></pre>
<p>This setup allows us to collect various profiles exposed by the <code>net/http/pprof</code> package on-demand.</p>
<p><strong>A note on safety:</strong> never expose <code>/debug/pprof</code> on a public interface; profiling endpoints reveal internal details and add load. Bind them to localhost as above and reach them via port-forward or a private network.</p>
<p><strong>Go profiling tools at a glance</strong></p>
<table>
<thead>
<tr>
<th style="text-align:left">Profile / tool</th>
<th style="text-align:left">How to collect</th>
<th style="text-align:left">What it answers</th>
</tr>
</thead>
<tbody>
<tr>
<td style="text-align:left">CPU profile</td>
<td style="text-align:left">/debug/pprof/profile?seconds=30</td>
<td style="text-align:left">Where CPU time goes</td>
</tr>
<tr>
<td style="text-align:left">Heap profile</td>
<td style="text-align:left">/debug/pprof/heap (compare with -diff_base )</td>
<td style="text-align:left">What is allocating and retaining memory</td>
</tr>
<tr>
<td style="text-align:left">Goroutine dump</td>
<td style="text-align:left">/debug/pprof/goroutine?debug=2</td>
<td style="text-align:left">Leaked or blocked goroutines</td>
</tr>
<tr>
<td style="text-align:left">Block profile</td>
<td style="text-align:left">/debug/pprof/block · off by default; enable with runtime.SetBlockProfileRate</td>
<td style="text-align:left">Where goroutines wait (channels, locks)</td>
</tr>
<tr>
<td style="text-align:left">Mutex profile</td>
<td style="text-align:left">/debug/pprof/mutex · off by default; enable with runtime.SetMutexProfileFraction</td>
<td style="text-align:left">Lock contention hotspots</td>
</tr>
<tr>
<td style="text-align:left">Execution trace</td>
<td style="text-align:left">/debug/pprof/trace?seconds=5</td>
<td style="text-align:left">Why goroutines aren't running (scheduling, GC)</td>
</tr>
<tr>
<td style="text-align:left">Flight Recorder (Go 1.25+)</td>
<td style="text-align:left">runtime/trace.FlightRecorder</td>
<td style="text-align:left">The seconds leading up to an incident</td>
</tr>
<tr>
<td style="text-align:left">Continuous profiling</td>
<td style="text-align:left">Pyroscope, Parca</td>
<td style="text-align:left">Trends over time, across the fleet</td>
</tr>
</tbody>
</table>
<p><strong>How to profile a Go service in production:</strong></p>
<ol>
<li>Import <code>net/http/pprof</code> and serve it on a dedicated localhost port.</li>
<li>Collect a CPU profile: <code>go tool pprof -http=:8080 "http://localhost:6060/debug/pprof/profile?seconds=30"</code>.</li>
<li>Read the flame graph; the widest frames are your hottest paths.</li>
<li>Snapshot the heap twice and diff: <code>go tool pprof -diff_base heap_t0.prof heap_t1.prof</code>.</li>
<li>Dump goroutines when things hang: <code>/debug/pprof/goroutine?debug=2</code>.</li>
<li>For rare incidents, keep Go 1.25's Flight Recorder running and snapshot after the event.</li>
</ol>
<h2 id="cpu-profiling">CPU Profiling</h2>
<p>CPU profiling helps understand where CPU time is being spent in the code. To collect a CPU profile, we can hit the <code>/debug/pprof/profile</code> endpoint with a <code>seconds</code> parameter.</p>
<pre><code class="language-bash">curl http://localhost:6060/debug/pprof/profile?seconds=30 -o cpu.prof
</code></pre>
<p>After collecting the profile, we can analyze it using the <code>go tool pprof</code> command.</p>
<pre><code class="language-bash">go tool pprof -http=:8080 cpu.prof
</code></pre>
<p><strong>Note:</strong><br>
Both of the above steps can be combined into a single command:</p>
<pre><code class="language-bash">go tool pprof -http=:8080 http://localhost:6060/debug/pprof/profile?seconds=30
</code></pre>
<p>This command starts a local HTTP server and opens the profile in the browser. We frequently use [flame graphs] (<a href="https://www.brendangregg.com/flamegraphs.html?ref=blog.oodle.ai">https://www.brendangregg.com/flamegraphs.html</a>) for profile analysis.</p>
<p><img src="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/2024/11/blog-profiler-cpu.png" alt="Flame graph view of a Go CPU profile in go tool pprof" loading="lazy"></p>
<p><strong>Note:</strong><br>
All screenshots referenced in this blog post are for the <a href="https://gist.github.com/mgaurav/80910918ff2fe3694e86c2462bfbadd2?ref=blog.oodle.ai">demo application</a>.</p>
<h3 id="per-tenant-profiles-in-a-multi-tenant-service">Per-tenant profiles in a multi-tenant service</h3>
<p>Per-tenant profiler labels use Go's <code>pprof.Do</code> to tag CPU samples with a tenant identifier, so one profile can show exactly which tenant drives load in a multi-tenant service.</p>
<p>As a multi-tenant SaaS platform, Oodle serves many tenants from the same deployment. Ingestion and query loads vary significantly among tenants. To understand CPU profiles for individual tenants, we use [profiler labels] (<a href="https://rakyll.org/profiler-labels/?ref=blog.oodle.ai">https://rakyll.org/profiler-labels/</a>) to add tenant identifiers to the profiles.</p>
<pre><code class="language-go">pprof.Do(ctx, pprof.Labels("tenant_id", tenantID), func(ctx context.Context) {
// Your code here.
})
</code></pre>
<p>These profiler labels attach additional information to stack frames in the collected profiles, enabling per-tenant CPU attribution analysis.</p>
<pre><code class="language-bash">$ go tool pprof cpu.prof
File: cpu
Type: cpu
...
(pprof) tags
tenant_id: Total 10.6s
6.7s (63.31%): tenant1
3.9s (36.69%): tenant2
</code></pre>
<p>We can also use the <a href="https://www.markhansen.co.nz/pprof-tagroot/?ref=blog.oodle.ai">tagroot</a> argument to add the tenant identifier as a pseudo stack frame at the callstack root.</p>
<pre><code class="language-bash">go tool pprof -http=:8080 -tagroot tenant_id cpu.prof
</code></pre>
<p><img src="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/2024/11/blog-profiler-cpu-tenant.png" alt="Go CPU profile flame graph with per-tenant pprof labels as root frames" loading="lazy"></p>
<h2 id="memory-profiling">Memory Profiling</h2>
<p>Memory profiling helps identify where memory allocations occur. Since Golang is a memory-managed language, minimizing allocations is crucial for reducing Garbage Collection (GC) overhead. Using our profiler server, we can collect a memory profile via the <code>/debug/pprof/heap</code> endpoint.</p>
<pre><code class="language-bash">go tool pprof -http=:8080 http://localhost:6060/debug/pprof/heap
</code></pre>
<p><img src="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/2024/11/blog-profiler-mem.png" alt="Go heap profile in go tool pprof showing top memory allocations" loading="lazy"></p>
<p><strong>Note:</strong><br>
Collecting memory profiles at different times and comparing them helps understand the service's memory behavior. Use the <code>-diff_base</code> flag to compare two profiles:</p>
<pre><code class="language-bash">go tool pprof -http=:8080 -diff_base heap_0935.prof heap_0940.prof
</code></pre>
<p>In practice the <code>-diff_base</code> workflow looks like this: save <code>/debug/pprof/heap</code> once when the service is healthy and again when memory is elevated, then open the diff; the entries with the largest positive <code>inuse_space</code> delta are what grew. Two related profiles answer different questions: the <strong>heap</strong> profile shows live memory (use it to find retention), while the <strong>allocs</strong> profile shows cumulative allocations since start (use it to find GC pressure).</p>
<h3 id="automated-memory-profiling">Automated Memory Profiling</h3>
<p>Memory footprint of a program can vary significantly depending on the workload. If the memory allocation rate is higher than the Garbage Collection (GC) rate, it can lead to Out of Memory (OOM) crashes. To understand memory usage patterns during high-memory situations, we use an auto profiler that:</p>
<ol>
<li>Captures memory profiles when footprint exceeds a percentage threshold</li>
<li>Uploads these profiles to object storage for offline analysis</li>
<li>Rate-limits profile collection to prevent excessive uploads</li>
</ol>
<p>Here is a rough blueprint of the auto profiler:</p>
<pre><code class="language-go">import (
"runtime/metrics"
"pontus.dev/cgroupmemlimited"
)
func startAutoProfiler(ctx context.Context) error {
memSamples := make([]metrics.Sample, 2)
memSamples[0].Name = "/memory/classes/total:bytes"
memSamples[1].Name = "/memory/classes/heap/released:bytes"
profileLimit := cgroupmemlimited.LimitAfterInit * (*autoProfilerPercentageThreshold / 100)
go func() {
ticker := time.NewTicker(*profilerInterval)
for {
select {
case &lt;-ctx.Done():
return
case &lt;-ticker.C:
metrics.Read(memSamples)
inUseMem := memSamples[0].Value.Uint64() - memSamples[1].Value.Uint64()
if inUseMem &gt; profileLimit {
if !limiter.Allow() { // rate limiter
continue
}
var buf bytes.Buffer
if err := pprof.WriteHeapProfile(&amp;buf); err != nil {
continue
}
// Upload the profile to object storage
}
}
}
}()
}
</code></pre>
<h2 id="goroutine-analysis">Goroutine Analysis</h2>
<p>While Golang makes concurrent programming easier with its powerful <a href="https://go.dev/doc/effective_go?ref=blog.oodle.ai#concurrency">concurrency primitives</a> like Goroutines and Channels, it's also possible to introduce goroutine leaks. For examples of common goroutine leak patterns, see <a href="https://www.uber.com/blog/leakprof-featherlight-in-production-goroutine-leak-detection/?ref=blog.oodle.ai">this</a> blog post.</p>
<p>We use <a href="https://github.com/uber-go/goleak?ref=blog.oodle.ai">goleak</a> to detect goroutine leaks in our tests. Enable it in a test by adding a <code>TestMain</code> function at the package level:</p>
<pre><code class="language-go">func TestMain(m *testing.M) {
goleak.VerifyTestMain(m)
}
</code></pre>
<p>To collect a goroutine dump from our services, we can use the <code>debug/pprof/goroutine</code> endpoint:</p>
<pre><code class="language-bash">curl "http://localhost:6060/debug/pprof/goroutine?debug=2" -o goroutine.dump
</code></pre>
<p>The goroutine dump is a text file containing stack traces of all process goroutines. For analyzing dumps with thousands of goroutines, we use <a href="https://github.com/linuxerwang/goroutine-inspect?ref=blog.oodle.ai">goroutine-inspect</a>, which provides helpful deduplication and search functionality.</p>
<h2 id="execution-tracing">Execution Tracing</h2>
<p>Execution tracing helps you understand how goroutines are scheduled and where they might be blocked. It captures several types of events:</p>
<ol>
<li>Goroutine lifecycle (creation, start, and end)</li>
<li>Blocking events (syscalls, channels, locks)</li>
<li>Network I/O operations</li>
<li>System calls</li>
<li>Garbage collection cycles</li>
</ol>
<p>While CPU profiling shows which functions consume the most CPU time, execution tracing reveals why goroutines might not be running. It helps identify if a goroutine is blocked on a mutex, waiting for a channel, or simply not getting enough CPU time. For a comprehensive introduction to execution tracing, see <a href="https://blog.gopheracademy.com/advent-2017/go-execution-tracer/?ref=blog.oodle.ai">this blog post</a>.</p>
<p>To collect an execution trace, use the <code>debug/pprof/trace</code> endpoint:</p>
<pre><code class="language-bash">curl "http://localhost:6060/debug/pprof/trace?seconds=30" -o trace.prof
</code></pre>
<p>To analyze the collected trace, use the <code>go tool trace</code> command:</p>
<pre><code class="language-bash">go tool trace trace.prof
</code></pre>
<p><img src="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/2024/11/blog-profiler-trace.png" alt="Go execution trace viewer showing goroutine scheduling and GC events" loading="lazy"></p>
<p>Recent versions of Go have <a href="https://go.dev/blog/execution-traces-2024?ref=blog.oodle.ai">significantly reduced</a> the overhead of execution tracing, making it practical for use in production environments.</p>
<p><strong>Where Go execution tracing stands in 2026:</strong> Go 1.25 (August 2025) promoted the flight recorder into the standard library as <a href="https://go.dev/blog/flight-recorder?ref=blog.oodle.ai">runtime/trace.FlightRecorder</a>.</p>
<p>It keeps the last few seconds of the execution trace in an in-memory ring buffer, so instead of collecting a trace window on demand and hoping the problem recurs, a service can snapshot the moments leading up to an incident (a failed health check, a latency spike) with <code>FlightRecorder.WriteTo</code>.</p>
<p>It complements the on-demand <code>/debug/pprof/trace</code> flow above: on-demand tracing for planned investigations, the flight recorder for events you can't predict.</p>
<p>For <strong>continuous profiling</strong> rather than on-demand snapshots; always-on profiles, queryable over time across your fleet; see <a href="https://grafana.com/oss/pyroscope/?ref=blog.oodle.ai">Pyroscope</a> and <a href="https://www.parca.dev/?ref=blog.oodle.ai">Parca</a>; Oodle ingests <a href="https://docs.oodle.ai/integrations/profiling/pyroscope?ref=blog.oodle.ai">Pyroscope profiles</a>.</p>
<h2 id="summary">Summary</h2>
<p>Production profiling in Go comes down to a small, repeatable workflow: expose <code>net/http/pprof</code> on a localhost-only port, pull a CPU profile when latency climbs, diff heap snapshots when memory grows, dump goroutines when things hang, and reach for execution tracing — or Go 1.25's flight recorder — when you need to see <em>why</em> goroutines aren't running.</p>
<p>That is the setup we run at Oodle: the same endpoints, plus per-tenant profiler labels and automated capture for the incidents we can't predict.</p>
<p>The CPU profiles you collect in production can also feed <a href="https://go.dev/doc/pgo?ref=blog.oodle.ai">profile-guided optimization</a>: check a representative profile in as <code>default.pgo</code> and Go (1.21+) uses it to optimize your hot paths at build time; a natural companion to the techniques in our <a href="https://blog.oodle.ai/high-performance-in-golang/">post on optimizing Go performance</a>.</p>
<p>See what these profiles look like against live, high-scale production data in the <a href="https://play.oodle.ai/?ref=blog.oodle.ai">Oodle playground</a>.</p>
<p><strong>Frequently asked questions</strong></p>
<h3 id="how-do-i-profile-a-go-application-in-production">How do I profile a Go application in production?</h3>
<p>Register <code>net/http/pprof</code> handlers and serve them on a dedicated localhost-only port at startup. You can then collect CPU, heap, goroutine, and trace profiles on demand from its endpoints without redeploying; keep the port unexposed publicly.</p>
<h3 id="how-do-i-collect-and-view-a-cpu-profile-with-pprof">How do I collect and view a CPU profile with pprof?</h3>
<p>Run <code>go tool pprof -http=:8080 "http://localhost:6060/debug/pprof/profile?seconds=30"</code>. It samples 30 seconds of CPU and opens an interactive flame graph in your browser.</p>
<h3 id="how-do-i-track-down-growing-memory-usage-in-a-go-service">How do I track down growing memory usage in a Go service?</h3>
<p>Capture <code>/debug/pprof/heap</code> at two points in time and compare them with <code>go tool pprof -diff_base</code>. For spikes you can't predict, automate capture when memory crosses a threshold; see the auto-profiler blueprint above.</p>
<h3 id="how-do-i-find-goroutine-leaks-in-go">How do I find goroutine leaks in Go?</h3>
<p>Use <a href="https://github.com/uber-go/goleak?ref=blog.oodle.ai">goleak</a> in <code>TestMain</code> to catch leaks in tests, and <code>/debug/pprof/goroutine?debug=2</code> dumps in production. For dumps with thousands of goroutines, goroutine-inspect deduplicates and searches stacks.</p>
<h3 id="what-other-profiles-does-nethttppprof-expose">What other profiles does net/http/pprof expose?</h3>
<p>Block, mutex, allocs, and threadcreate profiles.</p>
<p>Block and mutex profiling are off by default; enable them with <code>runtime.SetBlockProfileRate</code> and <code>runtime.SetMutexProfileFraction</code> at startup (they carry a small overhead); without those calls the endpoints return empty profiles.</p>
<p><strong>Related reading on the Oodle blog</strong></p>
<ul>
<li>Fix what the profiles find: <a href="https://blog.oodle.ai/high-performance-in-golang/">Go faster! Golang performance optimization in production</a></li>
<li>Scale the monitoring around your services: <a href="https://blog.oodle.ai/scaling-prometheus-from-single-node-to-enterprise-grade-observability/">Scaling Prometheus: from single node to enterprise-grade observability</a></li>
<li>The platform these practices power: <a href="https://blog.oodle.ai/how-oodle-keeps-observability-fast-at-scale/">How Oodle keeps observability fast at scale</a></li>
</ul>
</section>
</article>
</main>
<section class="footer-cta outer">
<div class="inner">
<h2 class="footer-cta-title">Sign up for more like this.</h2>
<a class="footer-cta-button" href="#/portal" data-portal>
<div class="footer-cta-input">Enter your email</div>
<span>Subscribe</span>
</a>
</div>
</section>
<aside class="read-more-wrap outer">
<div class="read-more inner">
<article class="post-card post">
<a class="post-card-image-link" href="/observability-for-ai-applications-using-oodle-and-openlit/">
<img class="post-card-image"
srcset="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w300/2026/08/Feature-Image-selection.png 300w,
https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w600/2026/08/Feature-Image-selection.png 600w,
https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w1000/2026/08/Feature-Image-selection.png 1000w,
https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w2000/2026/08/Feature-Image-selection.png 2000w"
sizes="(max-width: 1000px) 400px, 800px"
src="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w600/2026/08/Feature-Image-selection.png"
alt="Observability for AI Applications Using Oodle and OpenLIT"
loading="lazy"
/>
</a>
<div class="post-card-content">
<a class="post-card-content-link" href="/observability-for-ai-applications-using-oodle-and-openlit/">
<header class="post-card-header">
<div class="post-card-tags">
</div>
<h2 class="post-card-title">
Observability for AI Applications Using Oodle and OpenLIT
</h2>
</header>
<div class="post-card-excerpt">Instrument LLM apps and agents with OpenLIT in one line and ship both metrics and traces to Oodle</div>
</a>
<footer class="post-card-meta">
<time class="post-card-meta-date" datetime="2026-08-21">21 Aug 2026</time>
<span class="post-card-meta-length">9 min read</span>
</footer>
</div>
</article>
<article class="post-card post">
<a class="post-card-image-link" href="/someones-hack-is-someone-elses-bug/">
<img class="post-card-image"
srcset="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w300/2026/08/bug-archaeologist.png 300w,
https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w600/2026/08/bug-archaeologist.png 600w,
https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w1000/2026/08/bug-archaeologist.png 1000w,
https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w2000/2026/08/bug-archaeologist.png 2000w"
sizes="(max-width: 1000px) 400px, 800px"
src="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w600/2026/08/bug-archaeologist.png"
alt="Someone&#x27;s Hack is Someone Else&#x27;s Bug"
loading="lazy"
/>
</a>
<div class="post-card-content">
<a class="post-card-content-link" href="/someones-hack-is-someone-elses-bug/">
<header class="post-card-header">
<div class="post-card-tags">
</div>
<h2 class="post-card-title">
Someone&#x27;s Hack is Someone Else&#x27;s Bug
</h2>
</header>
<div class="post-card-excerpt">Story of how a 16 years old hack causes glitches even today</div>
</a>
<footer class="post-card-meta">
<time class="post-card-meta-date" datetime="2026-08-09">09 Aug 2026</time>
<span class="post-card-meta-length">3 min read</span>
</footer>
</div>
</article>
<article class="post-card post">
<a class="post-card-image-link" href="/ai-sensei-make-my-code-go-brrrrr/">
<img class="post-card-image"
srcset="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w300/2026/07/Gemini_Generated_Image_nkx6n5nkx6n5nkx6.png 300w,
https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w600/2026/07/Gemini_Generated_Image_nkx6n5nkx6n5nkx6.png 600w,
https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w1000/2026/07/Gemini_Generated_Image_nkx6n5nkx6n5nkx6.png 1000w,
https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w2000/2026/07/Gemini_Generated_Image_nkx6n5nkx6n5nkx6.png 2000w"
sizes="(max-width: 1000px) 400px, 800px"
src="https://storage.ghost.io/c/b2/48/b2485d36-0dcd-4e46-b7bd-418ab6e38e18/content/images/size/w600/2026/07/Gemini_Generated_Image_nkx6n5nkx6n5nkx6.png"
alt="AI Sensei, Make My Code Go Brrrrr!"
loading="lazy"
/>
</a>
<div class="post-card-content">
<a class="post-card-content-link" href="/ai-sensei-make-my-code-go-brrrrr/">
<header class="post-card-header">
<div class="post-card-tags">
</div>
<h2 class="post-card-title">
AI Sensei, Make My Code Go Brrrrr!
</h2>
</header>
<div class="post-card-excerpt">An engineer&#39;s account of how AI improved critical code, and made him more knowledgeable.</div>
</a>
<footer class="post-card-meta">
<time class="post-card-meta-date" datetime="2026-07-28">28 Jul 2026</time>
<span class="post-card-meta-length">5 min read</span>
</footer>
</div>
</article>
</div>
</aside>
</div>
<footer class="site-footer outer">
<div class="inner">
<section class="copyright"><a href="https://blog.oodle.ai">Oodle AI</a> &copy; 2026</section>
<div class="site-footer-center">
<div class="site-footer-social-links">
<a href="https://x.com/oodleai" target="_blank" rel="noopener" aria-label="X">
<svg class="icon" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M18.2439 2.25H21.5519L14.3249 10.51L22.8269 21.75H16.1699L10.9559 14.933L4.98991 21.75H1.67991L9.40991 12.915L1.25391 2.25H8.07991L12.7929 8.481L18.2439 2.25ZM17.0829 19.77H18.9159L7.08391 4.126H5.11691L17.0829 19.77Z" fill="currentColor"/>
</svg> </a>
</div>
<nav class="site-footer-nav">
<ul class="nav">
<li class="nav-sign-up"><a href="https://us1.oodle.ai/signup">Sign up</a></li>
</ul>
</nav>
</div>
<div class="gh-powered-by"><a href="https://ghost.org/" target="_blank" rel="noopener">Powered by Ghost</a></div>
</div>
</footer>
</div>
<div class="pswp" tabindex="-1" role="dialog" aria-hidden="true">
<div class="pswp__bg"></div>
<div class="pswp__scroll-wrap">
<div class="pswp__container">
<div class="pswp__item"></div>
<div class="pswp__item"></div>
<div class="pswp__item"></div>
</div>
<div class="pswp__ui pswp__ui--hidden">
<div class="pswp__top-bar">
<div class="pswp__counter"></div>
<button class="pswp__button pswp__button--close" title="Close (Esc)"></button>
<button class="pswp__button pswp__button--share" title="Share"></button>
<button class="pswp__button pswp__button--fs" title="Toggle fullscreen"></button>
<button class="pswp__button pswp__button--zoom" title="Zoom in/out"></button>
<div class="pswp__preloader">
<div class="pswp__preloader__icn">
<div class="pswp__preloader__cut">
<div class="pswp__preloader__donut"></div>
</div>
</div>
</div>
</div>
<div class="pswp__share-modal pswp__share-modal--hidden pswp__single-tap">
<div class="pswp__share-tooltip"></div>
</div>
<button class="pswp__button pswp__button--arrow--left" title="Previous (arrow left)"></button>
<button class="pswp__button pswp__button--arrow--right" title="Next (arrow right)"></button>
<div class="pswp__caption">
<div class="pswp__caption__center"></div>
</div>
</div>
</div>
</div>
<script src="https://blog.oodle.ai/assets/built/casper.js?v=YtPhrez3GuGLVDYs" defer></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/tocbot/4.32.2/tocbot.min.js" integrity="sha512-EovZhLMI2Z1FBS6Z37iDltDO4PkOZCmcamGhQHmcfporraL6LSfad582UieUBE3Boh165FVg3NKT1cz4g98V3A==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script>
tocbot.init({
tocSelector: '.gh-toc', // Selector for the ToC container
contentSelector: '.gh-content', // Selector for post content
headingSelector: 'h1, h2, h3, h4', // Select which headings to include
hasInnerContainers: true,
});
</script>
<script>document.addEventListener('DOMContentLoaded',function(){var contentElement=document.querySelector('.gh-content');if(contentElement){var asideElement=document.createElement('aside');asideElement.className='gh-sidebar';asideElement.innerHTML='<div class="gh-toc"></div>';var firstChildElement=contentElement.firstElementChild;if(firstChildElement){firstChildElement.insertAdjacentElement('beforebegin',asideElement)}else{contentElement.appendChild(asideElement)}tocbot.init({tocSelector:'.gh-toc',contentSelector:'.gh-content',headingSelector:'h1,h2,h3,h4',hasInnerContainers:true})}});</script>
</body>
</html>