Files
nexus/sreweekly/articles/361/06-hodor-overload-scenarios-and-the-evolution-of-their-detection-and-hand.html
2026-09-12 17:23:01 +08:00

1614 lines
101 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en">
<head>
<meta name="pageKey" content="d_lithograph_content">
<!----><!----><!----> <meta name="locale" content="en_US">
<!----> <meta id="config" data-app-id="lithograph" data-app-version="0.0.2138" data-call-tree-id="AAZbQqnr+LzkRMv2YtdHFw==" data-multiproduct-name="lithograph-publish-frontend" data-service-name="lithograph-publish-frontend" data-browser-id="94e2e55c-6197-49bb-84f6-931f6251fbf3" data-is-bot="false" data-page-instance="urn:li:page:d_lithograph_content;uk75xYGUQqKpLkUtq2XMiA==" data-disable-jsbeacon-pagekey-suffix="false" data-dna-member-lix-treatment="control" data-human-member-lix-treatment="control" data-dfp-member-lix-treatment="control" data-sync-apfc-headers-lix-treatment="control" data-sync-apfc-cb-lix-treatment="control" data-recaptcha-v3-integration-lix-value="control" data-network-interceptor-lix-value="control" data-is-epd-audit-event-enabled="false" data-is-feed-sponsored-tracking-kill-switch-enabled="false" data-sequence-auto-redirect-before-request-enabled="true">
<link rel="canonical" href="https://www.linkedin.com/blog/engineering/infrastructure/hodor-overload-scenarios-and-the-evolution-of-their-detection-a">
<!----><!---->
<!---->
<!---->
<!---->
<!---->
<link rel="icon" href="https://static.licdn.com/aero-v1/sc/h/al2o9zrvru7aqj8e1x2rzsrca">
<script>
function getDfd() {let yFn,nFn;const p=new Promise(function(y, n){yFn=y;nFn=n;});p.resolve=yFn;p.reject=nFn;return p;}
window.lazyloader = getDfd();
window.tracking = getDfd();
window.impressionTracking = getDfd();
window.ingraphTracking = getDfd();
window.appDetection = getDfd();
window.pemTracking = getDfd();
window.appRedirectCompleted = getDfd();
</script>
<!---->
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Hodor: Overload scenarios and the evolution of their detection and handling</title>
<!---->
<meta property="og:type" content="website">
<meta property="og:title" content="Hodor: Overload scenarios and the evolution of their detection and handling">
<!---->
<meta property="og:url" content="https://www.linkedin.com/blog/engineering/infrastructure/hodor-overload-scenarios-and-the-evolution-of-their-detection-a">
<meta property="og:image" content="https://media.licdn.com/dms/image/v2/D4D08AQFCoYADFeuL7g/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700688351202?e=2147483647&amp;v=beta&amp;t=AWe2SLTiCT4uYYorjfq4WeQqhU-eWw3LxTQuHT1M5sU">
<meta name="twitter:title" content="Hodor: Overload scenarios and the evolution of their detection and handling">
<meta name="twitter:site" content="@linkedin">
<!---->
<!---->
<meta name="robots" content="noarchive">
<meta name="linkedin:pageTag" content="/blog/engineering/infrastructure/hodor-overload-scenarios-and-the-evolution-of-their-detection-a">
<meta name="litmsProfileName" content="lithograph">
<!----> <meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="asset-url" id="artdeco/static/images/icons.svg" content="https://static.licdn.com/aero-v1/sc/h/6sz1e821vgereyxaakx87tt4z ">
<link rel="stylesheet" href="https://static.licdn.com/aero-v1/sc/h/lbz6744ddd0wsc02ey2a3dhx">
<!---->
<!----> </head>
<body dir="ltr">
<!---->
<a href="#lithograph-app" class="skip-link btn-md btn-primary absolute z-11 -top-[100vh] focus:top-0">
Skip to main content
</a>
<!---->
<div class="page-header__wrapper page-header__wrapper--margin" data-custom-nav-wrapper="true">
<div class="page-header__container">
<header id="page-header" class="page-header page-header--hide-dropshadow">
<div class="page-header__content-container">
<div class="header__logo">
<a href="https://www.linkedin.com/blog/engineering" class="custom-nav__logo-link">
<img class="header__linkedin-logo header__full-logo" src="https://static.licdn.com/aero-v1/sc/h/6zm111mce7vohqze950ilreo4" alt="LinkedIn Logo">
<img class="header__linkedin-logo header__in-bug" src="https://static.licdn.com/aero-v1/sc/h/5g0hjlcng3j5pgn50n2et1ca2" alt="LinkedIn Logo">
<span class="t-17 t-black t-bold default-header-title">
Engineering Blog
</span>
</a>
</div>
<!---->
<!---->
<button class="stripped header__expand-nav-button" data-mobile-header-navigation-button data-header-expand-navigation-button aria-expanded="false">
<li-icon class="header__expand-nav-icon" a11y-text="Open navigation" type="hamburger-icon"></li-icon>
</button>
<button class="stripped header__expand-nav-button hidden" data-mobile-header-navigation-button data-header-collapse-navigation-button aria-expanded="false">
<li-icon class="header__expand-nav-icon" a11y-text="Close navigation" type="cancel-icon"></li-icon>
</button>
</div>
</header>
</div>
<nav class="header-nav fixed" aria-label="main" data-header-navigation>
<ul class="header-nav__list">
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/data" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Data
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Data" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Graph System</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/economic-graph" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Economic Graph</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/skills-graph" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Skills Graph</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/knowledge" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Knowledge Graph</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/data-management" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Management</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/data-streaming-processing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Streams/Processing</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/data-science" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Science</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/research" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Research</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/ab-testing-experimentation" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">A/B Testing/Experimentation</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/artificial-intelligence" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
AI
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for AI" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/machine-learning" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Machine Learning</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/natural-language-processing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Natural Language Processing</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/generative-ai" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Generative AI</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/data-modeling" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Modeling</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/responsible-ai" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Responsible AI</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/trust-and-safety" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Trust & Safety
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Trust &amp;amp; Safety" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/fairness" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Fairness</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/verification" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Verification </a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/security" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Security</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/product-design" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Product Design
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Product Design" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/marketing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Marketing </a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/sales" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Sales</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/learning" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Learning</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/hiring" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Hiring</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Profile</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/messaging-notifications" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Messaging/Notifications</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/feed" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Feed</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Profile</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/groups" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Groups</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/accessibility" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Accessibility</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/member-customer-experience" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Member/Customer Experience</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/search" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Search</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/recommendations" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Recommendations</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/skills" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Skills</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/infrastructure" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Infrastructure
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Infrastructure" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/code" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Code</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/scalability" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Scalability</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/distributed-systems" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Distributed Systems</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Graph Systems</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/architecture" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Architecture</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/analytics" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Analytics</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/automation" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Automation</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/platform-platformization" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Platforms/Platformization</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/cloud-computing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Cloud Computing</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/open-source" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Open Source</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/additional-innovations" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Additional Innovations</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/talent" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Talent
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Talent" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/culture" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Culture</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/developer-experience-productivity" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Developer Experience/Productivity</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/leadership" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Leadership</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
</ul>
</nav>
<nav class="custom-header__nav--mobile invisible" data-mobile-navigation="true" aria-label="main">
<ul class="custom-header__list--mobile">
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/data" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Data
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Data" aria-expanded="false" data-label-expand="Expand to show more links for Data" data-label-collapse="Collapse to show less links for Data" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-sans t-black t-normal header-nav__link" target="_self">Graph System</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/economic-graph" class="t-sans t-black t-normal header-nav__link" target="_self">Economic Graph</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/skills-graph" class="t-sans t-black t-normal header-nav__link" target="_self">Skills Graph</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/knowledge" class="t-sans t-black t-normal header-nav__link" target="_self">Knowledge Graph</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/data-management" class="t-sans t-black t-normal header-nav__link" target="_self">Data Management</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/data-streaming-processing" class="t-sans t-black t-normal header-nav__link" target="_self">Data Streams/Processing</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/data-science" class="t-sans t-black t-normal header-nav__link" target="_self">Data Science</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/research" class="t-sans t-black t-normal header-nav__link" target="_self">Research</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/ab-testing-experimentation" class="t-sans t-black t-normal header-nav__link" target="_self">A/B Testing/Experimentation</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/artificial-intelligence" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
AI
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for AI" aria-expanded="false" data-label-expand="Expand to show more links for AI" data-label-collapse="Collapse to show less links for AI" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/machine-learning" class="t-sans t-black t-normal header-nav__link" target="_self">Machine Learning</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/natural-language-processing" class="t-sans t-black t-normal header-nav__link" target="_self">Natural Language Processing</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/generative-ai" class="t-sans t-black t-normal header-nav__link" target="_self">Generative AI</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/data-modeling" class="t-sans t-black t-normal header-nav__link" target="_self">Data Modeling</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/responsible-ai" class="t-sans t-black t-normal header-nav__link" target="_self">Responsible AI</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/trust-and-safety" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Trust &amp; Safety
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Trust &amp;amp; Safety" aria-expanded="false" data-label-expand="Expand to show more links for Trust &amp;amp; Safety" data-label-collapse="Collapse to show less links for Trust &amp;amp; Safety" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/fairness" class="t-sans t-black t-normal header-nav__link" target="_self">Fairness</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/verification" class="t-sans t-black t-normal header-nav__link" target="_self">Verification </a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/security" class="t-sans t-black t-normal header-nav__link" target="_self">Security</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/product-design" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Product Design
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Product Design" aria-expanded="false" data-label-expand="Expand to show more links for Product Design" data-label-collapse="Collapse to show less links for Product Design" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/marketing" class="t-sans t-black t-normal header-nav__link" target="_self">Marketing </a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/sales" class="t-sans t-black t-normal header-nav__link" target="_self">Sales</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/learning" class="t-sans t-black t-normal header-nav__link" target="_self">Learning</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/hiring" class="t-sans t-black t-normal header-nav__link" target="_self">Hiring</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-sans t-black t-normal header-nav__link" target="_self">Profile</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/messaging-notifications" class="t-sans t-black t-normal header-nav__link" target="_self">Messaging/Notifications</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/feed" class="t-sans t-black t-normal header-nav__link" target="_self">Feed</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-sans t-black t-normal header-nav__link" target="_self">Profile</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/groups" class="t-sans t-black t-normal header-nav__link" target="_self">Groups</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/accessibility" class="t-sans t-black t-normal header-nav__link" target="_self">Accessibility</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/member-customer-experience" class="t-sans t-black t-normal header-nav__link" target="_self">Member/Customer Experience</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/search" class="t-sans t-black t-normal header-nav__link" target="_self">Search</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/recommendations" class="t-sans t-black t-normal header-nav__link" target="_self">Recommendations</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/skills" class="t-sans t-black t-normal header-nav__link" target="_self">Skills</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/infrastructure" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Infrastructure
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Infrastructure" aria-expanded="false" data-label-expand="Expand to show more links for Infrastructure" data-label-collapse="Collapse to show less links for Infrastructure" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/code" class="t-sans t-black t-normal header-nav__link" target="_self">Code</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/scalability" class="t-sans t-black t-normal header-nav__link" target="_self">Scalability</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/distributed-systems" class="t-sans t-black t-normal header-nav__link" target="_self">Distributed Systems</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-sans t-black t-normal header-nav__link" target="_self">Graph Systems</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/architecture" class="t-sans t-black t-normal header-nav__link" target="_self">Architecture</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/analytics" class="t-sans t-black t-normal header-nav__link" target="_self">Analytics</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/automation" class="t-sans t-black t-normal header-nav__link" target="_self">Automation</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/platform-platformization" class="t-sans t-black t-normal header-nav__link" target="_self">Platforms/Platformization</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/cloud-computing" class="t-sans t-black t-normal header-nav__link" target="_self">Cloud Computing</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/open-source" class="t-sans t-black t-normal header-nav__link" target="_self">Open Source</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/additional-innovations" class="t-sans t-black t-normal header-nav__link" target="_self">Additional Innovations</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/talent" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Talent
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Talent" aria-expanded="false" data-label-expand="Expand to show more links for Talent" data-label-collapse="Collapse to show less links for Talent" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/culture" class="t-sans t-black t-normal header-nav__link" target="_self">Culture</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/developer-experience-productivity" class="t-sans t-black t-normal header-nav__link" target="_self">Developer Experience/Productivity</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/leadership" class="t-sans t-black t-normal header-nav__link" target="_self">Leadership</a>
</li>
</ul>
</li>
</ul>
<div class="custom-header__button-share-container--mobile">
<!---->
</div>
</nav>
</div>
<main role="main">
<div id="lithograph-app">
<section id="component-container" data-attach-collapse-behavior="true">
<div id class="component component-articleHeadline" data-component-type="articleHeadline">
<div class="article-headline-container">
<p class="article-headline__topic">
<a class="t-14" href="https://www.linkedin.com/blog/engineering/infrastructure">Infrastructure</a>
</p>
<h1 class="article-headline t-40" data-article-title="Hodor: Overload scenarios and the evolution of their detection and handling">Hodor: Overload scenarios and the evolution of their detection and handling</h1>
<div class="article-meta t-14 article-meta__hairline">
<div class="article-meta__byline-container">
<p class="t-14 t-black--light" data-published-date="2023-2-23">
February 23, 2023
</p>
</div>
<div class="social-share">
<ul class="social-share__list share-list" title="Share via" data-share-list>
<li class="share-list__item share-item">
<a href="#" class="share-item__link medium-round-muted-tertiary-button" role="button" data-tracking-control-name="_linkedin_share" data-share-type="linkedin" aria-label="Share on LinkedIn, New window will open">
<li-icon class="share-item__icon" type="linkedin-icon"></li-icon>
</a>
</li>
<li class="share-list__item share-item">
<a href="#" class="share-item__link medium-round-muted-tertiary-button" role="button" data-tracking-control-name="_facebook_share" data-share-type="facebook" aria-label="Share on Facebook, New window will open">
<li-icon class="share-item__icon" type="facebook-icon"></li-icon>
</a>
</li>
<li class="share-list__item share-item">
<a href="#" class="share-item__link medium-round-muted-tertiary-button" role="button" data-tracking-control-name="_twitter_share" data-share-type="twitter" aria-label="Share on X, New window will open">
<svg class="share-item__icon share-item__icon--twitter" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" fill="currentColor" focusable="false" aria-hidden="true" role="none">
<path d="M13.9 10.47L21.35 2h-1.76l-6.46 7.35L7.96 2H2l7.81 11.12L2 22h1.76l6.83-7.77L16.04 22H22l-8.1-11.53zm-2.42 2.75l-.79-1.11L4.4 3.3h2.71l5.08 7.11.79 1.11 6.6 9.24h-2.71l-5.39-7.54z"></path>
</svg>
</a>
</li>
</ul>
</div>
</div>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>Co-Authors -&nbsp;&nbsp;<a href="https://www.linkedin.com/in/abhishg/" target="_blank" data-tracking-control-name="_anchor_link">Abhishek Gilra</a>, <a href="https://www.linkedin.com/in/nadanidentity/" target="_blank" data-tracking-control-name="_anchor_link">Nizar Mankulangara</a>, <a href="https://www.linkedin.com/in/salilkanitkar/" target="_blank" data-tracking-control-name="_anchor_link">Salil Kanitkar</a>, and&nbsp;<a href="https://www.linkedin.com/in/vivdesh/" target="_blank" data-tracking-control-name="_anchor_link">Vivek Deshpande</a></p>
<h2>Introduction</h2>
<p>To connect professionals and make them more productive, it is crucial that LinkedIn is available at all times. For us, downtime means that our members and customers don’t have access to the conversations, connections, and knowledge that are essential to them achieving their objectives.. For linkedin.com to achieve our goal of 99.9% availability, there are ~1000 carefully orchestrated services working in tandem. This means that all of these underlying services &nbsp;need to be highly available and reliable.</p>
<p>Offering such high availability to deliver the desired member and customer experience necessitates being able to, in real-time, identify and remediate different types of overloads that can plague a system. Left to their own devices, overloads can prove critical, eventually bringing down the entire site.</p>
<p>In our previous blog post, <a href="https://engineering.linkedin.com/blog/2022/hodor--detecting-and-addressing-overload-in-linkedin-microservic" target="_blank" data-tracking-control-name="_anchor_link">Hodor: Detecting and addressing overload in LinkedIn microservices</a>, we introduced the Holistic Overload Detection and Overload Remediation (HODOR) framework. That blog post introduced the first detector we had designed for Hodor, the heartbeat detector, and discussed our concurrency based load-shedding strategy to remediate overloads. We also discussed how member impact is minimized by safely retrying dropped requests.</p>
<p>In this post, we’ll discuss overload scenarios that we have observed at LinkedIn, and revisit the goals of Hodor. We’ll also introduce new tools: our garbage collection (GC) overload detector and our application threadpool detector. Finally, we’ll talk about the concept of traffic tiering, which we introduced to minimize member impact while shedding load.</p>
<h2>Goals of Hodor</h2>
<p>Hodor serves to sense danger and protect against that danger (similar to the beloved fictional character). In this case, the danger is overloads to microservices. Here are the four main goals of Hodor:</p>
<ol>
<li><p>Detect different types of physical and virtual overloads in real-time with minimal overhead. This is important because Hodor runs on all services at all times with the goal of improving performance and availability. So any overhead because of Hodor is highly undesirable (and somewhat ironic).</p> </li>
<li><p>Take evasive action to mitigate overloads to improve resilience and availability, which currently includes load-shedding.</p> </li>
<li><p>Be an out-of-the-box solution that works for all LinkedIn services without service owners or SREs needing to tune the algorithm.</p> </li>
<li><p>Result in a net positive member impact. This is the most important goal – we built this solution to improve member experience so a net negative member experience because of Hodor is unacceptable.&nbsp;</p> </li>
</ol>
<h2>Holistic overload detection</h2>
<p>Hodor detects different types of overloads using three detectors:&nbsp;</p>
<ol>
<li><p>The heartbeat detector for CPU overloads (covered in our <a href="https://engineering.linkedin.com/blog/2022/hodor--detecting-and-addressing-overload-in-linkedin-microservic" data-tracking-control-name="_anchor_link">previous blog post</a>)&nbsp;</p> </li>
<li><p>The GC detector</p> </li>
<li><p>An application threadpool detector</p> </li>
</ol>
<p>Additionally, Hodor uses a latency confirmation filter to prevent false positive detections.&nbsp;</p>
<p>As the CPU heartbeat detector was covered in our previous post, we will focus on the GC detector, the application threadpool detector, and the latency confirmation filter.</p>
<h3>Garbage Collection (GC) detector</h3>
<p>LinkedIn production is primarily composed of Java microservices running on the Java Virtual Machine (JVM), therefore, we needed a mechanism to quickly and accurately detect increased garbage collection activity from auto-memory management. The idea is to observe the overhead of GC activity to detect overloads in real-time.</p>
<p>On each GC event, an amortized percentage of time spent in GC over a given timeframe is calculated, which is called the GC overhead. A schedule is applied on top of this GC overhead and the GC overhead percentage range is divided into tiers called GC overhead tiers. If the duration spent in a GC overhead tier exceeds the violation period for that tier, a GC overload is signaled. The violation period is smaller for higher GC overhead tiers because a higher GC overhead tier indicates more severe GC activity.&nbsp;</p>
<h3>Application threadpool detector</h3>
<p>A downstream service might become slow to respond, causing requests to cumulate upstream. This is known as backpressure. For example, in Figure 1, traffic moving from server A to server B to server C flows smoothly before server C starts to slow down. This results in server B experiencing backpressure from server C. This backpressure can percolate all the way up the call-tree to server A if left unchecked.</p>
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQHpmvLXBNvAUA/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700688351244?e=2147483647&amp;v=beta&amp;t=sff0juh190ZiZXGtoWf4cOfPaWGSuctr-R1bwsCH76Q" alt="Graphic of formation of server backpressure ">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>Figure 1: Formation of server backpressure</p>
<p>Backpressure typically results in a virtual resource overload (e.g., threadpool exhaustion) before a physical resource limitation is hit.</p>
<p>This is currently Hodor's only virtual resource overload detector. When developing the threadpool detector, the first metric that we considered was queue length because it is bound to grow when there is backpressure. The idea was to monitor the queue length to perceive the overload state of a service. However, this was quickly dismissed because the queue length is difficult to correlate back to a business impact indicator, such as client perceived metric. Also, the queue length at which a service can be considered overloaded may vary based on service design and business logic characteristics.&nbsp;</p>
<p>Queue wait times addressed the above concerns. Via experimentation, we found that there is an inflection point where queue wait times reach a point of no return for a service. Identifying this allowed us to design a detector that casts a wide net on service overloads. This helps us avoid client-perceived latency, which can result in a very real business impact.</p>
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQGReUJXZ6JPag/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700688351182?e=2147483647&amp;v=beta&amp;t=J72gtRTsgRaO-lWdTTblZovuMy514o1xikagNgchb6I" alt="Graphic of threadpool overload detection using queue wait times">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>Figure 2: Threadpool overload detection using queue wait times</p>
<p>Consider a synchronous service — requests spend more time waiting in the queue if current request processing time increases, either due to an issue in the current service or in one of its downstreams. The capacity of a service can also be reached when latencies of downstream traffic increase. This can cause the number of concurrent in-process requests in the current service to increase with no change to the incoming request rate. Without knowing anything about the downstream service, the upstream service can monitor its own threadpool queue wait-time to ascertain a downstream service's distress. This can then be alleviated by dropping traffic at the upstream itself. The concept of observing wait times to detect virtual resource overloads can be generalized and widely applied.</p>
<h2>Overload detection case study</h2>
<p>The following three graphs show distinct examples of overload in production systems. Notice that each of the three detector firings correlate with regressing business metrics, therefore showing successful overload detection.</p>
<h3>Overload detection using heartbeat detector</h3>
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQFpjb0wmH5V2w/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700688353798?e=2147483647&amp;v=beta&amp;t=pJDYGvdvHyMaX2NGGsWa66DOsjtuZxzrYy3U2U9C_Vc" alt="Graphic of heartbeat detection with corresponding latency increases in production ">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>Figure 3a: Heartbeat detection with corresponding latency increases in production</p>
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQEA28cNHcHvMQ/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700688351795?e=2147483647&amp;v=beta&amp;t=I2kBneFZcmIwnBLLtBb_zSGgCe__n4jExOesaM-lmKU" alt="Graphic of GC overload detection with corresponding latency increases in production">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>Figure 3b: GC overload detection with corresponding latency increases in production</p>
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQH8J6TiMy2geQ/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700688353627?e=2147483647&amp;v=beta&amp;t=FofwQDgyI9645QON_0puk2974WlL-fVkhAPIs7kYHkw" alt="FiGraphic of threadpool overload detection with corresponding latency increases in production">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>Figure 3c: Threadpool overload detection with corresponding latency increases in production</p>
<h2>Latency confirmation filter</h2>
<p>As mentioned previously, Hodor’s overload detectors are designed to optimize for precision over recall. However, after Hodor was rolled out to all LinkedIn services in production (about 1% of services), suboptimally-tuned garbage collection activity could result in micro stop-the-world GC pauses and the background heartbeat thread to experience CPU starvation. This would cause the heartbeat detector to fire a CPU overload even though there is no discernible impact on business metrics such as service latency.</p>
<p>To account for business impact, we designed the latency confirmation detector which uses a modified moving average crossover algorithm. Two sliding windows of significantly varying sizes are used to compute a slow moving average and a fast moving average. When the fast moving average crosses over the slow moving average, shifted higher by a certain factor, a latency confirmation is flagged. This algorithm is far from perfect for standalone anomaly detection, however, it works well with the heartbeat and GC detector as a confirmation filter to detect sudden and significant increases in latency.</p>
<p>Using latency as a metric to develop a primary detector was never a viable option because not every latency increase is an overload, but every physical resource overload eventually results in a latency increase.</p>
<h3>Latency confirmation filter case study</h3>
<p>In Figure 4, the top row contains graphs showing the pure heartbeat overload signal firing even though there is no impact to latency. The confirmation filter successfully suppresses this false firing. The bottom row in this diagram contains graphs showing the pure heartbeat overload being confirmed by the latency filter because there is indeed a latency impact.</p>
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQG4efbU-eSubg/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700688353812?e=2147483647&amp;v=beta&amp;t=gM-dIW1MdXBR7L2YO1UXGT9OcEhKsPZYt2GmKHpADEU" alt="FiguGraphic of latency confirmation filter suppressing possible false positives in heartbeat detection">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>Figure 4: Latency confirmation filter suppressing possible false positives in heartbeat detection</p>
<h2>Overload remediation</h2>
<p>LinkedIn microservices receive requests from members actively using the platform and from offline systems that need additional data from services. Needless to say, requests from members are more important than requests from offline jobs. Therefore, to prioritize dropping lower priority requests first, we introduced the concept of traffic tiering.</p>
<h3>Tier Assignments</h3>
<p>At LinkedIn, there are three traffic tiers: optional, degradable, and non-degradable.&nbsp;</p>
<p>Optional requests are those that can be dropped with no user impact at all. Most of the traffic from offline/nearline systems falls into this tier.</p>
<p>Degradable requests have some impact on the user experience, but if dropped, the impact is minimal. The primary use case for this tier is for product owners to mark requests loading degradable features on a certain page. This will help protect the backend during an overload. In the offline/nearline world, this is also used for the traffic that is time sensitive or critical due to other business requirements and considered more important than the generic offline/nearline requests.</p>
<p>The non-degradable tier is used for requests from online members which cannot be degraded.</p>
<p>To support future evolution of tiers, a numeric tier ID has been used to allow relative comparison and further prioritization of requests if needed. The optional tier is represented as tier ID 1000, the degradable tier uses ID 5000, and the non-degradable tier uses ID 10000. The bigger the number, the more important the traffic, and using larger numbers for more important traffic gives us more room to go higher.</p>
<p>The traffic tier of a request is passed along the distributed call tree. Once the traffic tier is set, it is used for the rest of the call tree. Irrespective of where the request originated from, the first service/application in our data center that handles the request sets a tier in the distributed call tree if one is not already set.</p>
<h3>Load shedder integration</h3>
<p>Requests are prioritized for dropping based on these traffic tiers. Optional requests are dropped in favor of degradable requests, whereas degradable requests are dropped in favor of non-degradable requests. When there are enough optional/degradable requests present, we avoid dropping non-degradable requests to minimize user impact of load-shedding.</p>
<p>Within a tier, users are placed into different buckets by simply hashing member IDs. We drop requests from selected buckets to ensure a consistent experience and minimize the number of impacted call trees, and therefore members and customers. If the service is unable to recover after dropping a small number of buckets or lower tier requests, the threshold can be raised to shed more buckets or move to a higher tier. This process can continue until the system enters a stable state, or is able to recover fully and again begin accepting all traffic.</p>
<p>In our <a href="https://engineering.linkedin.com/blog/2022/hodor--detecting-and-addressing-overload-in-linkedin-microservic" target="_blank" data-tracking-control-name="_anchor_link">previous blog post</a>, we discussed a concurrency based load-shedding algorithm, which employed concurrency as the normalization metric to represent the load on a system. In the concurrency-based load shedder, the concurrency value was used to modulate the load on the system by deciding how much traffic to shed. During the integration of traffic tiering with concurrency based load shedder, we found that concurrency based histograms do not work as expected because the requests dropped do not contribute anything meaningful to the concurrency histogram.&nbsp;</p>
<p>Due to this limitation, Hodor introduced a rate based load shedding strategy and a corresponding rate based histogram per tier. The rate based algorithm works the sameas the percentage and concurrency based algorithms described earlier, except that it uses throughput as the load metric. The load shedder now uses a throughput based histogram to track traffic tiers based on the incoming traffic patterns.</p>
<p>At runtime, for each endpoint, a histogram is built for the distribution of traffic tiers and user buckets of the incoming requests, which allows for more effective load-shedding. For instance, if a service only receives degradable traffic, then the algorithm knows that there is no lower priority than degradable to shed.</p>
<p>The traffic-tiering, load-shedder queries the histogram to identify the right traffic tier and bucket to drop. Any traffic lower than the identified tier and bucket is dropped by the load shedder on overload.&nbsp;</p>
<h2>Loadshedding to prevent overload case study</h2>
<p>The following graphs (Figure 5) are from an A/B experiment that was conducted. The red host was not protected by load shedding, whereas the blue host had load shedding enabled. Observe that as the overall QPS increases, the protected host is forced to increase the number of requests that are dropped. Also observe that the overall tail latency is lower on the protected blue host, however there are a few spikes where the load shedder is probing to increase the concurrency limit. In contrast, the tail latencies on the unprotected red host are much worse.</p>
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQGhc0Vjb6qp2g/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700688353075?e=2147483647&amp;v=beta&amp;t=d8aoUvmPvovpokCJemfdiO5kVUblT40Y93mX37kZ80w" alt="Graphic of load shedding and corresponding improvement in business metrics as a result">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>Figure 5: Load shedding and corresponding improvement in business metrics as a result</p>
<h2>Conclusion</h2>
<p>Hodor is currently running on 1000+ microservices at LinkedIn and has prevented hundreds of overloads in LinkedIn production systems. Due to its success and accuracy, there is widespread interest across LinkedIn to expand the use of Hodor’s signals for autoscaling and safe load testing in production. New detectors are also being considered for other types of virtual overloads and overloads in data services.&nbsp;</p>
<p>Under the watchful eye of Hodor, we hope to keep our teams safe from overloads and our members and customers safe from downtime.</p>
<h2>Acknowledgements</h2>
<p>This project would not have been possible without the dedication and hard work for multiple quarters by a wide range of talented people. A huge thank you to the core team: <a href="https://www.linkedin.com/in/winstonmzhang/" target="_blank" data-tracking-control-name="_anchor_link">Winston Zhang</a>, <a href="https://www.linkedin.com/in/ssheng/" target="_blank" data-tracking-control-name="_anchor_link">Sean Sheng</a>, <a href="https://www.linkedin.com/in/bryanbarkley/" target="_blank" data-tracking-control-name="_anchor_link">Bryan Barkley</a>, <a href="https://www.linkedin.com/in/xiabingfeng/" target="_blank" data-tracking-control-name="_anchor_link">Bingfeng Xia</a>, <a href="https://www.linkedin.com/in/supriya-madhuram-00a1627/" target="_blank" data-tracking-control-name="_anchor_link">Supriya Madhuram</a>, <a href="https://www.linkedin.com/in/dmitry-shestoperov-5a6a1b34/" target="_blank" data-tracking-control-name="_anchor_link">Dmitry Shestoperov</a>, <a href="https://www.linkedin.com/in/chris-stufflebeam-66276694/" target="_blank" data-tracking-control-name="_anchor_link">Chris Stufflebeam</a>, <a href="https://www.linkedin.com/in/zhuoqun-c/" target="_blank" data-tracking-control-name="_anchor_link">Bill Chen</a>, <a href="https://www.linkedin.com/in/chrisyzhang/" target="_blank" data-tracking-control-name="_anchor_link">Chris Zhang</a>, <a href="https://www.linkedin.com/in/rick-zhou/" target="_blank" data-tracking-control-name="_anchor_link">Rick Zhou</a>, <a href="https://www.linkedin.com/in/ronmeng/" target="_blank" data-tracking-control-name="_anchor_link">Ron Meng</a>, <a href="https://www.linkedin.com/in/johnpstewart/" target="_blank" data-tracking-control-name="_anchor_link">John Stewart</a>, <a href="https://www.linkedin.com/in/yaojialyu/" target="_blank" data-tracking-control-name="_anchor_link">Yaojia Lyu</a>, <a href="https://www.linkedin.com/in/nsurendranath/" target="_blank" data-tracking-control-name="_anchor_link">Neha Surendranath</a>, and <a href="https://www.linkedin.com/in/dbhatnagar/" target="_blank" data-tracking-control-name="_anchor_link">Deepak Bhatnagar</a>. Additionally, a ton of gratitude goes to <a href="https://www.linkedin.com/in/amishra01/" target="_blank" data-tracking-control-name="_anchor_link">Ash Mishra</a>, <a href="https://www.linkedin.com/in/gokselgenc/" target="_blank" data-tracking-control-name="_anchor_link">Goksel Genc</a>, <a href="https://www.linkedin.com/in/riteshmaheshwari/" target="_blank" data-tracking-control-name="_anchor_link">Ritesh Maheshwari</a>, <a href="https://www.linkedin.com/in/shivaraman/" target="_blank" data-tracking-control-name="_anchor_link">Shiva Raman</a>, <a href="https://www.linkedin.com/in/eddie-bernard-5497a02/" target="_blank" data-tracking-control-name="_anchor_link">Eddie Bernard</a>, and <a href="https://www.linkedin.com/in/heather-mckelvey-4b31441/" target="_blank" data-tracking-control-name="_anchor_link">Heather McKelvey</a> for their support and encouragement of the project. Special thanks also to <a href="https://www.linkedin.com/in/scott-meyer-0155872/" target="_blank" data-tracking-control-name="_anchor_link">Scott Meyer</a>, <a href="https://www.linkedin.com/in/dbuthay/" target="_blank" data-tracking-control-name="_anchor_link">Diego Buthay</a>, and <a href="https://www.linkedin.com/in/madubey/" target="_blank" data-tracking-control-name="_anchor_link">Manish Dubey</a> for their guidance and feedback. Finally, thank you to <a href="https://www.linkedin.com/in/kaylaguglielmo/" target="_blank" data-tracking-control-name="_anchor_link">Kayla Gugliemo</a>, <a href="https://www.linkedin.com/in/nishw/" target="_blank" data-tracking-control-name="_anchor_link">Nishan Weragama</a>, <a href="https://www.linkedin.com/in/gregearl" target="_blank" data-tracking-control-name="_anchor_link">Greg Earl</a>, <a href="https://www.linkedin.com/in/benito-leyva-4684014b" target="_blank" data-tracking-control-name="_anchor_link">Benito Levya</a>, and <a href="https://www.linkedin.com/in/khvaiente" target="_blank" data-tracking-control-name="_anchor_link">Katherine Vaiente</a> for helping to review and refine this post.</p>
</div>
</div>
<div id class="component component-postList" data-component-type="postList">
<section id="postList0FocusPoint" class="component__content-container list-layout" data-post-list>
<section class="simple-topic-list">
<p class="t-14 t-bold">Topics:
<a class="simple-topic-list__topic" href="https://www.linkedin.com/blog/engineering/data-management">
Data Management
</a>
<a class="simple-topic-list__topic" href="https://www.linkedin.com/blog/engineering/infrastructure">
Infrastructure
</a>
</p>
</section>
<hr class="artdeco-divider">
<section class="list-layout__headline-container">
<p class="list-layout__headline t-32 t-sans">Related articles</p>
<!---->
</section>
<ul class="list-layout__posts">
<li class="post-list__item list-post">
<div class="list-post__content-container">
<p class="list-post__content-container__topic">
<a class="t-14 t-bold" href="https://www.linkedin.com/blog/engineering/infrastructure">
Infrastructure
</a>
</p>
<div class="list-post__content-container__title">
<a class="list-post__link t-20 t-black" href="https://www.linkedin.com/blog/engineering/infrastructure/the-training-infrastructure-behind-ai-powered-job-search-eight-x-faster-multi-teacher-distillation">
The Training Infrastructure Behind AI-Powered Job Search: 8X F...
</a>
</div>
<div class="list-post__content-container__byline">
<p class="list-post__content-container__author t-14 t-bold t-black">
Jaideep Ray
</p>
<!---->
<p class="list-post__content-container__date t-14 t-black--light">
Aug 6, 2026
</p>
</div>
</div>
<div class="list-post__image-container">
<figure class="post__image-figure">
<img class="post__image" alt data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQEctZ-EpMCamw/croft-frontend-shrinkToFit600/B4DZ_Ti6GaGYAQ-/0/1785960571979?e=2147483647&amp;v=beta&amp;t=gpPNPoM7OkjDl_PzMT4Tub1yA6bII-BPfO1uqIYeHRI" data-ghost-url="https://static.licdn.com/aero-v1/sc/h/8pouyb1ly746xtru0x7p2axm0" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 480px">
</figure>
</div>
</li>
<li class="post-list__item list-post">
<div class="list-post__content-container">
<p class="list-post__content-container__topic">
<a class="t-14 t-bold" href="https://www.linkedin.com/blog/engineering/ai">
AI
</a>
</p>
<div class="list-post__content-container__title">
<a class="list-post__link t-20 t-black" href="https://www.linkedin.com/blog/engineering/ai/qa-agent-reimagining-software-quality-with-ai-driven-autonomous-testing">
Quality Assurance Agent: Reimagining Software Quality with AI-...
</a>
</div>
<div class="list-post__content-container__byline">
<p class="list-post__content-container__author t-14 t-bold t-black">
Asa Kusuma
</p>
<!---->
<p class="list-post__content-container__date t-14 t-black--light">
Jun 18, 2026
</p>
</div>
</div>
<div class="list-post__image-container">
<figure class="post__image-figure">
<img class="post__image" alt data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQFWxUw04ihjYQ/croft-frontend-shrinkToFit600/B4DZ7bHNPnGkAQ-/0/1781792560756?e=2147483647&amp;v=beta&amp;t=KiyIbEztZVaRRSzY53ezbvpjY4EYCksSDoMR7R8--Rk" data-ghost-url="https://static.licdn.com/aero-v1/sc/h/8pouyb1ly746xtru0x7p2axm0" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 480px">
</figure>
</div>
</li>
<li class="post-list__item list-post">
<div class="list-post__content-container">
<!---->
<div class="list-post__content-container__title">
<a class="list-post__link t-20 t-black" href="https://www.linkedin.com/blog/engineering/platform-platformization/accelerating-go-to-market-velocity-with-the-product-configuration-center">
Accelerating Go-To-Market Velocity with the Product Configurat...
</a>
</div>
<div class="list-post__content-container__byline">
<p class="list-post__content-container__author t-14 t-bold t-black">
Tanmay Khemka
</p>
<p class="list-post__content-container__date t-14 t-black--light">
Jun 12, 2026
</p>
</div>
</div>
<div class="list-post__image-container">
<figure class="post__image-figure">
<img class="post__image" alt data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQFOqEs5bho66Q/croft-frontend-shrinkToFit1920/B4DZ64OKZFKAAM-/0/1781207181422?e=2147483647&amp;v=beta&amp;t=rjSx77PGu8owDXBzSBjbV_W0ZbtYnq5c3Zu8LYbmOtM" data-ghost-url="https://static.licdn.com/aero-v1/sc/h/8pouyb1ly746xtru0x7p2axm0" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 480px">
</figure>
</div>
</li>
</ul>
<!----> </section>
</div>
<!----> </section>
</div>
<script src="https://static.licdn.com/aero-v1/sc/h/8hfbuq1ftcvnnx4dd5067pi0t" async></script>
<script src="https://static.licdn.com/aero-v1/sc/h/bknx7jzs0ylu540sn4wuo6bue" async></script>
<script src="https://static.licdn.com/aero-v1/sc/h/egqq1vs0784qd3h3801e2w9lr" async></script>
</main>
<footer class="li-footer bg-transparent w-full ">
<ul class="li-footer__list flex flex-wrap flex-row items-start justify-start w-full h-auto min-h-[50px] my-[0px] mx-auto py-3 px-2 papabear:p-0">
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<span class="sr-only">LinkedIn</span>
<icon class="li-footer__copy-logo text-color-logo-brand-alt inline-block self-center h-[14px] w-[56px] mr-1" data-delayed-url="https://static.licdn.com/aero-v1/sc/h/5mebydpuuijm3uhv1q375inqh"></icon>
<span class="li-footer__copy-text flex items-center">&copy; 2026</span>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://about.linkedin.com?trk=content_footer-about" data-tracking-control-name="content_footer-about" data-tracking-will-navigate>
About
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/accessibility?trk=content_footer-accessibility" data-tracking-control-name="content_footer-accessibility" data-tracking-will-navigate>
Accessibility
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/user-agreement?trk=content_footer-user-agreement" data-tracking-control-name="content_footer-user-agreement" data-tracking-will-navigate>
User Agreement
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/privacy-policy?trk=content_footer-privacy-policy" data-tracking-control-name="content_footer-privacy-policy" data-tracking-will-navigate>
Privacy Policy
</a>
</li>
<!---->
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/cookie-policy?trk=content_footer-cookie-policy" data-tracking-control-name="content_footer-cookie-policy" data-tracking-will-navigate>
Cookie Policy
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/copyright-policy?trk=content_footer-copyright-policy" data-tracking-control-name="content_footer-copyright-policy" data-tracking-will-navigate>
Copyright Policy
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://brand.linkedin.com/policies?trk=content_footer-brand-policy" data-tracking-control-name="content_footer-brand-policy" data-tracking-will-navigate>
Brand Policy
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/psettings/guest-controls?trk=content_footer-guest-controls" data-tracking-control-name="content_footer-guest-controls" data-tracking-will-navigate>
Guest Controls
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/help/linkedin/answer/34593?trk=content_footer-community-guide" data-tracking-control-name="content_footer-community-guide" data-tracking-will-navigate>
Community Guidelines
</a>
</li>
<!---->
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<div class="collapsible-dropdown collapsible-dropdown--footer collapsible-dropdown--up flex items-center relative hyphens-auto language-selector z-2">
<!---->
<ul class="collapsible-dropdown__list hidden container-raised absolute w-auto overflow-y-auto flex-col items-stretch z-[9999] bottom-[100%] top-auto" role="menu" tabindex="-1">
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="العربية (Arabic)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ar_AE" data-locale="ar_AE" role="menuitem" lang="ar_AE">
العربية (Arabic)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="বাংলা (Bangla)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-bn_IN" data-locale="bn_IN" role="menuitem" lang="bn_IN">
বাংলা (Bangla)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Čeština (Czech)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-cs_CZ" data-locale="cs_CZ" role="menuitem" lang="cs_CZ">
Čeština (Czech)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Dansk (Danish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-da_DK" data-locale="da_DK" role="menuitem" lang="da_DK">
Dansk (Danish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Deutsch (German)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-de_DE" data-locale="de_DE" role="menuitem" lang="de_DE">
Deutsch (German)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Ελληνικά (Greek)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-el_GR" data-locale="el_GR" role="menuitem" lang="el_GR">
Ελληνικά (Greek)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="English (English) selected" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link--selected" data-tracking-control-name="language-selector-en_US" data-locale="en_US" role="menuitem" lang="en_US">
<strong>English (English)</strong>
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Español (Spanish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-es_ES" data-locale="es_ES" role="menuitem" lang="es_ES">
Español (Spanish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="فارسی (Persian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-fa_IR" data-locale="fa_IR" role="menuitem" lang="fa_IR">
فارسی (Persian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Suomi (Finnish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-fi_FI" data-locale="fi_FI" role="menuitem" lang="fi_FI">
Suomi (Finnish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Français (French)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-fr_FR" data-locale="fr_FR" role="menuitem" lang="fr_FR">
Français (French)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="हिंदी (Hindi)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-hi_IN" data-locale="hi_IN" role="menuitem" lang="hi_IN">
हिंदी (Hindi)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Magyar (Hungarian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-hu_HU" data-locale="hu_HU" role="menuitem" lang="hu_HU">
Magyar (Hungarian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Bahasa Indonesia (Indonesian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-in_ID" data-locale="in_ID" role="menuitem" lang="in_ID">
Bahasa Indonesia (Indonesian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Italiano (Italian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-it_IT" data-locale="it_IT" role="menuitem" lang="it_IT">
Italiano (Italian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="עברית (Hebrew)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-iw_IL" data-locale="iw_IL" role="menuitem" lang="iw_IL">
עברית (Hebrew)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="日本語 (Japanese)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ja_JP" data-locale="ja_JP" role="menuitem" lang="ja_JP">
日本語 (Japanese)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="한국어 (Korean)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ko_KR" data-locale="ko_KR" role="menuitem" lang="ko_KR">
한국어 (Korean)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="मराठी (Marathi)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-mr_IN" data-locale="mr_IN" role="menuitem" lang="mr_IN">
मराठी (Marathi)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Bahasa Malaysia (Malay)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ms_MY" data-locale="ms_MY" role="menuitem" lang="ms_MY">
Bahasa Malaysia (Malay)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Nederlands (Dutch)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-nl_NL" data-locale="nl_NL" role="menuitem" lang="nl_NL">
Nederlands (Dutch)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Norsk (Norwegian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-no_NO" data-locale="no_NO" role="menuitem" lang="no_NO">
Norsk (Norwegian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="ਪੰਜਾਬੀ (Punjabi)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-pa_IN" data-locale="pa_IN" role="menuitem" lang="pa_IN">
ਪੰਜਾਬੀ (Punjabi)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Polski (Polish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-pl_PL" data-locale="pl_PL" role="menuitem" lang="pl_PL">
Polski (Polish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Português (Portuguese)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-pt_BR" data-locale="pt_BR" role="menuitem" lang="pt_BR">
Português (Portuguese)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Română (Romanian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ro_RO" data-locale="ro_RO" role="menuitem" lang="ro_RO">
Română (Romanian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Русский (Russian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ru_RU" data-locale="ru_RU" role="menuitem" lang="ru_RU">
Русский (Russian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Svenska (Swedish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-sv_SE" data-locale="sv_SE" role="menuitem" lang="sv_SE">
Svenska (Swedish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="తెలుగు (Telugu)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-te_IN" data-locale="te_IN" role="menuitem" lang="te_IN">
తెలుగు (Telugu)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="ภาษาไทย (Thai)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-th_TH" data-locale="th_TH" role="menuitem" lang="th_TH">
ภาษาไทย (Thai)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Tagalog (Tagalog)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-tl_PH" data-locale="tl_PH" role="menuitem" lang="tl_PH">
Tagalog (Tagalog)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Türkçe (Turkish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-tr_TR" data-locale="tr_TR" role="menuitem" lang="tr_TR">
Türkçe (Turkish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Українська (Ukrainian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-uk_UA" data-locale="uk_UA" role="menuitem" lang="uk_UA">
Українська (Ukrainian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Tiếng Việt (Vietnamese)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-vi_VN" data-locale="vi_VN" role="menuitem" lang="vi_VN">
Tiếng Việt (Vietnamese)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="简体中文 (Chinese (Simplified))" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-zh_CN" data-locale="zh_CN" role="menuitem" lang="zh_CN">
简体中文 (Chinese (Simplified))
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="正體中文 (Chinese (Traditional))" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-zh_TW" data-locale="zh_TW" role="menuitem" lang="zh_TW">
正體中文 (Chinese (Traditional))
</button>
</li>
<!---->
</ul>
<button class="language-selector__button select-none relative pr-2 font-sans text-xs font-bold text-color-text-low-emphasis hover:text-color-link-hover hover:cursor-pointer focus:text-color-link-focus focus:outline-dotted focus:outline-1" aria-expanded="false" data-tracking-control-name="footer-lang-dropdown_trigger">
<span class="language-selector__label-text mr-0.5 break-words">
Language
</span>
<icon class="language-selector__label-chevron w-2 h-2 absolute top-0 right-0" data-delayed-url="https://static.licdn.com/aero-v1/sc/h/cyolgscd0imw2ldqppkrb84vo"></icon>
</button>
</div>
</li>
</ul>
<!----> </footer>
<script src="https://static.licdn.com/aero-v1/sc/h/e69gtg8c61dzqklxridy0htl9" async></script>
<!---->
<!----><!---->
</body>
</html>