Files
nexus/sreweekly/articles/122/03-making-linkedin-s-organic-feed-handle-peak-traffic.html
2026-09-12 17:23:01 +08:00

1616 lines
104 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en">
<head>
<meta name="pageKey" content="d_lithograph_content">
<!----><!----><!----> <meta name="locale" content="en_US">
<!----> <meta id="config" data-app-id="lithograph" data-app-version="0.0.2138" data-call-tree-id="AAZbLKvqDhME0NJChoLw4Q==" data-multiproduct-name="lithograph-publish-frontend" data-service-name="lithograph-publish-frontend" data-browser-id="803e1c31-0ce7-4eb5-8d47-3d38bde769b9" data-is-bot="false" data-page-instance="urn:li:page:d_lithograph_content;TE42e4NGS7Ca+hJZHzoTPw==" data-disable-jsbeacon-pagekey-suffix="false" data-dna-member-lix-treatment="control" data-human-member-lix-treatment="control" data-dfp-member-lix-treatment="control" data-sync-apfc-headers-lix-treatment="control" data-sync-apfc-cb-lix-treatment="control" data-recaptcha-v3-integration-lix-value="control" data-network-interceptor-lix-value="control" data-is-epd-audit-event-enabled="false" data-is-feed-sponsored-tracking-kill-switch-enabled="false" data-sequence-auto-redirect-before-request-enabled="true">
<link rel="canonical" href="https://www.linkedin.com/blog/engineering/optimization/making-linkedin-s-organic-feed-handle-peak-traffic">
<!----><!---->
<!---->
<!---->
<!---->
<!---->
<link rel="icon" href="https://static.licdn.com/aero-v1/sc/h/al2o9zrvru7aqj8e1x2rzsrca">
<script>
function getDfd() {let yFn,nFn;const p=new Promise(function(y, n){yFn=y;nFn=n;});p.resolve=yFn;p.reject=nFn;return p;}
window.lazyloader = getDfd();
window.tracking = getDfd();
window.impressionTracking = getDfd();
window.ingraphTracking = getDfd();
window.appDetection = getDfd();
window.pemTracking = getDfd();
window.appRedirectCompleted = getDfd();
</script>
<!---->
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Making LinkedIn's Organic Feed Handle Peak Traffic</title>
<!---->
<meta property="og:type" content="website">
<meta property="og:title" content="Making LinkedIn's Organic Feed Handle Peak Traffic">
<!---->
<meta property="og:url" content="https://www.linkedin.com/blog/engineering/optimization/making-linkedin-s-organic-feed-handle-peak-traffic">
<meta property="og:image" content="https://media.licdn.com/dms/image/v2/D4D08AQHl5XYNfTmIOQ/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700687683709?e=2147483647&amp;v=beta&amp;t=RIUzRkJ7jxPZpTq4WRCtGvj0KexD04CG0HJSQUEeqbI">
<meta name="twitter:title" content="Making LinkedIn's Organic Feed Handle Peak Traffic">
<meta name="twitter:site" content="@linkedin">
<!---->
<!---->
<meta name="robots" content="noarchive">
<meta name="linkedin:pageTag" content="/blog/engineering/optimization/making-linkedin-s-organic-feed-handle-peak-traffic">
<meta name="litmsProfileName" content="lithograph">
<!----> <meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="asset-url" id="artdeco/static/images/icons.svg" content="https://static.licdn.com/aero-v1/sc/h/6sz1e821vgereyxaakx87tt4z ">
<link rel="stylesheet" href="https://static.licdn.com/aero-v1/sc/h/lbz6744ddd0wsc02ey2a3dhx">
<!---->
<!----> </head>
<body dir="ltr">
<!---->
<a href="#lithograph-app" class="skip-link btn-md btn-primary absolute z-11 -top-[100vh] focus:top-0">
Skip to main content
</a>
<!---->
<div class="page-header__wrapper page-header__wrapper--margin" data-custom-nav-wrapper="true">
<div class="page-header__container">
<header id="page-header" class="page-header page-header--hide-dropshadow">
<div class="page-header__content-container">
<div class="header__logo">
<a href="https://www.linkedin.com/blog/engineering" class="custom-nav__logo-link">
<img class="header__linkedin-logo header__full-logo" src="https://static.licdn.com/aero-v1/sc/h/6zm111mce7vohqze950ilreo4" alt="LinkedIn Logo">
<img class="header__linkedin-logo header__in-bug" src="https://static.licdn.com/aero-v1/sc/h/5g0hjlcng3j5pgn50n2et1ca2" alt="LinkedIn Logo">
<span class="t-17 t-black t-bold default-header-title">
Engineering Blog
</span>
</a>
</div>
<!---->
<!---->
<button class="stripped header__expand-nav-button" data-mobile-header-navigation-button data-header-expand-navigation-button aria-expanded="false">
<li-icon class="header__expand-nav-icon" a11y-text="Open navigation" type="hamburger-icon"></li-icon>
</button>
<button class="stripped header__expand-nav-button hidden" data-mobile-header-navigation-button data-header-collapse-navigation-button aria-expanded="false">
<li-icon class="header__expand-nav-icon" a11y-text="Close navigation" type="cancel-icon"></li-icon>
</button>
</div>
</header>
</div>
<nav class="header-nav fixed" aria-label="main" data-header-navigation>
<ul class="header-nav__list">
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/data" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Data
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Data" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Graph System</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/economic-graph" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Economic Graph</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/skills-graph" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Skills Graph</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/knowledge" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Knowledge Graph</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/data-management" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Management</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/data-streaming-processing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Streams/Processing</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/data-science" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Science</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/research" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Research</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/ab-testing-experimentation" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">A/B Testing/Experimentation</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/artificial-intelligence" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
AI
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for AI" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/machine-learning" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Machine Learning</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/natural-language-processing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Natural Language Processing</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/generative-ai" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Generative AI</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/data-modeling" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Modeling</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/responsible-ai" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Responsible AI</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/trust-and-safety" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Trust & Safety
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Trust &amp;amp; Safety" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/fairness" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Fairness</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/verification" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Verification </a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/security" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Security</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/product-design" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Product Design
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Product Design" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/marketing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Marketing </a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/sales" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Sales</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/learning" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Learning</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/hiring" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Hiring</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Profile</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/messaging-notifications" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Messaging/Notifications</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/feed" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Feed</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Profile</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/groups" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Groups</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/accessibility" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Accessibility</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/member-customer-experience" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Member/Customer Experience</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/search" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Search</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/recommendations" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Recommendations</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/skills" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Skills</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/infrastructure" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Infrastructure
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Infrastructure" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/code" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Code</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/scalability" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Scalability</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/distributed-systems" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Distributed Systems</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Graph Systems</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/architecture" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Architecture</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/analytics" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Analytics</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/automation" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Automation</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/platform-platformization" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Platforms/Platformization</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/cloud-computing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Cloud Computing</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/open-source" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Open Source</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/additional-innovations" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Additional Innovations</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/talent" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Talent
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Talent" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/culture" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Culture</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/developer-experience-productivity" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Developer Experience/Productivity</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/leadership" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Leadership</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
</ul>
</nav>
<nav class="custom-header__nav--mobile invisible" data-mobile-navigation="true" aria-label="main">
<ul class="custom-header__list--mobile">
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/data" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Data
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Data" aria-expanded="false" data-label-expand="Expand to show more links for Data" data-label-collapse="Collapse to show less links for Data" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-sans t-black t-normal header-nav__link" target="_self">Graph System</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/economic-graph" class="t-sans t-black t-normal header-nav__link" target="_self">Economic Graph</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/skills-graph" class="t-sans t-black t-normal header-nav__link" target="_self">Skills Graph</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/knowledge" class="t-sans t-black t-normal header-nav__link" target="_self">Knowledge Graph</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/data-management" class="t-sans t-black t-normal header-nav__link" target="_self">Data Management</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/data-streaming-processing" class="t-sans t-black t-normal header-nav__link" target="_self">Data Streams/Processing</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/data-science" class="t-sans t-black t-normal header-nav__link" target="_self">Data Science</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/research" class="t-sans t-black t-normal header-nav__link" target="_self">Research</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/ab-testing-experimentation" class="t-sans t-black t-normal header-nav__link" target="_self">A/B Testing/Experimentation</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/artificial-intelligence" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
AI
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for AI" aria-expanded="false" data-label-expand="Expand to show more links for AI" data-label-collapse="Collapse to show less links for AI" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/machine-learning" class="t-sans t-black t-normal header-nav__link" target="_self">Machine Learning</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/natural-language-processing" class="t-sans t-black t-normal header-nav__link" target="_self">Natural Language Processing</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/generative-ai" class="t-sans t-black t-normal header-nav__link" target="_self">Generative AI</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/data-modeling" class="t-sans t-black t-normal header-nav__link" target="_self">Data Modeling</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/responsible-ai" class="t-sans t-black t-normal header-nav__link" target="_self">Responsible AI</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/trust-and-safety" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Trust &amp; Safety
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Trust &amp;amp; Safety" aria-expanded="false" data-label-expand="Expand to show more links for Trust &amp;amp; Safety" data-label-collapse="Collapse to show less links for Trust &amp;amp; Safety" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/fairness" class="t-sans t-black t-normal header-nav__link" target="_self">Fairness</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/verification" class="t-sans t-black t-normal header-nav__link" target="_self">Verification </a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/security" class="t-sans t-black t-normal header-nav__link" target="_self">Security</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/product-design" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Product Design
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Product Design" aria-expanded="false" data-label-expand="Expand to show more links for Product Design" data-label-collapse="Collapse to show less links for Product Design" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/marketing" class="t-sans t-black t-normal header-nav__link" target="_self">Marketing </a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/sales" class="t-sans t-black t-normal header-nav__link" target="_self">Sales</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/learning" class="t-sans t-black t-normal header-nav__link" target="_self">Learning</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/hiring" class="t-sans t-black t-normal header-nav__link" target="_self">Hiring</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-sans t-black t-normal header-nav__link" target="_self">Profile</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/messaging-notifications" class="t-sans t-black t-normal header-nav__link" target="_self">Messaging/Notifications</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/feed" class="t-sans t-black t-normal header-nav__link" target="_self">Feed</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-sans t-black t-normal header-nav__link" target="_self">Profile</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/groups" class="t-sans t-black t-normal header-nav__link" target="_self">Groups</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/accessibility" class="t-sans t-black t-normal header-nav__link" target="_self">Accessibility</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/member-customer-experience" class="t-sans t-black t-normal header-nav__link" target="_self">Member/Customer Experience</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/search" class="t-sans t-black t-normal header-nav__link" target="_self">Search</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/recommendations" class="t-sans t-black t-normal header-nav__link" target="_self">Recommendations</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/skills" class="t-sans t-black t-normal header-nav__link" target="_self">Skills</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/infrastructure" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Infrastructure
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Infrastructure" aria-expanded="false" data-label-expand="Expand to show more links for Infrastructure" data-label-collapse="Collapse to show less links for Infrastructure" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/code" class="t-sans t-black t-normal header-nav__link" target="_self">Code</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/scalability" class="t-sans t-black t-normal header-nav__link" target="_self">Scalability</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/distributed-systems" class="t-sans t-black t-normal header-nav__link" target="_self">Distributed Systems</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-sans t-black t-normal header-nav__link" target="_self">Graph Systems</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/architecture" class="t-sans t-black t-normal header-nav__link" target="_self">Architecture</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/analytics" class="t-sans t-black t-normal header-nav__link" target="_self">Analytics</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/automation" class="t-sans t-black t-normal header-nav__link" target="_self">Automation</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/platform-platformization" class="t-sans t-black t-normal header-nav__link" target="_self">Platforms/Platformization</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/cloud-computing" class="t-sans t-black t-normal header-nav__link" target="_self">Cloud Computing</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/open-source" class="t-sans t-black t-normal header-nav__link" target="_self">Open Source</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/additional-innovations" class="t-sans t-black t-normal header-nav__link" target="_self">Additional Innovations</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/talent" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Talent
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Talent" aria-expanded="false" data-label-expand="Expand to show more links for Talent" data-label-collapse="Collapse to show less links for Talent" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/culture" class="t-sans t-black t-normal header-nav__link" target="_self">Culture</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/developer-experience-productivity" class="t-sans t-black t-normal header-nav__link" target="_self">Developer Experience/Productivity</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/leadership" class="t-sans t-black t-normal header-nav__link" target="_self">Leadership</a>
</li>
</ul>
</li>
</ul>
<div class="custom-header__button-share-container--mobile">
<!---->
</div>
</nav>
</div>
<main role="main">
<div id="lithograph-app">
<section id="component-container" data-attach-collapse-behavior="true">
<div id class="component component-articleHeadline" data-component-type="articleHeadline">
<div class="article-headline-container">
<p class="article-headline__topic">
<a class="t-14" href="https://www.linkedin.com/blog/engineering/optimization">Optimization</a>
</p>
<h1 class="article-headline t-40" data-article-title="Making LinkedIn's Organic Feed Handle Peak Traffic">Making LinkedIn's Organic Feed Handle Peak Traffic</h1>
<div class="article-meta t-14 article-meta__hairline">
<div class="article-meta__byline-container">
<div class="author-profile__author-container">
<img class="author-profile__author-image" srcset="https://media.licdn.com/dms/image/v2/D5603AQH_ob9ef2FWcA/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1646693676653?e=2147483647&amp;v=beta&amp;t=WyOKys1CuqsfznYMBUt5YvsPWtHxia1hY496dGrS8qM 100w,https://media.licdn.com/dms/image/v2/D5603AQH_ob9ef2FWcA/profile-displayphoto-shrink_200_200/profile-displayphoto-shrink_200_200/0/1646693676653?e=2147483647&amp;v=beta&amp;t=C0swFwA20aBPGNlWQDBLt5XASSvWmG8fcz_O7i28jbQ 200w,https://media.licdn.com/dms/image/v2/D5603AQH_ob9ef2FWcA/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1646693676653?e=2147483647&amp;v=beta&amp;t=P6o5DQHxeKthBf0pyHl-J_zglLL38bTiWc-ihLN0WDg 400w,https://media.licdn.com/dms/image/v2/D5603AQH_ob9ef2FWcA/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1646693676653?e=2147483647&amp;v=beta&amp;t=6ncES6hDkgjNEVkXU00NqNIv8styBmhbFhLWlBS2OYE 800w," sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" src="https://media.licdn.com/dms/image/v2/D5603AQH_ob9ef2FWcA/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1646693676653?e=2147483647&amp;v=beta&amp;t=6ncES6hDkgjNEVkXU00NqNIv8styBmhbFhLWlBS2OYE" alt="Val Markovic">
<div class="author-profile__author-text-container">
<a href="https://www.linkedin.com/in/valmarkovic" target="_blank" class="t-14 t-bold"><span class="screen-reader">Authored by</span>Val Markovic</a>
<p class="t-14">Engineering Lead, ex-Meta, ex-Google</p>
<p class="t-14 t-black--light" data-published-date="2018-5-17">
May 17, 2018
</p>
</div>
</div>
<!---->
</div>
<div class="social-share">
<ul class="social-share__list share-list" title="Share via" data-share-list>
<li class="share-list__item share-item">
<a href="#" class="share-item__link medium-round-muted-tertiary-button" role="button" data-tracking-control-name="_linkedin_share" data-share-type="linkedin" aria-label="Share on LinkedIn, New window will open">
<li-icon class="share-item__icon" type="linkedin-icon"></li-icon>
</a>
</li>
<li class="share-list__item share-item">
<a href="#" class="share-item__link medium-round-muted-tertiary-button" role="button" data-tracking-control-name="_facebook_share" data-share-type="facebook" aria-label="Share on Facebook, New window will open">
<li-icon class="share-item__icon" type="facebook-icon"></li-icon>
</a>
</li>
<li class="share-list__item share-item">
<a href="#" class="share-item__link medium-round-muted-tertiary-button" role="button" data-tracking-control-name="_twitter_share" data-share-type="twitter" aria-label="Share on X, New window will open">
<svg class="share-item__icon share-item__icon--twitter" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" fill="currentColor" focusable="false" aria-hidden="true" role="none">
<path d="M13.9 10.47L21.35 2h-1.76l-6.46 7.35L7.96 2H2l7.81 11.12L2 22h1.76l6.83-7.77L16.04 22H22l-8.1-11.53zm-2.42 2.75l-.79-1.11L4.4 3.3h2.71l5.08 7.11.79 1.11 6.6 9.24h-2.71l-5.39-7.54z"></path>
</svg>
</a>
</li>
</ul>
</div>
</div>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>LinkedIn has recently experienced issues with handling peak load in <a href="https://engineering.linkedin.com/blog/2016/03/followfeed--linkedin-s-feed-made-faster-and-smarter" target="_blank" data-tracking-control-name="_anchor_link">FollowFeed</a>, our custom timeline service that powers the organic updates members see in their activity feed. Examples of organic updates are text/article/image shares from members you are connected to, updates about comments or likes from your connections etc. In other words, the interactions from people in your social network at LinkedIn. Organic updates comprise the vast majority of all updates seen in the feed.</p>
<p>In this post, we’ll be talking about issues we’ve seen with handling load tests of site traffic in FollowFeed.</p>
<h2>How this all got started </h2>
<p>On Monday, Feb. 5, 2018, there was a scheduled load test of one of LinkedIn’s data centers. We regularly perform such load tests to ensure that our systems can handle the traffic growth we foresee coming in the next few months.</p>
<p>During this load test, FollowFeed had issues serving the site traffic; our main metrics for “quality of responses” took a nosedive.</p>
<p>A rough understanding of FollowFeed’s architecture is necessary. When you load the LinkedIn homepage, services “above” FollowFeed receive your request and send another request to &nbsp;followfeed-query, our broker layer. That service then creates multiple parallel requests to followfeed-storage (which stores member activity timelines). Followfeed-query then collects all the responses, merges them together, and returns the top results back.</p>
<p>We have two metrics we track for QoS (Quality of Service): </p>
<p></p>
<ul>
<li><b>Full QoS.</b> The percentage of queries that received data from all the member timelines that were queried comprises Full QoS. So, if 100 timelines were requested, and those timelines are on 10 followfeed-storage machines, and all 10 fetches (including retries) were successful, then the Full QoS for this request was 1. If 9 out of 10 fetches succeeded, then Full QoS is 0. The top graph below shows the ratio of queries with Full QoS versus total queries received.</li>
<li><b>Partial QoS.</b> For the above example, Partial QoS is 1 for both 10/10 successful fetches and 9/10, all the way to 1/10. As long as we received some data back, Partial QoS is happy.</li>
</ul>
<p></p>
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQFB87vqbgDm4Q/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700687685508?e=2147483647&amp;v=beta&amp;t=hIfdTf4CqLY_z7lSEG0vmPDCM1OhiP0qoa4GtKeYia4" alt="organic-feed-peak-traffic-2">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>This distinction between Full and Partial QoS is important: if Full QoS takes a dip, the relevance of the returned results might suffer, but the service is still returning responses to members. But if Partial QoS takes a dip, that means we returned no results, which means the members see only non-organic updates in their feeds.</p>
<p>The graphs above are pretty bad; 10% of all calls got no results and a full 40% got a degraded feed experience.</p>
<p>We identified two high-level problems during this load test:</p>
<ol>
<li><p><b>We didn’t appear to have the capacity to handle the planned increase in load.</b></p> </li>
<li><p><b>We weren’t gracefully handling the extra load.</b> If we can normally take X amount of traffic without issues and we start seeing X + 5% of traffic, we want only the 5% of excess traffic to see errors, not more than that.</p> </li>
</ol>
<p>Both of these are problems and both needed to be addressed. But first, a root cause analysis.</p>
<h2>Initial investigation</h2>
<p>We noticed that our JVM OldGen Used Memory had spiked up on some machines the previous Friday, after which the JVM started doing back-to-back GC collections, which didn’t reduce the used memory at all.</p>
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQGj1BJPM5Y_Iw/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700687684809?e=2147483647&amp;v=beta&amp;t=LtTBzDoJDSKaLvPerM6ALkzsawezZQu2x77il9BCxn4" alt="organic-feed-peak-traffic-3">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>This was an unhealthy state, but because of redundancy in our service design, other machines hosting the same partitions managed to prevent this issue from affecting site traffic until Monday, when we did a scheduled load test. At that point, the problem blew up.</p>
<p>We looked at the history of these graphs and saw similar spikes in the past, but they would not last for two full days (or anywhere near that). </p>
<h2>Heap dump analysis</h2>
<p>The obvious next step was analyzing heap dumps to identify the source of this increase in RAM usage. We tried using jhat, but it was failing to open our heap dumps (which are sizeable, at 60+ GB). During an unrelated chat with <a href="https://www.linkedin.com/in/bpgoldsb/" target="_blank" data-tracking-control-name="_anchor_link">Ben Goldsbury</a>, one of our SREs, the topic of problems with analyzing our heap dump came about; Ben mentioned <a href="https://www.eclipse.org/mat/" target="_blank" data-tracking-control-name="_anchor_link">Eclipse MAT</a>. He had used it in the past with great success. He ended up helping us get up to speed with MAT by running MAT reports on the file, answering tons of our questions, etc.</p>
<p>We took it from there and quickly found the culprit:</p>
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQGHEfLOg3xDfw/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700687684133?e=2147483647&amp;v=beta&amp;t=j3opgqTmSFn-Mgn3Pk-ZN-778T1d4wNf80IaZRbUIEg" alt="organic-feed-peak-traffic-4">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>The cache objects are normal, but the Jetty ServerConnector using ~7.5 GB of RAM wasn’t.</p>
<p>An aside about how Eclipse MAT works: it can do many things and generate lots of reports, but one of the most useful tools MAT offers is the Dominator Tree. This tool sorts all objects by their retained heap. Note that this is different than an object’s shallow heap.</p>
<p>An object’s shallow heap is the size (in bytes) of just that object in memory, not counting the size of any objects that this object might be referencing. An object’s retained set (not heap) is the set of objects which would be removed by the GC if this object were removed (in other words, other objects exclusively reachable through this object). The object’s retained heap is the total shallow heap of all the objects in its retained set.</p>
<p>To illustrate this, imagine a Java List&lt;Integer&gt; with a million entries in it. The List object itself occupies only a handful of bytes, mostly counts, and, critically, a reference to an array of objects where the data is stored. It is that array that is the large object, not the List itself.</p>
<p>So the List has a tiny shallow heap, but a large retained heap.</p>
<p>In the above picture we can see that the ServerConnector is holding onto a very large tree of objects. Here’s what those objects are:</p>
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQFPS2XswRIgsg/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700687686052?e=2147483647&amp;v=beta&amp;t=GMaQUje_-3g_MtsBxVLsvc3mHFuB_HQGNk7qpkDvGVI" alt="organic-feed-peak-traffic-5">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>It’s 500,000 SelectChannelEndPoint objects in a ConcurrentHashMap used as a Set. The biggest contributor to the size of that object is an HttpConnection object which stores our full request bytes.</p>
<p>So Jetty appears to be holding on to 500K requests. This did look like a bug in Jetty, but the next step was to look at our custom Jetty configuration to verify if we were doing something silly to cause this behavior.</p>
<p>And silliness we did find, though not enough to explain what we see here. But before we talk about what we found, we need to talk about queues.</p>
<h2>Queues in systems at scale</h2>
<p>It’s important to understand how queues between producers and consumers behave at scale. We won’t go into much detail (there’s literally <a href="https://en.wikipedia.org/wiki/Queueing_theory" target="_blank" data-tracking-control-name="_anchor_link">an entire field of math</a> devoted to this topic), but we will cover the parts relevant to our discussion.</p>
<p>Imagine we have the following setup in a server application:</p>
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQG1R5n03RYhVA/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700687684326?e=2147483647&amp;v=beta&amp;t=qj3_HlYprHM7gBClUwpwFHCERCtUF8WlMws1gz-RCZc" alt="organic-feed-peak-traffic-6">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>The producer is adding items to the queue (from multiple threads) and the consumer is taking items from the queue (also from multiple threads).</p>
<p>Now assume that the producer is producing at a constant rate of 900 Queries Per Second (QPS). The capacity of the consumer is 1,000 QPS; that’s how much it can process. How many items are in the queue? Usually about a couple of items; one or two, maybe up to five. It depends on how threads get scheduled.</p>
<p>In such a setup, the maximum size of the queue (its configured capacity) doesn’t need to be big; the queue is acting as a hand-off point.</p>
<p>Now imagine that the producer starts producing at a faster rate than the capacity of the consumer. Say at 1,100 QPS, while the consumer can still only consume 1,000 QPS. How many items are in the queue? If the size of the queue was 10, then the number of items in it is 10. If it’s 1,000, then the number of items grows by 100 every second until it gets full. The queue very quickly becomes full, regardless of size.</p>
<p>If the queue is configured to reject items when full, the producer will receive feedback from the consumer that it can’t process items at the rate they are being produced. This feedback is called “backpressure.”</p>
<p>But in a production system, there is rarely just a single producer, queue, and consumer; there’s almost certainly a chain of these, where a consumer is also a producer for the next link in the chain. In these situations, we want the backpressure to propagate throughout the chain quickly.</p>
<p>Correct management of backpressure is critical to the health of systems at peak load. If little thought has been put into backpressure, then once the system hits its capacity limit, it won’t degrade gracefully even if theoretically it could have handled the peak load. In fact, it might just enter a death spiral.</p>
<p>The most common example would be a queue with an inappropriately large maximum size (or worse, an unbounded queue). The queue can start taking up more and more RAM, triggering GC more frequently (in a system with a GC) and thus reducing the throughput (and thus capacity of the system). The queue then becomes even fuller, the GC triggers even more frequently, and down the spiral of death we go.</p>
<p>The problem isn’t even limited to languages with a GC. Similar issues arise with threadpools with unbounded (or very high) max active thread limits. If thousands of threads are all trying to make progress (and few, if any, are blocked on IO), the capacity of the system plummets due to the cost of <a href="https://en.wikipedia.org/wiki/Context_switch" target="_blank" data-tracking-control-name="_anchor_link">context switching</a> (and the side effects of that, like CPU cache thrashing, etc.). Naturally, this further increases the processing backlog.</p>
<p>This doesn’t even mention possible failure to handle the backpressure feedback on the producer side, which brings a separate set of problems.</p>
<h2>Auditing our Jetty configuration</h2>
<p>Cognizant of the above, and with a hunch that some Jetty-related queues were too big, we started looking into every single Jetty config value we were setting. Some of those configs did stand out as wrong.</p>
<p>First, our Jetty request queue max size was set to 5,000. Experience and intuition told us this was likely wrong. Looking around, <a href="https://wiki.eclipse.org/Jetty/Howto/High_Load#Jetty_Tuning" target="_blank" data-tracking-control-name="_anchor_link">Jetty documentation recommends a value between 50 and 500.</a> The documentation makes an important point: make sure the queue is large enough to handle a traffic spike. Unlike our simple example in the backpressure section, producer load in reality can often be uneven.</p>
<p>Now let’s imagine a worst-case scenario: suddenly, 500 queries arrive in the queue at the exact same moment. We know that the client (followfeed-query) has a total request timeout of 400 ms. We also know that our peak capacity for a followfeed-storage machine at this time is about 1,200 QPS. So in that 400 ms budget, we’ll be able to respond to 1,200 queries/s * 0.4 s = 480 queries. The other 20 queries we will still process and respond to, but the client won’t care (it has already moved on because of the timeout).</p>
<p>So that’s roughly the number of queries we can be usefully buffering up. With a request queue size of 5,000, it would take us ~4.17 seconds to respond to all of them (assuming the machine isn’t melting!), and ~4,520 responses would be wasted work.</p>
<p>Thus, we reduced the request queue size to 500. This is still actually too high because followfeed-query won’t sit idly by while 400 ms pass; it will start sending an identical request ~150 ms after the first one (to a separate followfeed-storage replica) if the request hasn’t arrived by that point. Those other replicas are likely to have responded to a large fraction of those 480 queries before the machine with the full queue does. We’ll be further tuning the request queue size in the near future, but a 10x reduction should be immediately helpful.</p>
<p>The second queue that appeared to be too big was Jetty’s acceptQueueSize parameter. We had it overridden to 2,000. Here’s what the Jetty docs have to say about it:</p>
<p> </p>
<p>“The size of the pending connection backlog. The exact interpretation is JVM and operating system specific and you can ignore it. Higher values allow more connections to wait pending an acceptor thread. Because the exact interpretation is deployment dependent, it is best to keep this value as the default unless there is a specific connection issue for a specific OS that you need to address.”</p>
<p>You’ll notice that this has an admonition to not touch the default without a very good reason. This piled yet more suspicion on that 2,000 size, because the default is 0. Now 0 clearly looked like a sentinel value, so the question became, “What is the default?” Digging through code, we can see that <a href="https://github.com/eclipse/jetty.project/blob/jetty-9.3.15.v20161220/jetty-server/src/main/java/org/eclipse/jetty/server/ServerConnector.java#L317" target="_blank" data-tracking-control-name="_anchor_link">Jetty passes the provided acceptQueueSize value</a> to ServerSocket.bind() as the value of a backlog parameter. ServerSocket is a standard class in the Java Runtime Environment (JRE), so we can look into its source to see what it does when the provided value is 0 (the JavaDocs state the value is “implementation dependent”).</p>
<p>The source code shows that the <a href="http://grepcode.com/file/repository.grepcode.com/java/root/jdk/openjdk/8u40-b25/java/net/ServerSocket.java#369" target="_blank" data-tracking-control-name="_anchor_link">actual default value is 50</a> and is provided to the <a href="http://man7.org/linux/man-pages/man2/listen.2.html" target="_blank" data-tracking-control-name="_anchor_link">listen(2)</a> syscall. (For those that want to learn more about this TCP connection queue, <a href="http://veithen.github.io/2014/01/01/how-tcp-backlog-works-in-linux.html" target="_blank" data-tracking-control-name="_anchor_link">here’s a great article.</a>) For our purposes, knowing that the JRE default is 50 (with JRE defaults usually being sensible), and that we were using 2,000 for no obvious reason, was enough to experiment with using the default.</p>
<p>So we packaged up this config change along with the request queue size change and benchmarked them with a copy of production traffic on an isolated machine. This experiment was a success.</p>
<p>Deploying this change to production helped us handle excess load gracefully; we now reject requests before they eat up RAM, and the smaller request queue means it takes us less time to recover. The smaller TCP connection accept queue means we start rejecting connections when under load, which works well because those requests will be retried on other replicas.</p>
<p>But these changes did something else, too: they increased our capacity. We now see far fewer retries attempted from followfeed-query for the same amount of site traffic. The request queue size reduction lowered GC pressure when the machines are under heavy load. The TCP accept queue size reduction provided backpressure to followfeed-query machines that are trying to reconnect to the followfeed-storage machine that was returning errors (and closing connections on the client) when under load. </p>
<h2>Over-exuberant retry strategy </h2>
<p>As is common with wide-ranging investigations that look for possible causes of production issues, we’ve found plenty of problems that needed addressing besides those listed above.</p>
<p>One of the biggest issues was with our retry strategy in followfeed-query (the broker service in front of followfeed-storage). When a followfeed-query machine sends a request to a followfeed-storage machine hosting a particular partition, it waits for about 150 ms before concluding that something is very wrong (median latency for followfeed-storage is ~8 ms and 99th percentile latency is ~100 ms). At that point, it makes the same call to a separate replica of followfeed-storage hosting the same partition (while still waiting for the older call to complete, just in case it comes back). There’s a third fetch attempt after the second one if that too is still pending after an additional 150 ms.</p>
<p>The reason why this is needed is because of GC pauses; if a followfeed-storage machine hits a long GC pause, the calls that it is working on need to be retried for the sake of reducing user-visible latency.</p>
<p>Now imagine what will happen when a followfeed-storage machine spikes up in latency for a temporary reason (GC pause, loading data from disk into caches, etc.). A part of the call volume it was getting will be redirected (temporarily) to other machines hosting the same partitions. Since we have 4 replicas hosting an identical set of partitions, 3 machines now start taking more traffic. If those machines were near peak operating load, they start having issues as well, causing more latency increases, which causes more retries from followfeed-query, which causes yet more latency increases…this becomes a death spiral.</p>
<p>This is fairly obvious, so naturally, followfeed-query had a system in place to prevent this. There were (well-tuned!) rate-limiters in place for the second and third fetch attempts to stop such a situation for happening. Unfortunately, we realized that the rate-limiters were global, not per-host. In other words, the total number of retries across all the followfeed-storage machines was rate-limited in followfeed-query (and the max retry call volume was capped at ~2.5% of peak non-retry volume). Unfortunately, this meant that if a single set of machines hosting a set of partitions started having issues, there was more than enough “retry call budget” to drown those machines in 2x more requests, effectively DDOS-ing the system.</p>
<p>So, we made the rate-limiters per-host.</p>
<p>After thorough testing of this change, we deployed it everywhere. One of our data centers didn’t have the Jetty configuration changes deployed by that point and one did, so we could make direct comparisons between them (at comparable levels of traffic), since they used the same hardware. For the data center with only the per-host rate-limiter changes, we could see a sizable number of retries getting blocked by the new rate-limiters (and the death spiral was being prevented, as expected). For the data center with both the rate-limiters and the Jetty changes, almost no retries were being blocked. Thus we know the Jetty changes increased capacity. </p>
<h2>Other ideas we tried</h2>
<p>For context, the original configuration values were set several years ago. Knowledge of why those values were chosen is lost to time. They probably seemed like good choices back then, but the system had never really seen load that exceeded capacity until now; thus, there wasn’t a pressing need to tune those parameters.</p>
<p>We found several other parameters that needed tuning and other places where we could add graceful degradation as a result of our investigation. Some of these issues were hurting peak load behavior, and some these we tuned out of caution.</p>
<ul>
<li><p><b>TCP connection limits in followfeed-query were wrong.</b> We had min/max TCP connections per followfeed-storage host set to 10/100. We only really need 0.4 TCP connections per (followfeed-query, followfeed-storage) machine pair, even for peak load. Reducing the min/max to 1/4 completely removed TCP connection spikes during load tests. The spikes had been 7x normal TCP established connections on each followfeed-storage machine; after the change, the TCP connection count remains flat during peak load. </p> </li>
<li><p><b>HTTP request timeouts at the library level were not tuned to match our application-level timeouts.</b> App-level timeouts were 400 ms, while HTTP request timeouts were 3.5 seconds. This means that after a long GC pause, followfeed-storage could still try to process some requests that the client has long since stopped caring about.</p> </li>
<li><p><b>We had no request rate-limiting inside followfeed-storage.</b> We know the QPS at which we start failing, so putting a rate-limiter on the server end is useful for graceful degradation and providing backpressure by returning <a href="https://httpstatuses.com/429" target="_blank" data-tracking-control-name="_anchor_link">429 Too Many Requests</a> responses.</p> </li>
<li><p><b>Out-of-date GC configuration.</b> We haven’t needed to tune our GC configuration for a long time, and the configuration we had in production wasn’t a great match for today’s workloads. Tuning these parameters helped address GC pauses.</p> </li>
</ul>
<p>Another critical issue we identified was incorrect metric computation. We discovered that the QoS metrics we showed at the start of the blog post were overcounting errors by 300%; several years ago when they were created, the data produced was correct, but because of architectural changes made since then, the algorithms computing the metrics were out of date.</p>
<p>We’ve rebuilt our QoS metrics computation from scratch so that we account for these changes, add additional accuracy, and include new QoS metrics for an even clearer picture of production behavior. &nbsp;&nbsp;</p>
<h2>Future work</h2>
<p>We’ve also identified future work that will improve our capacity, graceful degradation, and ability to debug issues like this in the future.</p>
<ul>
<li><p><b>Our request orchestration code in followfeed-query has accumulated technical debt</b> over time that makes it very hard to change and reason about. We’ve also learned a lot about the behavior of our system since it was launched several years ago; that knowledge can be put to good use in request orchestration. We expect to be able to improve the reliability of the system here.</p> </li>
<li><p><b>The way we distribute data partitions across followfeed-storage machines isn’t ideal for peak load.</b> Today, followfeed-storage machine replicas host the same 20 data partitions (with 4 replicas). Thus if a followfeed-storage machine is having issues, the request volume gets distributed across 3 other machines. If we assigned partitions heterogeneously (where every machine would effectively have a random set of 20 partitions) while still assuring at least 4 machines host each partition, then when a machine starts having issues, the call volume would be distributed to ~80 other machines instead of 3. This would massively reduce the extra load per-machine. <br> <br> Here’s why it would be 80 machines: one machine is hosting 20 partitions. There are 4 machines hosting each partition. With effectively random partition allocation, the total number of machines involved that host at least one of the 20 affected partitions is 4 * 20 = 80 machines. (Technically 79, because one machine is having problems.) Note: This calculation assumes that no machine is hosting two or more partitions from the affected set.</p> </li>
<li><p><b>Missing service-level continuous headroom testing.</b> Our issues with reaching capacity limits were only discovered during site-wide load tests. Ideally, we’d be able to identify these limits well before that point. </p> </li>
</ul>
<h2>Conclusion</h2>
<p>We’re still not sure what was the real underlying cause of that blow-up in Jetty RAM usage; we know it’s related to the number of in-flight requests and connections, and working to reduce those did help. We have pulled in the Jetty experts at the company to look at our issue.</p>
<p>But that issue was only one of many problems we needed to address to increase capacity and improve our ability to gracefully degrade at peak load.</p>
<p>Meanwhile, the changes we’ve made have stabilized the system and we’ve successfully passed several load tests, doing great not only at our original data center traffic target but at a target 10% higher as well. We have plenty of follow-up work to improve capacity and pay down accumulated technical debt in our serving path, but that work has already begun.</p>
</div>
</div>
<div id class="component component-postList" data-component-type="postList">
<section id="postList0FocusPoint" class="component__content-container list-layout" data-post-list>
<section class="simple-topic-list">
<p class="t-14 t-bold">Topics:
<a class="simple-topic-list__topic" href="https://www.linkedin.com/blog/engineering/feed">
Feed
</a>
<a class="simple-topic-list__topic" href="https://www.linkedin.com/blog/engineering/optimization">
Optimization
</a>
</p>
</section>
<hr class="artdeco-divider">
<section class="list-layout__headline-container">
<p class="list-layout__headline t-32 t-sans">Related articles</p>
<!---->
</section>
<ul class="list-layout__posts">
<li class="post-list__item list-post">
<div class="list-post__content-container">
<p class="list-post__content-container__topic">
<a class="t-14 t-bold" href="https://www.linkedin.com/blog/engineering/feed">
Feed
</a>
</p>
<div class="list-post__content-container__title">
<a class="list-post__link t-20 t-black" href="https://www.linkedin.com/blog/engineering/feed/engineering-the-next-generation-of-linkedins-feed">
Engineering the next generation of LinkedIn’s Feed
</a>
</div>
<div class="list-post__content-container__byline">
<p class="list-post__content-container__author t-14 t-bold t-black">
Hristo Danchev
</p>
<p class="list-post__content-container__date t-14 t-black--light">
Mar 12, 2026
</p>
</div>
</div>
<div class="list-post__image-container">
<figure class="post__image-figure">
<img class="post__image" alt data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQHcUvj-cr_UxQ/croft-frontend-shrinkToFit600/B4DZzia1YNIwAY-/0/1773325212167?e=2147483647&amp;v=beta&amp;t=K6Z7BnjoDJOayE_sIqtfjytksB_HGf90JYBcx787D38" data-ghost-url="https://static.licdn.com/aero-v1/sc/h/8pouyb1ly746xtru0x7p2axm0" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 480px">
</figure>
</div>
</li>
<li class="post-list__item list-post">
<div class="list-post__content-container">
<p class="list-post__content-container__topic">
<a class="t-14 t-bold" href="https://www.linkedin.com/blog/engineering/feed">
Feed
</a>
</p>
<div class="list-post__content-container__title">
<a class="list-post__link t-20 t-black" href="https://www.linkedin.com/blog/engineering/feed/putting-members-first-testing-and-measuring-how-content-appears-in-your-feed">
Putting members first: testing and measuring how content appea...
</a>
</div>
<div class="list-post__content-container__byline">
<p class="list-post__content-container__author t-14 t-bold t-black">
Sakshi Jain
</p>
<p class="list-post__content-container__date t-14 t-black--light">
Nov 20, 2025
</p>
</div>
</div>
<div class="list-post__image-container">
<figure class="post__image-figure">
<img class="post__image" alt data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQH-LYSmqFmbKw/croft-frontend-shrinkToFit1024/B4DZqjCEDMKIAc-/0/1763671819105?e=2147483647&amp;v=beta&amp;t=hZvTJQ9rTq-SJ-ekLu0P4DRwYZo86qpdPdTIxPOl5yU" data-ghost-url="https://static.licdn.com/aero-v1/sc/h/8pouyb1ly746xtru0x7p2axm0" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 480px">
</figure>
</div>
</li>
<li class="post-list__item list-post">
<div class="list-post__content-container">
<p class="list-post__content-container__topic">
<a class="t-14 t-bold" href="https://www.linkedin.com/blog/engineering/infrastructure">
Infrastructure
</a>
</p>
<div class="list-post__content-container__title">
<a class="list-post__link t-20 t-black" href="https://www.linkedin.com/blog/engineering/infrastructure/fishdb-a-generic-retrieval-engine-for-scaling-linkedins-feed">
FishDB: a generic retrieval engine for scaling LinkedIn’s feed
</a>
</div>
<div class="list-post__content-container__byline">
<p class="list-post__content-container__author t-14 t-bold t-black">
Nisheedh Raveendran
</p>
<!----><!----><!----><!----><!---->
<p class="list-post__content-container__date t-14 t-black--light">
Nov 17, 2025
</p>
</div>
</div>
<div class="list-post__image-container">
<figure class="post__image-figure">
<img class="post__image" alt data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQHH99y6k0N4xg/croft-frontend-shrinkToFit767/B4DZqTd0V.JAAQ-/0/1763410659531?e=2147483647&amp;v=beta&amp;t=P4Q-snS_0ifRloLA0YRdxoHWtWbS9srcNRfNk43Fd9E" data-ghost-url="https://static.licdn.com/aero-v1/sc/h/8pouyb1ly746xtru0x7p2axm0" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 480px">
</figure>
</div>
</li>
</ul>
<!----> </section>
</div>
<!----> </section>
</div>
<script src="https://static.licdn.com/aero-v1/sc/h/8hfbuq1ftcvnnx4dd5067pi0t" async></script>
<script src="https://static.licdn.com/aero-v1/sc/h/bknx7jzs0ylu540sn4wuo6bue" async></script>
<script src="https://static.licdn.com/aero-v1/sc/h/egqq1vs0784qd3h3801e2w9lr" async></script>
</main>
<footer class="li-footer bg-transparent w-full ">
<ul class="li-footer__list flex flex-wrap flex-row items-start justify-start w-full h-auto min-h-[50px] my-[0px] mx-auto py-3 px-2 papabear:p-0">
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<span class="sr-only">LinkedIn</span>
<icon class="li-footer__copy-logo text-color-logo-brand-alt inline-block self-center h-[14px] w-[56px] mr-1" data-delayed-url="https://static.licdn.com/aero-v1/sc/h/5mebydpuuijm3uhv1q375inqh"></icon>
<span class="li-footer__copy-text flex items-center">&copy; 2026</span>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://about.linkedin.com?trk=content_footer-about" data-tracking-control-name="content_footer-about" data-tracking-will-navigate>
About
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/accessibility?trk=content_footer-accessibility" data-tracking-control-name="content_footer-accessibility" data-tracking-will-navigate>
Accessibility
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/user-agreement?trk=content_footer-user-agreement" data-tracking-control-name="content_footer-user-agreement" data-tracking-will-navigate>
User Agreement
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/privacy-policy?trk=content_footer-privacy-policy" data-tracking-control-name="content_footer-privacy-policy" data-tracking-will-navigate>
Privacy Policy
</a>
</li>
<!---->
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/cookie-policy?trk=content_footer-cookie-policy" data-tracking-control-name="content_footer-cookie-policy" data-tracking-will-navigate>
Cookie Policy
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/copyright-policy?trk=content_footer-copyright-policy" data-tracking-control-name="content_footer-copyright-policy" data-tracking-will-navigate>
Copyright Policy
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://brand.linkedin.com/policies?trk=content_footer-brand-policy" data-tracking-control-name="content_footer-brand-policy" data-tracking-will-navigate>
Brand Policy
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/psettings/guest-controls?trk=content_footer-guest-controls" data-tracking-control-name="content_footer-guest-controls" data-tracking-will-navigate>
Guest Controls
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/help/linkedin/answer/34593?trk=content_footer-community-guide" data-tracking-control-name="content_footer-community-guide" data-tracking-will-navigate>
Community Guidelines
</a>
</li>
<!---->
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<div class="collapsible-dropdown collapsible-dropdown--footer collapsible-dropdown--up flex items-center relative hyphens-auto language-selector z-2">
<!---->
<ul class="collapsible-dropdown__list hidden container-raised absolute w-auto overflow-y-auto flex-col items-stretch z-[9999] bottom-[100%] top-auto" role="menu" tabindex="-1">
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="العربية (Arabic)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ar_AE" data-locale="ar_AE" role="menuitem" lang="ar_AE">
العربية (Arabic)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="বাংলা (Bangla)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-bn_IN" data-locale="bn_IN" role="menuitem" lang="bn_IN">
বাংলা (Bangla)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Čeština (Czech)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-cs_CZ" data-locale="cs_CZ" role="menuitem" lang="cs_CZ">
Čeština (Czech)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Dansk (Danish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-da_DK" data-locale="da_DK" role="menuitem" lang="da_DK">
Dansk (Danish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Deutsch (German)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-de_DE" data-locale="de_DE" role="menuitem" lang="de_DE">
Deutsch (German)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Ελληνικά (Greek)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-el_GR" data-locale="el_GR" role="menuitem" lang="el_GR">
Ελληνικά (Greek)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="English (English) selected" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link--selected" data-tracking-control-name="language-selector-en_US" data-locale="en_US" role="menuitem" lang="en_US">
<strong>English (English)</strong>
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Español (Spanish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-es_ES" data-locale="es_ES" role="menuitem" lang="es_ES">
Español (Spanish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="فارسی (Persian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-fa_IR" data-locale="fa_IR" role="menuitem" lang="fa_IR">
فارسی (Persian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Suomi (Finnish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-fi_FI" data-locale="fi_FI" role="menuitem" lang="fi_FI">
Suomi (Finnish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Français (French)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-fr_FR" data-locale="fr_FR" role="menuitem" lang="fr_FR">
Français (French)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="हिंदी (Hindi)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-hi_IN" data-locale="hi_IN" role="menuitem" lang="hi_IN">
हिंदी (Hindi)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Magyar (Hungarian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-hu_HU" data-locale="hu_HU" role="menuitem" lang="hu_HU">
Magyar (Hungarian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Bahasa Indonesia (Indonesian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-in_ID" data-locale="in_ID" role="menuitem" lang="in_ID">
Bahasa Indonesia (Indonesian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Italiano (Italian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-it_IT" data-locale="it_IT" role="menuitem" lang="it_IT">
Italiano (Italian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="עברית (Hebrew)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-iw_IL" data-locale="iw_IL" role="menuitem" lang="iw_IL">
עברית (Hebrew)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="日本語 (Japanese)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ja_JP" data-locale="ja_JP" role="menuitem" lang="ja_JP">
日本語 (Japanese)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="한국어 (Korean)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ko_KR" data-locale="ko_KR" role="menuitem" lang="ko_KR">
한국어 (Korean)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="मराठी (Marathi)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-mr_IN" data-locale="mr_IN" role="menuitem" lang="mr_IN">
मराठी (Marathi)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Bahasa Malaysia (Malay)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ms_MY" data-locale="ms_MY" role="menuitem" lang="ms_MY">
Bahasa Malaysia (Malay)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Nederlands (Dutch)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-nl_NL" data-locale="nl_NL" role="menuitem" lang="nl_NL">
Nederlands (Dutch)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Norsk (Norwegian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-no_NO" data-locale="no_NO" role="menuitem" lang="no_NO">
Norsk (Norwegian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="ਪੰਜਾਬੀ (Punjabi)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-pa_IN" data-locale="pa_IN" role="menuitem" lang="pa_IN">
ਪੰਜਾਬੀ (Punjabi)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Polski (Polish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-pl_PL" data-locale="pl_PL" role="menuitem" lang="pl_PL">
Polski (Polish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Português (Portuguese)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-pt_BR" data-locale="pt_BR" role="menuitem" lang="pt_BR">
Português (Portuguese)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Română (Romanian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ro_RO" data-locale="ro_RO" role="menuitem" lang="ro_RO">
Română (Romanian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Русский (Russian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ru_RU" data-locale="ru_RU" role="menuitem" lang="ru_RU">
Русский (Russian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Svenska (Swedish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-sv_SE" data-locale="sv_SE" role="menuitem" lang="sv_SE">
Svenska (Swedish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="తెలుగు (Telugu)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-te_IN" data-locale="te_IN" role="menuitem" lang="te_IN">
తెలుగు (Telugu)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="ภาษาไทย (Thai)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-th_TH" data-locale="th_TH" role="menuitem" lang="th_TH">
ภาษาไทย (Thai)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Tagalog (Tagalog)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-tl_PH" data-locale="tl_PH" role="menuitem" lang="tl_PH">
Tagalog (Tagalog)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Türkçe (Turkish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-tr_TR" data-locale="tr_TR" role="menuitem" lang="tr_TR">
Türkçe (Turkish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Українська (Ukrainian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-uk_UA" data-locale="uk_UA" role="menuitem" lang="uk_UA">
Українська (Ukrainian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Tiếng Việt (Vietnamese)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-vi_VN" data-locale="vi_VN" role="menuitem" lang="vi_VN">
Tiếng Việt (Vietnamese)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="简体中文 (Chinese (Simplified))" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-zh_CN" data-locale="zh_CN" role="menuitem" lang="zh_CN">
简体中文 (Chinese (Simplified))
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="正體中文 (Chinese (Traditional))" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-zh_TW" data-locale="zh_TW" role="menuitem" lang="zh_TW">
正體中文 (Chinese (Traditional))
</button>
</li>
<!---->
</ul>
<button class="language-selector__button select-none relative pr-2 font-sans text-xs font-bold text-color-text-low-emphasis hover:text-color-link-hover hover:cursor-pointer focus:text-color-link-focus focus:outline-dotted focus:outline-1" aria-expanded="false" data-tracking-control-name="footer-lang-dropdown_trigger">
<span class="language-selector__label-text mr-0.5 break-words">
Language
</span>
<icon class="language-selector__label-chevron w-2 h-2 absolute top-0 right-0" data-delayed-url="https://static.licdn.com/aero-v1/sc/h/cyolgscd0imw2ldqppkrb84vo"></icon>
</button>
</div>
</li>
</ul>
<!----> </footer>
<script src="https://static.licdn.com/aero-v1/sc/h/e69gtg8c61dzqklxridy0htl9" async></script>
<!---->
<!----><!---->
</body>
</html>