Files
nexus/sreweekly/articles/231/05-theory-vs-practice-learnings-from-a-recent-hadoop-incident.html
2026-09-12 17:23:01 +08:00

1483 lines
90 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en">
<head>
<meta name="pageKey" content="d_lithograph_content">
<!----><!----><!----> <meta name="locale" content="en_US">
<!----> <meta id="config" data-app-id="lithograph" data-app-version="0.0.2138" data-call-tree-id="AAZbPpSyQndnLUapNPVT6A==" data-multiproduct-name="lithograph-publish-frontend" data-service-name="lithograph-publish-frontend" data-browser-id="d1881e3a-27d5-40b8-8a38-c4868d67d8bc" data-is-bot="false" data-page-instance="urn:li:page:d_lithograph_content;ZMKxMiE9RDqtAZ2jew3/cQ==" data-disable-jsbeacon-pagekey-suffix="false" data-dna-member-lix-treatment="control" data-human-member-lix-treatment="control" data-dfp-member-lix-treatment="control" data-sync-apfc-headers-lix-treatment="control" data-sync-apfc-cb-lix-treatment="control" data-recaptcha-v3-integration-lix-value="control" data-network-interceptor-lix-value="control" data-is-epd-audit-event-enabled="false" data-is-feed-sponsored-tracking-kill-switch-enabled="false" data-sequence-auto-redirect-before-request-enabled="true">
<link rel="canonical" href="https://www.linkedin.com/blog/engineering/open-source/learnings-from-a-recent-hadoop-incident">
<!----><!---->
<!---->
<!---->
<!---->
<!---->
<link rel="icon" href="https://static.licdn.com/aero-v1/sc/h/al2o9zrvru7aqj8e1x2rzsrca">
<script>
function getDfd() {let yFn,nFn;const p=new Promise(function(y, n){yFn=y;nFn=n;});p.resolve=yFn;p.reject=nFn;return p;}
window.lazyloader = getDfd();
window.tracking = getDfd();
window.impressionTracking = getDfd();
window.ingraphTracking = getDfd();
window.appDetection = getDfd();
window.pemTracking = getDfd();
window.appRedirectCompleted = getDfd();
</script>
<!---->
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Theory vs. Practice: Learnings from a recent Hadoop incident</title>
<!---->
<meta property="og:type" content="website">
<meta property="og:title" content="Theory vs. Practice: Learnings from a recent Hadoop incident">
<!---->
<meta property="og:url" content="https://www.linkedin.com/blog/engineering/open-source/learnings-from-a-recent-hadoop-incident">
<meta property="og:image" content="https://media.licdn.com/dms/image/v2/D4D08AQGqy24cYrI3zA/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700684578103?e=2147483647&amp;v=beta&amp;t=lgCq3J3ZBNqQ20sJD1XXV4LKaX8dRCuLik-cS-F-I9U">
<meta name="twitter:title" content="Theory vs. Practice: Learnings from a recent Hadoop incident">
<meta name="twitter:site" content="@linkedin">
<!---->
<!---->
<meta name="robots" content="noarchive">
<meta name="linkedin:pageTag" content="/blog/engineering/open-source/learnings-from-a-recent-hadoop-incident">
<meta name="litmsProfileName" content="lithograph">
<!----> <meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="asset-url" id="artdeco/static/images/icons.svg" content="https://static.licdn.com/aero-v1/sc/h/6sz1e821vgereyxaakx87tt4z ">
<link rel="stylesheet" href="https://static.licdn.com/aero-v1/sc/h/lbz6744ddd0wsc02ey2a3dhx">
<!---->
<!----> </head>
<body dir="ltr">
<!---->
<a href="#lithograph-app" class="skip-link btn-md btn-primary absolute z-11 -top-[100vh] focus:top-0">
Skip to main content
</a>
<!---->
<div class="page-header__wrapper page-header__wrapper--margin" data-custom-nav-wrapper="true">
<div class="page-header__container">
<header id="page-header" class="page-header page-header--hide-dropshadow">
<div class="page-header__content-container">
<div class="header__logo">
<a href="https://www.linkedin.com/blog/engineering" class="custom-nav__logo-link">
<img class="header__linkedin-logo header__full-logo" src="https://static.licdn.com/aero-v1/sc/h/6zm111mce7vohqze950ilreo4" alt="LinkedIn Logo">
<img class="header__linkedin-logo header__in-bug" src="https://static.licdn.com/aero-v1/sc/h/5g0hjlcng3j5pgn50n2et1ca2" alt="LinkedIn Logo">
<span class="t-17 t-black t-bold default-header-title">
Engineering Blog
</span>
</a>
</div>
<!---->
<!---->
<button class="stripped header__expand-nav-button" data-mobile-header-navigation-button data-header-expand-navigation-button aria-expanded="false">
<li-icon class="header__expand-nav-icon" a11y-text="Open navigation" type="hamburger-icon"></li-icon>
</button>
<button class="stripped header__expand-nav-button hidden" data-mobile-header-navigation-button data-header-collapse-navigation-button aria-expanded="false">
<li-icon class="header__expand-nav-icon" a11y-text="Close navigation" type="cancel-icon"></li-icon>
</button>
</div>
</header>
</div>
<nav class="header-nav fixed" aria-label="main" data-header-navigation>
<ul class="header-nav__list">
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/data" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Data
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Data" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Graph System</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/economic-graph" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Economic Graph</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/skills-graph" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Skills Graph</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/knowledge" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Knowledge Graph</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/data-management" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Management</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/data-streaming-processing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Streams/Processing</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/data-science" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Science</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/research" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Research</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/ab-testing-experimentation" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">A/B Testing/Experimentation</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/artificial-intelligence" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
AI
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for AI" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/machine-learning" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Machine Learning</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/natural-language-processing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Natural Language Processing</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/generative-ai" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Generative AI</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/data-modeling" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Modeling</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/responsible-ai" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Responsible AI</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/trust-and-safety" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Trust & Safety
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Trust &amp;amp; Safety" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/fairness" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Fairness</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/verification" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Verification </a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/security" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Security</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/product-design" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Product Design
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Product Design" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/marketing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Marketing </a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/sales" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Sales</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/learning" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Learning</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/hiring" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Hiring</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Profile</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/messaging-notifications" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Messaging/Notifications</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/feed" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Feed</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Profile</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/groups" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Groups</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/accessibility" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Accessibility</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/member-customer-experience" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Member/Customer Experience</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/search" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Search</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/recommendations" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Recommendations</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/skills" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Skills</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/infrastructure" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Infrastructure
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Infrastructure" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/code" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Code</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/scalability" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Scalability</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/distributed-systems" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Distributed Systems</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Graph Systems</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/architecture" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Architecture</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/analytics" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Analytics</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/automation" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Automation</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/platform-platformization" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Platforms/Platformization</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/cloud-computing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Cloud Computing</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/open-source" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Open Source</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/additional-innovations" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Additional Innovations</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/talent" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Talent
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Talent" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/culture" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Culture</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/developer-experience-productivity" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Developer Experience/Productivity</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/leadership" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Leadership</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
</ul>
</nav>
<nav class="custom-header__nav--mobile invisible" data-mobile-navigation="true" aria-label="main">
<ul class="custom-header__list--mobile">
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/data" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Data
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Data" aria-expanded="false" data-label-expand="Expand to show more links for Data" data-label-collapse="Collapse to show less links for Data" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-sans t-black t-normal header-nav__link" target="_self">Graph System</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/economic-graph" class="t-sans t-black t-normal header-nav__link" target="_self">Economic Graph</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/skills-graph" class="t-sans t-black t-normal header-nav__link" target="_self">Skills Graph</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/knowledge" class="t-sans t-black t-normal header-nav__link" target="_self">Knowledge Graph</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/data-management" class="t-sans t-black t-normal header-nav__link" target="_self">Data Management</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/data-streaming-processing" class="t-sans t-black t-normal header-nav__link" target="_self">Data Streams/Processing</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/data-science" class="t-sans t-black t-normal header-nav__link" target="_self">Data Science</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/research" class="t-sans t-black t-normal header-nav__link" target="_self">Research</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/ab-testing-experimentation" class="t-sans t-black t-normal header-nav__link" target="_self">A/B Testing/Experimentation</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/artificial-intelligence" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
AI
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for AI" aria-expanded="false" data-label-expand="Expand to show more links for AI" data-label-collapse="Collapse to show less links for AI" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/machine-learning" class="t-sans t-black t-normal header-nav__link" target="_self">Machine Learning</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/natural-language-processing" class="t-sans t-black t-normal header-nav__link" target="_self">Natural Language Processing</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/generative-ai" class="t-sans t-black t-normal header-nav__link" target="_self">Generative AI</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/data-modeling" class="t-sans t-black t-normal header-nav__link" target="_self">Data Modeling</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/responsible-ai" class="t-sans t-black t-normal header-nav__link" target="_self">Responsible AI</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/trust-and-safety" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Trust &amp; Safety
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Trust &amp;amp; Safety" aria-expanded="false" data-label-expand="Expand to show more links for Trust &amp;amp; Safety" data-label-collapse="Collapse to show less links for Trust &amp;amp; Safety" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/fairness" class="t-sans t-black t-normal header-nav__link" target="_self">Fairness</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/verification" class="t-sans t-black t-normal header-nav__link" target="_self">Verification </a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/security" class="t-sans t-black t-normal header-nav__link" target="_self">Security</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/product-design" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Product Design
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Product Design" aria-expanded="false" data-label-expand="Expand to show more links for Product Design" data-label-collapse="Collapse to show less links for Product Design" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/marketing" class="t-sans t-black t-normal header-nav__link" target="_self">Marketing </a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/sales" class="t-sans t-black t-normal header-nav__link" target="_self">Sales</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/learning" class="t-sans t-black t-normal header-nav__link" target="_self">Learning</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/hiring" class="t-sans t-black t-normal header-nav__link" target="_self">Hiring</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-sans t-black t-normal header-nav__link" target="_self">Profile</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/messaging-notifications" class="t-sans t-black t-normal header-nav__link" target="_self">Messaging/Notifications</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/feed" class="t-sans t-black t-normal header-nav__link" target="_self">Feed</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-sans t-black t-normal header-nav__link" target="_self">Profile</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/groups" class="t-sans t-black t-normal header-nav__link" target="_self">Groups</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/accessibility" class="t-sans t-black t-normal header-nav__link" target="_self">Accessibility</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/member-customer-experience" class="t-sans t-black t-normal header-nav__link" target="_self">Member/Customer Experience</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/search" class="t-sans t-black t-normal header-nav__link" target="_self">Search</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/recommendations" class="t-sans t-black t-normal header-nav__link" target="_self">Recommendations</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/skills" class="t-sans t-black t-normal header-nav__link" target="_self">Skills</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/infrastructure" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Infrastructure
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Infrastructure" aria-expanded="false" data-label-expand="Expand to show more links for Infrastructure" data-label-collapse="Collapse to show less links for Infrastructure" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/code" class="t-sans t-black t-normal header-nav__link" target="_self">Code</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/scalability" class="t-sans t-black t-normal header-nav__link" target="_self">Scalability</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/distributed-systems" class="t-sans t-black t-normal header-nav__link" target="_self">Distributed Systems</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-sans t-black t-normal header-nav__link" target="_self">Graph Systems</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/architecture" class="t-sans t-black t-normal header-nav__link" target="_self">Architecture</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/analytics" class="t-sans t-black t-normal header-nav__link" target="_self">Analytics</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/automation" class="t-sans t-black t-normal header-nav__link" target="_self">Automation</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/platform-platformization" class="t-sans t-black t-normal header-nav__link" target="_self">Platforms/Platformization</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/cloud-computing" class="t-sans t-black t-normal header-nav__link" target="_self">Cloud Computing</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/open-source" class="t-sans t-black t-normal header-nav__link" target="_self">Open Source</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/additional-innovations" class="t-sans t-black t-normal header-nav__link" target="_self">Additional Innovations</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/talent" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Talent
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Talent" aria-expanded="false" data-label-expand="Expand to show more links for Talent" data-label-collapse="Collapse to show less links for Talent" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/culture" class="t-sans t-black t-normal header-nav__link" target="_self">Culture</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/developer-experience-productivity" class="t-sans t-black t-normal header-nav__link" target="_self">Developer Experience/Productivity</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/leadership" class="t-sans t-black t-normal header-nav__link" target="_self">Leadership</a>
</li>
</ul>
</li>
</ul>
<div class="custom-header__button-share-container--mobile">
<!---->
</div>
</nav>
</div>
<main role="main">
<div id="lithograph-app">
<section id="component-container" data-attach-collapse-behavior="true">
<div id class="component component-articleHeadline" data-component-type="articleHeadline">
<div class="article-headline-container">
<p class="article-headline__topic">
<a class="t-14" href="https://www.linkedin.com/blog/engineering/open-source">Open Source</a>
</p>
<h1 class="article-headline t-40" data-article-title="Theory vs. Practice: Learnings from a recent Hadoop incident">Theory vs. Practice: Learnings from a recent Hadoop incident</h1>
<div class="article-meta t-14 article-meta__hairline">
<div class="article-meta__byline-container">
<div class="author-profile__author-container">
<!----> <div class="author-profile__author-text-container">
<a href="https://www.linkedin.com/in/sandhyaramu" target="_blank" class="t-14 t-bold"><span class="screen-reader">Authored by</span>Sandhya Ramu</a>
<p class="t-14">Engineering Leadership at Crusoe</p>
<p class="t-14 t-black--light" data-published-date="2020-8-6">
August 6, 2020
</p>
</div>
</div>
<!---->
</div>
<div class="social-share">
<ul class="social-share__list share-list" title="Share via" data-share-list>
<li class="share-list__item share-item">
<a href="#" class="share-item__link medium-round-muted-tertiary-button" role="button" data-tracking-control-name="_linkedin_share" data-share-type="linkedin" aria-label="Share on LinkedIn, New window will open">
<li-icon class="share-item__icon" type="linkedin-icon"></li-icon>
</a>
</li>
<li class="share-list__item share-item">
<a href="#" class="share-item__link medium-round-muted-tertiary-button" role="button" data-tracking-control-name="_facebook_share" data-share-type="facebook" aria-label="Share on Facebook, New window will open">
<li-icon class="share-item__icon" type="facebook-icon"></li-icon>
</a>
</li>
<li class="share-list__item share-item">
<a href="#" class="share-item__link medium-round-muted-tertiary-button" role="button" data-tracking-control-name="_twitter_share" data-share-type="twitter" aria-label="Share on X, New window will open">
<svg class="share-item__icon share-item__icon--twitter" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" fill="currentColor" focusable="false" aria-hidden="true" role="none">
<path d="M13.9 10.47L21.35 2h-1.76l-6.46 7.35L7.96 2H2l7.81 11.12L2 22h1.76l6.83-7.77L16.04 22H22l-8.1-11.53zm-2.42 2.75l-.79-1.11L4.4 3.3h2.71l5.08 7.11.79 1.11 6.6 9.24h-2.71l-5.39-7.54z"></path>
</svg>
</a>
</li>
</ul>
</div>
</div>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p><i>Co-authors:&nbsp;<a href="https://www.linkedin.com/in/sandhyaramu" target="_blank" data-tracking-control-name="_anchor_link">Sandhya Ramu</a> and <a href="https://www.linkedin.com/in/vasanth-rajamani-471b84" target="_blank" data-tracking-control-name="_anchor_link">Vasanth Rajamani</a></i></p>
<p>For companies and organizations, failure tends to be far more illuminating than success and the lingering effects of a failure can be harmful if the team moves too quickly and does not resolve the issue in a thorough and transparent manner. We recently ran into a large incident that involved data loss in our big data ecosystem and by reflecting on our diagnosis and response, we hope that our learnings from an impactful incident in our big data ecosystem will be insightful.</p>
<p>Here’s what happened: roughly 2% of the machines across a handful of racks were inadvertently reimaged. This was caused by procedural gaps in our Hadoop infrastructure’s host life cycle management. Compounding our woes, the incident happened on our business-critical production cluster.</p>
<h2>Theory meets practice</h2>
<p>Like most big data systems, our architecture has built-in safety nets that will save the day until a certain threshold. At LinkedIn, we have two large <a href="https://engineering.linkedin.com/blog/search?q=hadoop" target="_blank" data-tracking-control-name="_anchor_link">Hadoop</a> clusters independently populated with data. Within each cluster, every data block is replicated three times providing rack-level failure redundancy. The two ingestion pipelines in each cluster have completely independent paths for ingesting tracking, database data, etc., in parallel. Below is a depiction of the high-level architecture diagram:</p>
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQGBOcfLcd7zNg/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700684578118?e=2147483647&amp;v=beta&amp;t=YuJXftOGM200zpXKbxE19PLk3sFmrORBvN_4hu88ITU" alt="diagram-showing-hadoop-dual-ingest-pipelines">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>Our disaster recovery strategy has largely been focused around replicating data from a cluster in the event of data loss in another. Let’s compare how this practice held up with theory under the cloud of a real and significant incident.</p>
<p><b>Large consequences of small losses<br> </b>Compared to the loss of machines that was inadvertently spread across a handful of racks, the data loss was relatively small (~0.02%). However, we knew this would result in permanent data loss for the (HDFS) blocks in which all three replicas happened to be on those affected racks. Had we lost all machines in the affected racks, the data loss would still be small (~0.15%). We did not lose all machines in the affected racks and hence, the number was likely to be even smaller (~0.03%). However, we discovered that the actual number of lost blocks was even smaller (approximately ~0.02%). We hypothesize that during the short interval when we were losing machines, the Hadoop NameNode diligently replicated some of the blocks reducing the overall impact. In aggregate, the issue affected 0.05% of all possible files.</p>
<p>At LinkedIn, the vast majority of scheduled workflows (Spark-based applications) are triggered via <a href="https://azkaban.github.io/" target="_blank" data-tracking-control-name="_anchor_link">Azkaban</a>. Corruption in a small number of files will cause a large number of Azkaban workflows to fail. The damaged files belonged to hot datasets that many workflows consumed. Further, when an Azkaban flow tries to read a large dataset containing many files, even a single file corruption will cause the flow to fail. The combination of these factors meant that while only 0.02% data was corrupted, 10% of our workflows failed (many of which were revenue-impacting).</p>
<p>To summarize, combinatorics worked in our favor to minimize the number of block losses. However, even this seemingly small amount of data corruption resulted in a significant impact on production.</p>
<h2>Challenges</h2>
<p>As alluded to earlier, our architecture is designed to have two Hadoop clusters in different data centers with their own parallel data ingestion pipelines. Data can thus be recovered from one cluster to another. When we first started responding to the incident, we assumed that we could easily recover all data generated in parallel across the two clusters and that the primary challenge would be in handling corruptions in the intermediate data generated by Azkaban workflows that are not replicated across clusters unless explicitly requested. Due to several external factors (capital expenditures, frequency of change, ease of regeneration, and recovery time objective), not all intermediate data generated by Azkaban workflows in one cluster is replicated to the other cluster by default and we encountered some unexpected challenges along the way.</p>
<p>The first discovery was that our recovery would require significantly more data copy than anticipated. This is because each Hadoop cluster, completely independent of the other, organized the data slightly differently. We use a time-based data layout while ingesting new data. Consequently, the files in a particular folder can differ across clusters due to inherent non-determinism. Further, we compact older data into large files in a single daily partition for query efficiency and system performance. The culmination of these subtle differences meant that it was not straightforward to identify the lost files in the unaffected cluster. We would in fact have to copy the entire partition for the day whenever there was a small corruption inside that daily partition. For further context, copying a single file might mean moving 1 GB worth of data, while copying an entire day’s partition would be ~1TB. The increase in the amount of data that needed to be copied increased the time for recovery significantly.&nbsp;</p>
<p>To mitigate the process, we leveraged the following techniques:</p>
<ul>
<li>At LinkedIn, we have QoS (Quality of Service) controls implemented that manage our network resources by setting priorities for specific types of data on the network. Member traffic takes precedence over HDFS data. In order to help our recovery efforts, network backbone paths were temporarily altered to give twice the bandwidth to significantly speed up data recovery.</li>
<li>We performed recovery in a data-driven manner. Recovery of datasets were prioritized based on a combination of the percentage of data lost in a given partition, freshness of the data, and the usage level of the data lost.&nbsp;</li>
<li>We coordinated the deletion of the corrupted data with flow owners. The challenge was that Azkaban flows can retrigger based on a predefined schedule and some flow owners strictly preferred not to run their flows with partial data (with the deleted corrupted files), while others were sensitive to latency and had a strong preference to restart computation. If more than one flow used the same piece of corrupted data, a global policy had to be agreed upon across flow owners.</li>
<li>When we added 2x network bandwidth to speed up data copies across clusters, a configuration error in the network routing tier caused a higher stress to the overall system resulting in compounding failures. The stress on the system from an accelerated large-scale recovery can sometimes be riskier than performing a gradual recovery without pushing the system well past its limits. While we understood this intellectually going in, we did not fully grasp the risk involved.</li>
<li>The team published a few options for flow recovery. Based on business impact and costs, the application owners decide the degree of resilience for their respective flows. Some of our most critical business metric computation flows run on both clusters in an active/active HA (high availability) mode. These SLA-critical computation flows had zero impact due to this outage. However, this is also the most costly option. We provided three alternatives for flows without well-defined preexisting resiliency strategies: run the flow with partial data, wait for full data recovery (which could take several days), or run the flow in the D/R cluster that did not experience the failure. This last option worked well for some cases that had Azkaban flows running active/active mode; however, not all flows readily had this setup.</li>
</ul>
<p>Given the scale of Hadoop at LinkedIn, we had to work with several flow owners (our customers) to hone in on their strategies in real time. Some owners, whose flows were latency-sensitive, chose to immediately work off partial data while others chose to wait for the complete recovery of the data.</p>
<h2>The logistics for speedy recovery</h2>
<p>While we missed the traditional mechanisms (i.e., gathering in person in a war-room) for organization due to remote working, we spun up a virtual team over night across U.S. and India time zones. Organizationally, this looked like the following:</p>
<ul>
<li>A hub-and-spoke model consisting of multiple tracks with a designated track lead. Each track lead established success criteria for recovery in their respective tracks.</li>
<li>Track-specific group messaging was set up to reduce chatter in the central team messaging. This provided a space for tracks to handle ad hoc queries as they surfaced during the multi-day recovery process. Similarly, additional messaging channels were created for specific customer groups to increase transparency and coordination.</li>
<li>Communication cadence was established with regular touch points among track leads and within tracks in smaller groups to keep everyone on the same page.&nbsp;</li>
<li>The Grid team was responsible for tying all the tracks together and broadcasting email communications at a regular cadence about the status, while keeping a detailed log of the recovery.</li>
</ul>
<p>Overall, recovery was split into two major initiatives: data recovery and flow recovery. While data recovery was handled by the Grid team, flow recovery had dedicated track leads who worked within their business units to identify the best approach for their flows. They then proceeded with the most viable option and tracked recovery progress to share more broadly (see options in the next section). The Grid team established a phased approach, based on global priority across business units, to keep our complex system balanced and ensure that they would not tip over under undue stress created by the combination of data recovery and flow recovery compute needs happening in tandem along with regular workloads. The team added compute capacity gradually to keep up with excess demand from the recovery.</p>
<h2>Learnings institutionalized</h2>
<p>There are always learnings at the end of a large incident. Here are several that we are following through on:</p>
<ul>
<li>Establishing a robust and a more comprehensive host life cycle management for Hadoop Infrastructure.</li>
<li>Building a better understanding of our network behaviors across data centers under load and ensuring that an automated way to modify network routing on demand is in place.</li>
<li>We’re currently building our <a href="https://engineering.linkedin.com/blog/2019/building-next-infra" target="_blank" data-tracking-control-name="_anchor_link">next-generation infrastructure on Azure</a> including Hadoop stack. For the medium term, we will have an additional cluster that is built on a completely different tech stack, which should further help with redundancy.&nbsp;</li>
<li>Investigating the feasibility of other architectures as a part of our Azure migration. For example, we could ingest the data once and copy the same data to a D/R Cluster and eat the latency costs through data layout and query planning optimizations. We are in the process of adopting <a href="https://iceberg.apache.org/" target="_blank" data-tracking-control-name="_anchor_link">Apache Iceberg</a> as our table format. With Iceberg, we should be able to perform targeted recovery for just the affected files significantly better. In the interim for our current architecture, we have built several tools that allow us to aid with recovery (e.g., recovering all but the corrupt data, recovering large files from the other cluster more easily, etc.) and built runbooks around it for easy access.</li>
<li>Working towards auditing our flows to ensure that they have well defined disaster recovery protocols.</li>
<li>Increasing the frequency of our fire drills, in addition to reviewing the scorecard of how the flows performed in the fire drills against their stated recovery policies.</li>
<li>Continue investigating our tooling around understanding lineage as it proved to be extremely useful in identifying dependencies around flows and data. This will also offer the ability to understand the ecosystem’s connection graph end-to-end—this will be invaluable during a large coordination event like a disaster recovery.</li>
<li>Some flow owners are baking in resiliency in their application workflows itself. For example, latency-sensitive applications that generate key business hourly and daily metrics are making an explicit data staleness vs. resiliency tradeoff in the application logic itself.</li>
<li>Focusing on improving our ability to project data availability SLAs for data recovery to have the ability to publish them quickly if an event of this nature were to happen again. Our internal data consumers can use these SLAs and make informed decisions with respect to their decision choices in the recovery protocol.</li>
</ul>
<h2>Acknowledgements</h2>
<p>Our virtual team spanned large swaths of the organization ranging from teams in product engineering, AI and data science, infrastructure, to systems and networking. Acknowledging everyone individually would take more space than the contents of this blog! A heartfelt thank you to everyone that helped out, especially to those in the virtual tiger team that consisted of engineers from Grid, Artificial Intelligence, and Product engineering partners. Last, but not the least, thanks to our data engineering leadership team for their unyielding support during the entire episode.</p>
</div>
</div>
<div id class="component component-postList" data-component-type="postList">
<section id="postList0FocusPoint" class="component__content-container list-layout" data-post-list>
<section class="simple-topic-list">
<p class="t-14 t-bold">Topics:
<a class="simple-topic-list__topic" href="https://www.linkedin.com/blog/engineering/optimization">
Optimization
</a>
<a class="simple-topic-list__topic" href="https://www.linkedin.com/blog/engineering/open-source">
Open Source
</a>
<a class="simple-topic-list__topic" href="https://www.linkedin.com/blog/engineering/data">
Data
</a>
<a class="simple-topic-list__topic" href="https://www.linkedin.com/blog/engineering/data-management">
Data Management
</a>
</p>
</section>
<hr class="artdeco-divider">
<section class="list-layout__headline-container">
<p class="list-layout__headline t-32 t-sans">Related articles</p>
<!---->
</section>
<ul class="list-layout__posts">
<li class="post-list__item list-post">
<div class="list-post__content-container">
<!---->
<div class="list-post__content-container__title">
<a class="list-post__link t-20 t-black" href="https://www.linkedin.com/blog/engineering/platform-platformization/accelerating-go-to-market-velocity-with-the-product-configuration-center">
Accelerating Go-To-Market Velocity with the Product Configurat...
</a>
</div>
<div class="list-post__content-container__byline">
<p class="list-post__content-container__author t-14 t-bold t-black">
Tanmay Khemka
</p>
<p class="list-post__content-container__date t-14 t-black--light">
Jun 12, 2026
</p>
</div>
</div>
<div class="list-post__image-container">
<figure class="post__image-figure">
<img class="post__image" alt data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQFOqEs5bho66Q/croft-frontend-shrinkToFit1920/B4DZ64OKZFKAAM-/0/1781207181422?e=2147483647&amp;v=beta&amp;t=rjSx77PGu8owDXBzSBjbV_W0ZbtYnq5c3Zu8LYbmOtM" data-ghost-url="https://static.licdn.com/aero-v1/sc/h/8pouyb1ly746xtru0x7p2axm0" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 480px">
</figure>
</div>
</li>
<li class="post-list__item list-post">
<div class="list-post__content-container">
<p class="list-post__content-container__topic">
<a class="t-14 t-bold" href="https://www.linkedin.com/blog/engineering/ai">
AI
</a>
</p>
<div class="list-post__content-container__title">
<a class="list-post__link t-20 t-black" href="https://www.linkedin.com/blog/engineering/ai/scaling-llm-based-ranking-systems-with-sglang-at-linkedin">
Scaling LLM-Based ranking systems with SGLang at LinkedIn
</a>
</div>
<div class="list-post__content-container__byline">
<p class="list-post__content-container__author t-14 t-bold t-black">
Chanh Nguyen
</p>
<!----><!----><!----><!---->
<p class="list-post__content-container__date t-14 t-black--light">
Feb 20, 2026
</p>
</div>
</div>
<div class="list-post__image-container">
<figure class="post__image-figure">
<img class="post__image" alt data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQHrlPu9iSPRcQ/croft-frontend-shrinkToFit480/B4DZx70UoNHIAM-/0/1771603840485?e=2147483647&amp;v=beta&amp;t=u7rWuKzSegBjGCP2eQyZO4dqHqlsIsOwFwl6iUwa4y4" data-ghost-url="https://static.licdn.com/aero-v1/sc/h/8pouyb1ly746xtru0x7p2axm0" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 480px">
</figure>
</div>
</li>
<li class="post-list__item list-post">
<div class="list-post__content-container">
<p class="list-post__content-container__topic">
<a class="t-14 t-bold" href="https://www.linkedin.com/blog/engineering/ai">
AI
</a>
</p>
<div class="list-post__content-container__title">
<a class="list-post__link t-20 t-black" href="https://www.linkedin.com/blog/engineering/ai/turbocharging-linkedins-recommendation-systems-with-sglang">
Turbocharging LinkedIn’s Recommendation Systems with SGLang
</a>
</div>
<div class="list-post__content-container__byline">
<p class="list-post__content-container__author t-14 t-bold t-black">
Steven Shimizu
</p>
<!----><!----><!----><!----><!----><!----><!----><!----><!----><!---->
<p class="list-post__content-container__date t-14 t-black--light">
Dec 9, 2025
</p>
</div>
</div>
<div class="list-post__image-container">
<figure class="post__image-figure">
<img class="post__image" alt data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQGYk3FhVZQfeQ/croft-frontend-shrinkToFit480/B4DZrvWZLCIEAM-/0/1764952217834?e=2147483647&amp;v=beta&amp;t=4XJmBvD9Y7-t3QyDGllZCaW2m-T_l3-9u5hOtpDj3XM" data-ghost-url="https://static.licdn.com/aero-v1/sc/h/8pouyb1ly746xtru0x7p2axm0" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 480px">
</figure>
</div>
</li>
</ul>
<!----> </section>
</div>
<!----> </section>
</div>
<script src="https://static.licdn.com/aero-v1/sc/h/8hfbuq1ftcvnnx4dd5067pi0t" async></script>
<script src="https://static.licdn.com/aero-v1/sc/h/bknx7jzs0ylu540sn4wuo6bue" async></script>
<script src="https://static.licdn.com/aero-v1/sc/h/egqq1vs0784qd3h3801e2w9lr" async></script>
</main>
<footer class="li-footer bg-transparent w-full ">
<ul class="li-footer__list flex flex-wrap flex-row items-start justify-start w-full h-auto min-h-[50px] my-[0px] mx-auto py-3 px-2 papabear:p-0">
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<span class="sr-only">LinkedIn</span>
<icon class="li-footer__copy-logo text-color-logo-brand-alt inline-block self-center h-[14px] w-[56px] mr-1" data-delayed-url="https://static.licdn.com/aero-v1/sc/h/5mebydpuuijm3uhv1q375inqh"></icon>
<span class="li-footer__copy-text flex items-center">&copy; 2026</span>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://about.linkedin.com?trk=content_footer-about" data-tracking-control-name="content_footer-about" data-tracking-will-navigate>
About
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/accessibility?trk=content_footer-accessibility" data-tracking-control-name="content_footer-accessibility" data-tracking-will-navigate>
Accessibility
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/user-agreement?trk=content_footer-user-agreement" data-tracking-control-name="content_footer-user-agreement" data-tracking-will-navigate>
User Agreement
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/privacy-policy?trk=content_footer-privacy-policy" data-tracking-control-name="content_footer-privacy-policy" data-tracking-will-navigate>
Privacy Policy
</a>
</li>
<!---->
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/cookie-policy?trk=content_footer-cookie-policy" data-tracking-control-name="content_footer-cookie-policy" data-tracking-will-navigate>
Cookie Policy
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/copyright-policy?trk=content_footer-copyright-policy" data-tracking-control-name="content_footer-copyright-policy" data-tracking-will-navigate>
Copyright Policy
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://brand.linkedin.com/policies?trk=content_footer-brand-policy" data-tracking-control-name="content_footer-brand-policy" data-tracking-will-navigate>
Brand Policy
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/psettings/guest-controls?trk=content_footer-guest-controls" data-tracking-control-name="content_footer-guest-controls" data-tracking-will-navigate>
Guest Controls
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/help/linkedin/answer/34593?trk=content_footer-community-guide" data-tracking-control-name="content_footer-community-guide" data-tracking-will-navigate>
Community Guidelines
</a>
</li>
<!---->
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<div class="collapsible-dropdown collapsible-dropdown--footer collapsible-dropdown--up flex items-center relative hyphens-auto language-selector z-2">
<!---->
<ul class="collapsible-dropdown__list hidden container-raised absolute w-auto overflow-y-auto flex-col items-stretch z-[9999] bottom-[100%] top-auto" role="menu" tabindex="-1">
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="العربية (Arabic)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ar_AE" data-locale="ar_AE" role="menuitem" lang="ar_AE">
العربية (Arabic)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="বাংলা (Bangla)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-bn_IN" data-locale="bn_IN" role="menuitem" lang="bn_IN">
বাংলা (Bangla)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Čeština (Czech)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-cs_CZ" data-locale="cs_CZ" role="menuitem" lang="cs_CZ">
Čeština (Czech)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Dansk (Danish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-da_DK" data-locale="da_DK" role="menuitem" lang="da_DK">
Dansk (Danish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Deutsch (German)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-de_DE" data-locale="de_DE" role="menuitem" lang="de_DE">
Deutsch (German)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Ελληνικά (Greek)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-el_GR" data-locale="el_GR" role="menuitem" lang="el_GR">
Ελληνικά (Greek)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="English (English) selected" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link--selected" data-tracking-control-name="language-selector-en_US" data-locale="en_US" role="menuitem" lang="en_US">
<strong>English (English)</strong>
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Español (Spanish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-es_ES" data-locale="es_ES" role="menuitem" lang="es_ES">
Español (Spanish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="فارسی (Persian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-fa_IR" data-locale="fa_IR" role="menuitem" lang="fa_IR">
فارسی (Persian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Suomi (Finnish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-fi_FI" data-locale="fi_FI" role="menuitem" lang="fi_FI">
Suomi (Finnish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Français (French)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-fr_FR" data-locale="fr_FR" role="menuitem" lang="fr_FR">
Français (French)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="हिंदी (Hindi)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-hi_IN" data-locale="hi_IN" role="menuitem" lang="hi_IN">
हिंदी (Hindi)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Magyar (Hungarian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-hu_HU" data-locale="hu_HU" role="menuitem" lang="hu_HU">
Magyar (Hungarian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Bahasa Indonesia (Indonesian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-in_ID" data-locale="in_ID" role="menuitem" lang="in_ID">
Bahasa Indonesia (Indonesian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Italiano (Italian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-it_IT" data-locale="it_IT" role="menuitem" lang="it_IT">
Italiano (Italian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="עברית (Hebrew)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-iw_IL" data-locale="iw_IL" role="menuitem" lang="iw_IL">
עברית (Hebrew)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="日本語 (Japanese)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ja_JP" data-locale="ja_JP" role="menuitem" lang="ja_JP">
日本語 (Japanese)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="한국어 (Korean)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ko_KR" data-locale="ko_KR" role="menuitem" lang="ko_KR">
한국어 (Korean)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="मराठी (Marathi)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-mr_IN" data-locale="mr_IN" role="menuitem" lang="mr_IN">
मराठी (Marathi)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Bahasa Malaysia (Malay)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ms_MY" data-locale="ms_MY" role="menuitem" lang="ms_MY">
Bahasa Malaysia (Malay)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Nederlands (Dutch)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-nl_NL" data-locale="nl_NL" role="menuitem" lang="nl_NL">
Nederlands (Dutch)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Norsk (Norwegian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-no_NO" data-locale="no_NO" role="menuitem" lang="no_NO">
Norsk (Norwegian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="ਪੰਜਾਬੀ (Punjabi)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-pa_IN" data-locale="pa_IN" role="menuitem" lang="pa_IN">
ਪੰਜਾਬੀ (Punjabi)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Polski (Polish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-pl_PL" data-locale="pl_PL" role="menuitem" lang="pl_PL">
Polski (Polish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Português (Portuguese)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-pt_BR" data-locale="pt_BR" role="menuitem" lang="pt_BR">
Português (Portuguese)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Română (Romanian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ro_RO" data-locale="ro_RO" role="menuitem" lang="ro_RO">
Română (Romanian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Русский (Russian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ru_RU" data-locale="ru_RU" role="menuitem" lang="ru_RU">
Русский (Russian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Svenska (Swedish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-sv_SE" data-locale="sv_SE" role="menuitem" lang="sv_SE">
Svenska (Swedish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="తెలుగు (Telugu)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-te_IN" data-locale="te_IN" role="menuitem" lang="te_IN">
తెలుగు (Telugu)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="ภาษาไทย (Thai)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-th_TH" data-locale="th_TH" role="menuitem" lang="th_TH">
ภาษาไทย (Thai)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Tagalog (Tagalog)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-tl_PH" data-locale="tl_PH" role="menuitem" lang="tl_PH">
Tagalog (Tagalog)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Türkçe (Turkish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-tr_TR" data-locale="tr_TR" role="menuitem" lang="tr_TR">
Türkçe (Turkish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Українська (Ukrainian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-uk_UA" data-locale="uk_UA" role="menuitem" lang="uk_UA">
Українська (Ukrainian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Tiếng Việt (Vietnamese)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-vi_VN" data-locale="vi_VN" role="menuitem" lang="vi_VN">
Tiếng Việt (Vietnamese)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="简体中文 (Chinese (Simplified))" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-zh_CN" data-locale="zh_CN" role="menuitem" lang="zh_CN">
简体中文 (Chinese (Simplified))
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="正體中文 (Chinese (Traditional))" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-zh_TW" data-locale="zh_TW" role="menuitem" lang="zh_TW">
正體中文 (Chinese (Traditional))
</button>
</li>
<!---->
</ul>
<button class="language-selector__button select-none relative pr-2 font-sans text-xs font-bold text-color-text-low-emphasis hover:text-color-link-hover hover:cursor-pointer focus:text-color-link-focus focus:outline-dotted focus:outline-1" aria-expanded="false" data-tracking-control-name="footer-lang-dropdown_trigger">
<span class="language-selector__label-text mr-0.5 break-words">
Language
</span>
<icon class="language-selector__label-chevron w-2 h-2 absolute top-0 right-0" data-delayed-url="https://static.licdn.com/aero-v1/sc/h/cyolgscd0imw2ldqppkrb84vo"></icon>
</button>
</div>
</li>
</ul>
<!----> </footer>
<script src="https://static.licdn.com/aero-v1/sc/h/e69gtg8c61dzqklxridy0htl9" async></script>
<!---->
<!----><!---->
</body>
</html>