1360 lines
84 KiB
HTML
1360 lines
84 KiB
HTML
<!DOCTYPE html>
|
||
|
||
|
||
|
||
|
||
<html lang="en">
|
||
<head>
|
||
<meta name="pageKey" content="d_lithograph_content">
|
||
<!----><!----><!----> <meta name="locale" content="en_US">
|
||
<!----> <meta id="config" data-app-id="lithograph" data-app-version="0.0.2138" data-call-tree-id="AAZbPqD2fqEwD6fuL9LfRQ==" data-multiproduct-name="lithograph-publish-frontend" data-service-name="lithograph-publish-frontend" data-browser-id="78884c60-a01b-45d5-8f14-66fe0553bff1" data-is-bot="false" data-page-instance="urn:li:page:d_lithograph_content;zXA6YJlETDat0xv9EaNs6A==" data-disable-jsbeacon-pagekey-suffix="false" data-dna-member-lix-treatment="control" data-human-member-lix-treatment="control" data-dfp-member-lix-treatment="control" data-sync-apfc-headers-lix-treatment="control" data-sync-apfc-cb-lix-treatment="control" data-recaptcha-v3-integration-lix-value="control" data-network-interceptor-lix-value="control" data-is-epd-audit-event-enabled="false" data-is-feed-sponsored-tracking-kill-switch-enabled="false" data-sequence-auto-redirect-before-request-enabled="true">
|
||
|
||
<link rel="canonical" href="https://www.linkedin.com/blog/engineering/infrastructure/fixing-linux-filesystem-performance-regressions">
|
||
<!----><!---->
|
||
<!---->
|
||
<!---->
|
||
<!---->
|
||
<!---->
|
||
<link rel="icon" href="https://static.licdn.com/aero-v1/sc/h/al2o9zrvru7aqj8e1x2rzsrca">
|
||
|
||
|
||
<script>
|
||
function getDfd() {let yFn,nFn;const p=new Promise(function(y, n){yFn=y;nFn=n;});p.resolve=yFn;p.reject=nFn;return p;}
|
||
window.lazyloader = getDfd();
|
||
window.tracking = getDfd();
|
||
window.impressionTracking = getDfd();
|
||
window.ingraphTracking = getDfd();
|
||
window.appDetection = getDfd();
|
||
window.pemTracking = getDfd();
|
||
window.appRedirectCompleted = getDfd();
|
||
</script>
|
||
|
||
<!---->
|
||
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
|
||
|
||
<title>Fixing Linux filesystem performance regressions</title>
|
||
<!---->
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:title" content="Fixing Linux filesystem performance regressions">
|
||
<!---->
|
||
<meta property="og:url" content="https://www.linkedin.com/blog/engineering/infrastructure/fixing-linux-filesystem-performance-regressions">
|
||
<meta property="og:image" content="https://media.licdn.com/dms/image/v2/D4D08AQEWa2gIeRIR1w/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700688307162?e=2147483647&v=beta&t=LvKT25h8SYLw-p72dRny-r-V5V9pribfdxP1iSaNsiU">
|
||
<meta name="twitter:title" content="Fixing Linux filesystem performance regressions">
|
||
<meta name="twitter:site" content="@linkedin">
|
||
<!---->
|
||
<!---->
|
||
<meta name="robots" content="noarchive">
|
||
|
||
<meta name="linkedin:pageTag" content="/blog/engineering/infrastructure/fixing-linux-filesystem-performance-regressions">
|
||
<meta name="litmsProfileName" content="lithograph">
|
||
|
||
|
||
|
||
<!----> <meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||
<meta name="asset-url" id="artdeco/static/images/icons.svg" content="https://static.licdn.com/aero-v1/sc/h/6sz1e821vgereyxaakx87tt4z ">
|
||
<link rel="stylesheet" href="https://static.licdn.com/aero-v1/sc/h/lbz6744ddd0wsc02ey2a3dhx">
|
||
<!---->
|
||
|
||
<!----> </head>
|
||
<body dir="ltr">
|
||
<!---->
|
||
|
||
|
||
|
||
<a href="#lithograph-app" class="skip-link btn-md btn-primary absolute z-11 -top-[100vh] focus:top-0">
|
||
Skip to main content
|
||
</a>
|
||
|
||
<!---->
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="page-header__wrapper page-header__wrapper--margin" data-custom-nav-wrapper="true">
|
||
<div class="page-header__container">
|
||
<header id="page-header" class="page-header page-header--hide-dropshadow">
|
||
<div class="page-header__content-container">
|
||
<div class="header__logo">
|
||
<a href="https://www.linkedin.com/blog/engineering" class="custom-nav__logo-link">
|
||
<img class="header__linkedin-logo header__full-logo" src="https://static.licdn.com/aero-v1/sc/h/6zm111mce7vohqze950ilreo4" alt="LinkedIn Logo">
|
||
<img class="header__linkedin-logo header__in-bug" src="https://static.licdn.com/aero-v1/sc/h/5g0hjlcng3j5pgn50n2et1ca2" alt="LinkedIn Logo">
|
||
<span class="t-17 t-black t-bold default-header-title">
|
||
Engineering Blog
|
||
</span>
|
||
</a>
|
||
</div>
|
||
|
||
<!---->
|
||
<!---->
|
||
<button class="stripped header__expand-nav-button" data-mobile-header-navigation-button data-header-expand-navigation-button aria-expanded="false">
|
||
<li-icon class="header__expand-nav-icon" a11y-text="Open navigation" type="hamburger-icon"></li-icon>
|
||
</button>
|
||
<button class="stripped header__expand-nav-button hidden" data-mobile-header-navigation-button data-header-collapse-navigation-button aria-expanded="false">
|
||
<li-icon class="header__expand-nav-icon" a11y-text="Close navigation" type="cancel-icon"></li-icon>
|
||
</button>
|
||
</div>
|
||
</header>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<nav class="header-nav fixed" aria-label="main" data-header-navigation>
|
||
<ul class="header-nav__list">
|
||
<li>
|
||
|
||
|
||
|
||
<div class="artdeco-dropdown">
|
||
<div class="artdeco-dropdown__trigger">
|
||
<a href="https://www.linkedin.com/blog/engineering/data" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
|
||
Data
|
||
</a>
|
||
|
||
<button class="header-navigation__button" aria-label="Expand to show more links for Data" aria-expanded="false" data-trigger-dropdown-menu type="button">
|
||
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
|
||
</button>
|
||
|
||
|
||
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
|
||
<ul>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Graph System</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/economic-graph" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Economic Graph</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/skills-graph" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Skills Graph</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/knowledge" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Knowledge Graph</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/data-management" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Management</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/data-streaming-processing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Streams/Processing</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/data-science" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Science</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/research" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Research</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/ab-testing-experimentation" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">A/B Testing/Experimentation</a>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
</li>
|
||
<li>
|
||
|
||
|
||
|
||
<div class="artdeco-dropdown">
|
||
<div class="artdeco-dropdown__trigger">
|
||
<a href="https://www.linkedin.com/blog/engineering/artificial-intelligence" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
|
||
AI
|
||
</a>
|
||
|
||
<button class="header-navigation__button" aria-label="Expand to show more links for AI" aria-expanded="false" data-trigger-dropdown-menu type="button">
|
||
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
|
||
</button>
|
||
|
||
|
||
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
|
||
<ul>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/machine-learning" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Machine Learning</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/natural-language-processing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Natural Language Processing</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/generative-ai" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Generative AI</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/data-modeling" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Modeling</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/responsible-ai" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Responsible AI</a>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
</li>
|
||
<li>
|
||
|
||
|
||
|
||
<div class="artdeco-dropdown">
|
||
<div class="artdeco-dropdown__trigger">
|
||
<a href="https://www.linkedin.com/blog/engineering/trust-and-safety" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
|
||
Trust & Safety
|
||
</a>
|
||
|
||
<button class="header-navigation__button" aria-label="Expand to show more links for Trust &amp; Safety" aria-expanded="false" data-trigger-dropdown-menu type="button">
|
||
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
|
||
</button>
|
||
|
||
|
||
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
|
||
<ul>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/fairness" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Fairness</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/verification" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Verification </a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/security" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Security</a>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
</li>
|
||
<li>
|
||
|
||
|
||
|
||
<div class="artdeco-dropdown">
|
||
<div class="artdeco-dropdown__trigger">
|
||
<a href="https://www.linkedin.com/blog/engineering/product-design" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
|
||
Product Design
|
||
</a>
|
||
|
||
<button class="header-navigation__button" aria-label="Expand to show more links for Product Design" aria-expanded="false" data-trigger-dropdown-menu type="button">
|
||
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
|
||
</button>
|
||
|
||
|
||
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
|
||
<ul>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/marketing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Marketing </a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/sales" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Sales</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/learning" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Learning</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/hiring" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Hiring</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Profile</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/messaging-notifications" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Messaging/Notifications</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/feed" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Feed</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Profile</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/groups" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Groups</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/accessibility" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Accessibility</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/member-customer-experience" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Member/Customer Experience</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/search" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Search</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/recommendations" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Recommendations</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/skills" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Skills</a>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
</li>
|
||
<li>
|
||
|
||
|
||
|
||
<div class="artdeco-dropdown">
|
||
<div class="artdeco-dropdown__trigger">
|
||
<a href="https://www.linkedin.com/blog/engineering/infrastructure" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
|
||
Infrastructure
|
||
</a>
|
||
|
||
<button class="header-navigation__button" aria-label="Expand to show more links for Infrastructure" aria-expanded="false" data-trigger-dropdown-menu type="button">
|
||
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
|
||
</button>
|
||
|
||
|
||
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
|
||
<ul>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/code" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Code</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/scalability" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Scalability</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/distributed-systems" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Distributed Systems</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Graph Systems</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/architecture" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Architecture</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/analytics" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Analytics</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/automation" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Automation</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/platform-platformization" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Platforms/Platformization</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/cloud-computing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Cloud Computing</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/open-source" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Open Source</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/additional-innovations" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Additional Innovations</a>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
</li>
|
||
<li>
|
||
|
||
|
||
|
||
<div class="artdeco-dropdown">
|
||
<div class="artdeco-dropdown__trigger">
|
||
<a href="https://www.linkedin.com/blog/engineering/talent" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
|
||
Talent
|
||
</a>
|
||
|
||
<button class="header-navigation__button" aria-label="Expand to show more links for Talent" aria-expanded="false" data-trigger-dropdown-menu type="button">
|
||
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
|
||
</button>
|
||
|
||
|
||
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
|
||
<ul>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/culture" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Culture</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/developer-experience-productivity" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Developer Experience/Productivity</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="artdeco-dropdown__item">
|
||
<a href="https://www.linkedin.com/blog/engineering/leadership" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Leadership</a>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
</li>
|
||
</ul>
|
||
</nav>
|
||
|
||
|
||
|
||
|
||
|
||
<nav class="custom-header__nav--mobile invisible" data-mobile-navigation="true" aria-label="main">
|
||
|
||
<ul class="custom-header__list--mobile">
|
||
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
|
||
|
||
|
||
|
||
|
||
<a href="https://www.linkedin.com/blog/engineering/data" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
|
||
Data
|
||
</a>
|
||
|
||
<button class="header-navigation__button" aria-label="Expand to show more links for Data" aria-expanded="false" data-label-expand="Expand to show more links for Data" data-label-collapse="Collapse to show less links for Data" data-mobile-dropdown-trigger type="button">
|
||
<li-icon size="small" type="chevron-down-icon"></li-icon>
|
||
</button>
|
||
|
||
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-sans t-black t-normal header-nav__link" target="_self">Graph System</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/economic-graph" class="t-sans t-black t-normal header-nav__link" target="_self">Economic Graph</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/skills-graph" class="t-sans t-black t-normal header-nav__link" target="_self">Skills Graph</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/knowledge" class="t-sans t-black t-normal header-nav__link" target="_self">Knowledge Graph</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/data-management" class="t-sans t-black t-normal header-nav__link" target="_self">Data Management</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/data-streaming-processing" class="t-sans t-black t-normal header-nav__link" target="_self">Data Streams/Processing</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/data-science" class="t-sans t-black t-normal header-nav__link" target="_self">Data Science</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/research" class="t-sans t-black t-normal header-nav__link" target="_self">Research</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/ab-testing-experimentation" class="t-sans t-black t-normal header-nav__link" target="_self">A/B Testing/Experimentation</a>
|
||
</li>
|
||
</ul>
|
||
|
||
|
||
</li>
|
||
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
|
||
|
||
|
||
|
||
|
||
<a href="https://www.linkedin.com/blog/engineering/artificial-intelligence" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
|
||
AI
|
||
</a>
|
||
|
||
<button class="header-navigation__button" aria-label="Expand to show more links for AI" aria-expanded="false" data-label-expand="Expand to show more links for AI" data-label-collapse="Collapse to show less links for AI" data-mobile-dropdown-trigger type="button">
|
||
<li-icon size="small" type="chevron-down-icon"></li-icon>
|
||
</button>
|
||
|
||
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/machine-learning" class="t-sans t-black t-normal header-nav__link" target="_self">Machine Learning</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/natural-language-processing" class="t-sans t-black t-normal header-nav__link" target="_self">Natural Language Processing</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/generative-ai" class="t-sans t-black t-normal header-nav__link" target="_self">Generative AI</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/data-modeling" class="t-sans t-black t-normal header-nav__link" target="_self">Data Modeling</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/responsible-ai" class="t-sans t-black t-normal header-nav__link" target="_self">Responsible AI</a>
|
||
</li>
|
||
</ul>
|
||
|
||
|
||
</li>
|
||
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
|
||
|
||
|
||
|
||
|
||
<a href="https://www.linkedin.com/blog/engineering/trust-and-safety" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
|
||
Trust & Safety
|
||
</a>
|
||
|
||
<button class="header-navigation__button" aria-label="Expand to show more links for Trust &amp; Safety" aria-expanded="false" data-label-expand="Expand to show more links for Trust &amp; Safety" data-label-collapse="Collapse to show less links for Trust &amp; Safety" data-mobile-dropdown-trigger type="button">
|
||
<li-icon size="small" type="chevron-down-icon"></li-icon>
|
||
</button>
|
||
|
||
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/fairness" class="t-sans t-black t-normal header-nav__link" target="_self">Fairness</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/verification" class="t-sans t-black t-normal header-nav__link" target="_self">Verification </a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/security" class="t-sans t-black t-normal header-nav__link" target="_self">Security</a>
|
||
</li>
|
||
</ul>
|
||
|
||
|
||
</li>
|
||
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
|
||
|
||
|
||
|
||
|
||
<a href="https://www.linkedin.com/blog/engineering/product-design" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
|
||
Product Design
|
||
</a>
|
||
|
||
<button class="header-navigation__button" aria-label="Expand to show more links for Product Design" aria-expanded="false" data-label-expand="Expand to show more links for Product Design" data-label-collapse="Collapse to show less links for Product Design" data-mobile-dropdown-trigger type="button">
|
||
<li-icon size="small" type="chevron-down-icon"></li-icon>
|
||
</button>
|
||
|
||
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/marketing" class="t-sans t-black t-normal header-nav__link" target="_self">Marketing </a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/sales" class="t-sans t-black t-normal header-nav__link" target="_self">Sales</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/learning" class="t-sans t-black t-normal header-nav__link" target="_self">Learning</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/hiring" class="t-sans t-black t-normal header-nav__link" target="_self">Hiring</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-sans t-black t-normal header-nav__link" target="_self">Profile</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/messaging-notifications" class="t-sans t-black t-normal header-nav__link" target="_self">Messaging/Notifications</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/feed" class="t-sans t-black t-normal header-nav__link" target="_self">Feed</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-sans t-black t-normal header-nav__link" target="_self">Profile</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/groups" class="t-sans t-black t-normal header-nav__link" target="_self">Groups</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/accessibility" class="t-sans t-black t-normal header-nav__link" target="_self">Accessibility</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/member-customer-experience" class="t-sans t-black t-normal header-nav__link" target="_self">Member/Customer Experience</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/search" class="t-sans t-black t-normal header-nav__link" target="_self">Search</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/recommendations" class="t-sans t-black t-normal header-nav__link" target="_self">Recommendations</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/skills" class="t-sans t-black t-normal header-nav__link" target="_self">Skills</a>
|
||
</li>
|
||
</ul>
|
||
|
||
|
||
</li>
|
||
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
|
||
|
||
|
||
|
||
|
||
<a href="https://www.linkedin.com/blog/engineering/infrastructure" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
|
||
Infrastructure
|
||
</a>
|
||
|
||
<button class="header-navigation__button" aria-label="Expand to show more links for Infrastructure" aria-expanded="false" data-label-expand="Expand to show more links for Infrastructure" data-label-collapse="Collapse to show less links for Infrastructure" data-mobile-dropdown-trigger type="button">
|
||
<li-icon size="small" type="chevron-down-icon"></li-icon>
|
||
</button>
|
||
|
||
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/code" class="t-sans t-black t-normal header-nav__link" target="_self">Code</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/scalability" class="t-sans t-black t-normal header-nav__link" target="_self">Scalability</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/distributed-systems" class="t-sans t-black t-normal header-nav__link" target="_self">Distributed Systems</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-sans t-black t-normal header-nav__link" target="_self">Graph Systems</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/architecture" class="t-sans t-black t-normal header-nav__link" target="_self">Architecture</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/analytics" class="t-sans t-black t-normal header-nav__link" target="_self">Analytics</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/automation" class="t-sans t-black t-normal header-nav__link" target="_self">Automation</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/platform-platformization" class="t-sans t-black t-normal header-nav__link" target="_self">Platforms/Platformization</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/cloud-computing" class="t-sans t-black t-normal header-nav__link" target="_self">Cloud Computing</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/open-source" class="t-sans t-black t-normal header-nav__link" target="_self">Open Source</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/additional-innovations" class="t-sans t-black t-normal header-nav__link" target="_self">Additional Innovations</a>
|
||
</li>
|
||
</ul>
|
||
|
||
|
||
</li>
|
||
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
|
||
|
||
|
||
|
||
|
||
<a href="https://www.linkedin.com/blog/engineering/talent" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
|
||
Talent
|
||
</a>
|
||
|
||
<button class="header-navigation__button" aria-label="Expand to show more links for Talent" aria-expanded="false" data-label-expand="Expand to show more links for Talent" data-label-collapse="Collapse to show less links for Talent" data-mobile-dropdown-trigger type="button">
|
||
<li-icon size="small" type="chevron-down-icon"></li-icon>
|
||
</button>
|
||
|
||
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/culture" class="t-sans t-black t-normal header-nav__link" target="_self">Culture</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/developer-experience-productivity" class="t-sans t-black t-normal header-nav__link" target="_self">Developer Experience/Productivity</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://www.linkedin.com/blog/engineering/leadership" class="t-sans t-black t-normal header-nav__link" target="_self">Leadership</a>
|
||
</li>
|
||
</ul>
|
||
|
||
|
||
</li>
|
||
</ul>
|
||
|
||
|
||
<div class="custom-header__button-share-container--mobile">
|
||
<!---->
|
||
</div>
|
||
</nav>
|
||
|
||
</div>
|
||
|
||
|
||
<main role="main">
|
||
|
||
<div id="lithograph-app">
|
||
|
||
|
||
<section id="component-container" data-attach-collapse-behavior="true">
|
||
|
||
|
||
<div id class="component component-articleHeadline" data-component-type="articleHeadline">
|
||
|
||
|
||
|
||
|
||
<div class="article-headline-container">
|
||
<p class="article-headline__topic">
|
||
<a class="t-14" href="https://www.linkedin.com/blog/engineering/infrastructure">Infrastructure</a>
|
||
</p>
|
||
|
||
<h1 class="article-headline t-40" data-article-title="Fixing Linux filesystem performance regressions">Fixing Linux filesystem performance regressions</h1>
|
||
<div class="article-meta t-14 article-meta__hairline">
|
||
<div class="article-meta__byline-container">
|
||
<p class="t-14 t-black--light" data-published-date="2020-10-16">
|
||
October 16, 2020
|
||
</p>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="social-share">
|
||
<ul class="social-share__list share-list" title="Share via" data-share-list>
|
||
|
||
|
||
<li class="share-list__item share-item">
|
||
<a href="#" class="share-item__link medium-round-muted-tertiary-button" role="button" data-tracking-control-name="_linkedin_share" data-share-type="linkedin" aria-label="Share on LinkedIn, New window will open">
|
||
<li-icon class="share-item__icon" type="linkedin-icon"></li-icon>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
<li class="share-list__item share-item">
|
||
<a href="#" class="share-item__link medium-round-muted-tertiary-button" role="button" data-tracking-control-name="_facebook_share" data-share-type="facebook" aria-label="Share on Facebook, New window will open">
|
||
<li-icon class="share-item__icon" type="facebook-icon"></li-icon>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
<li class="share-list__item share-item">
|
||
<a href="#" class="share-item__link medium-round-muted-tertiary-button" role="button" data-tracking-control-name="_twitter_share" data-share-type="twitter" aria-label="Share on X, New window will open">
|
||
<svg class="share-item__icon share-item__icon--twitter" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" fill="currentColor" focusable="false" aria-hidden="true" role="none">
|
||
<path d="M13.9 10.47L21.35 2h-1.76l-6.46 7.35L7.96 2H2l7.81 11.12L2 22h1.76l6.83-7.77L16.04 22H22l-8.1-11.53zm-2.42 2.75l-.79-1.11L4.4 3.3h2.71l5.08 7.11.79 1.11 6.6 9.24h-2.71l-5.39-7.54z"></path>
|
||
</svg>
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
<div id class="component component-migratedContent" data-component-type="migratedContent">
|
||
|
||
|
||
|
||
|
||
<div class="rich-text">
|
||
<p>As companies grow, adapt, morph, and mature, one item remains the same: the need for reinvention. Technical infrastructure is no exception. As our member community grew, our priorities were to keep up with that growth, or as we say, ensure continuous “site up.” (Read: adding servers to scale from hundreds to hundreds of thousands.) We ran into challenges about how to plan for this type of scaling—in particular, in keeping the platform images and kernels installed on our servers up to date. We moved forward in fits and starts, reimaging our entire physical server fleet in ad-hoc all-hands efforts in order to respond to various extrinsic factors, such as publicly disclosed CPU bugs.</p>
|
||
<p>The learnings we took away from these prior efforts allowed us to build a more refined and automated process for reimaging servers going forward, and to more crisply define the lifecycle of the servers on which we deploy LinkedIn’s production stack. With this increased confidence, we undertook an effort to reimage all of the servers comprising <a href="https://engineering.linkedin.com/blog/2016/05/rain--better-resource-allocation-through-containerization" target="_blank" data-tracking-control-name="_anchor_link">Rain, LinkedIn’s private cloud</a>, to <a href="https://www.centos.org/" target="_blank" data-tracking-control-name="_anchor_link">CentOS</a> with a modern kernel. This blog post aims to share a new set of learnings from our most recent effort.</p>
|
||
<h2>The problem</h2>
|
||
<p>At its start, the CentOS reimaging process went mostly according to plan. However, as we neared completion, we suddenly halted the process because of multiple reports of severe 99th percentile latency increases for a serving application when an instance of another application was being deployed on the same physical server. The problem only affected servers with the new image, so we had our work cut out for us to avoid a lengthy and discouraging rollback process across tens of thousands of servers. The bug itself could have disrupted service for our members, and a platform image rollback would carry yet another set of risks.</p>
|
||
<p><a href="https://events19.linuxfoundation.org/wp-content/uploads/2017/12/Are-You-Insured-Against-Your-Noisy-Neighbor-Sunku-Ranganath-Intel-1.pdf" target="_blank" data-tracking-control-name="_anchor_link">"Noisy neighbor" problems</a> are well known in multi-tenant scheduling environments like most cloud platforms. To avoid the <a href="https://en.wikipedia.org/wiki/Tragedy_of_the_commons" target="_blank" data-tracking-control-name="_anchor_link">tragedy of the commons</a>, abstractions such as containerization and time-sharing are introduced. However, abstractions are leaky—one tenant is often able to breach the agreement and unfairly exclude other users of a global resource.</p>
|
||
<p>In maintaining our private cloud, we have become familiar enough with this class of problem that we knew exactly where to look: load average, system CPU utilization, page cache utilization, free page scans, disk queues. <a href="https://www.atoptool.nl/" target="_blank" data-tracking-control-name="_anchor_link">Atop</a> is a great tool for diagnosing a shared server at a glance.</p>
|
||
<p>The interesting thing about this problem is that no shared resource was being exhausted. This pointed in the direction of a mutual exclusion problem. Some tools for diagnosing mutual exclusion problems are to look at the stacks of each process, to echo l > /proc/sysrq-trigger to snapshot each core’s stack, and to use the perf top and/or perf record utilities. All of these tools comprise an approach to determine where the system is applying its wall-clock time, since it seems to not be spending enough of it executing the workload that serves our members.</p>
|
||
<p>Strangely, these efforts turned up nothing of interest. The system wasn’t busy; it was just slower for our workload, according to the wall clock, than the older platform image.</p>
|
||
<p>Fresh out of quick explanations, we attempted to create test cases to reproduce the problem. A valid test case would be fast on the old platform image and slow on the new one. One team created a test case which reproduced the problem by downloading several large artifacts in parallel. Another team then created a benchmark utilizing the <a href="https://github.com/axboe/fio" target="_blank" data-tracking-control-name="_anchor_link">fio</a> test framework, which was fast on the old image but exceeded its configured runtime by multiple minutes on the new image. We determined that the problem existed in both kernels 4.19 and 5.4.</p>
|
||
<p>Concurrently, we deduced that this problem exclusively impacted older servers with HDD (rotating) root disks, which we arrange in a software RAID1 mirror configuration—newer servers with SSD (solid-state) root disks were unaffected. One engineer noticed that there was an <a href="https://bugzilla.kernel.org/show_bug.cgi?id=204253" target="_blank" data-tracking-control-name="_anchor_link">upstream bug</a> tracking an issue seemingly related to <a href="https://www.thomas-krenn.com/en/wiki/Linux_Multi-Queue_Block_IO_Queueing_Mechanism_(blk-mq)_Details" target="_blank" data-tracking-control-name="_anchor_link">blk-mq</a>. It seemed plausible that this was a blk-mq problem since blk-mq was introduced after the release of kernel 3.10 (our previous golden image kernel). It was also evident that the system was not actually busy while the latency problem existed, so it seemed to be reasonable to hypothesize that inefficient I/O submission was the root cause.</p>
|
||
<p>Applying <a href="https://bugzilla.kernel.org/attachment.cgi?id=287655&action=diff" target="_blank" data-tracking-control-name="_anchor_link">the patch</a> attached to that upstream bug (to reduce the number of queues in the scalable bitmap layer) did improve performance enough to move some teams forward. Based on this result, we explored the possibility that the regression was related to the <a href="https://www.phoronix.com/scan.php?page=news_item&px=Linux-SCSI-MQ-Default-Soon" target="_blank" data-tracking-control-name="_anchor_link">scsi-mq migration</a> and the new I/O schedulers it required. However, after trying a number of configurations, it was clear that the choice or configuration of the I/O scheduler had little to no impact on the problem.</p>
|
||
<h2>ext4 precedent</h2>
|
||
<p>Due to prior experience in our storage tier, we were also familiar with the <a href="https://www.percona.com/blog/2019/11/12/watch-out-for-disk-i-o-performance-issues-when-running-ext4/" target="_blank" data-tracking-control-name="_anchor_link">ext4 regression introduced in Linux 4.9 for direct I/O workloads</a>. There was no equivalent guidance that we could find addressing increased latency on normal buffered I/O workloads in kernel 4.x. With suspicion of ext4 aroused by that existing direct I/O issue, we decided to replace the ext4 filesystem with XFS on some test servers to determine whether ext4 was again at fault here.</p>
|
||
<p>Surprisingly, we found that the problem was indeed nonexistent on XFS. Remember, this problem exclusively affected servers with rotating disk storage, so what could cause a latency problem exclusively on ext4, only on rotating HDD devices, and on a server that isn’t busy?</p>
|
||
<h2>Rooting out the cause</h2>
|
||
<p>Lacking other palatable options and knowing that the problem was introduced sometime between kernel 3.10 and 4.19, we proceeded to take one of the affected servers out of rotation and bisect the kernel. Bisecting the kernel on a physical host in a datacenter came with its own set of complications and workarounds, requiring additional assistance across multiple teams.</p>
|
||
<p>In kernel 4.19, we reverted the following two commits that were introduced during the 4.6 release cycle to fix the regression. The second had previously been reverted by Linus:</p>
|
||
<ul>
|
||
<li>Commit <a href="https://github.com/torvalds/linux/commit/06bd3c36a733ac27962fea7d6f47168841376824" target="_blank" data-tracking-control-name="_anchor_link">06bd3c36a733</a> ("ext4: fix data exposure after a crash")</li>
|
||
<li>Commit <a href="https://github.com/torvalds/linux/commit/1f60fbe7274918adb8db2f616e321890730ab7e3" target="_blank" data-tracking-control-name="_anchor_link">1f60fbe72749</a> ("ext4: allow readdir()'s of large empty directories to be interrupted")</li>
|
||
</ul>
|
||
<p>In kernel 5.4, we cherry-picked the following commit that was introduced during the 5.6 release cycle to fix the regression:</p>
|
||
<p>“ext4: make dioread_nolock the default” from commit <a href="https://github.com/torvalds/linux/commit/e5da4c933c50d98d7990a7c1ca0bbf8946e80c4a" target="_blank" data-tracking-control-name="_anchor_link">e5da4c933</a>, an ext4 merge</p>
|
||
<p></p>
|
||
<ul>
|
||
<li>Interestingly, setting the dioread_nolock boot param had no effect.</li>
|
||
<li>Our workload is buffered I/O; what should a change around dioread_nolock, a direct I/O knob, have to offer in this situation?</li>
|
||
</ul>
|
||
<p></p>
|
||
<p>The following graph shows approximate testing results as we iterated to converge on the 3.10 kernel’s performance:</p>
|
||
</div>
|
||
|
||
</div>
|
||
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
|
||
|
||
|
||
|
||
|
||
<div class="standalone-image-component standalone-image-component--article-width">
|
||
<figure>
|
||
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQFdoWZjQgnvRw/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700688307211?e=2147483647&v=beta&t=WZTI64m6s8-Lr55MW37Fj4ltgW3dJ6PCtVPM5lM0iug" alt="graph-showing-the-approximate-testing-results-of-kernel-iterations">
|
||
<!----> </figure>
|
||
</div>
|
||
|
||
</div>
|
||
<div id class="component component-migratedContent" data-component-type="migratedContent">
|
||
|
||
|
||
|
||
|
||
<div class="rich-text">
|
||
<p><i>Convergence of kernel 4.x to 3.10 fio benchmark completion time during development.</i></p>
|
||
<h2>Configuration matters as much as code</h2>
|
||
<p>We stumbled on configuration regressions several times in the back and forth with the kernel vendor.</p>
|
||
<p>One unexpected example was that in the back and forth, the scsi-mq layer had been disabled in the vendor’s kernel config in favor of the legacy block I/O layer. According to popular wisdom, scsi-mq was expected to perform worse on single-queue devices, due to the additional overhead of concurrently submitting I/O and the potential result of a globally suboptimal request ordering. We found the opposite in kernel 4.19: the old deadline I/O scheduler had worse tail latency under heavy I/O to a single-queue rotating HDD device than even the noop multi-queue scheduler.</p>
|
||
<p>Why is it that we found ourselves testing against the noop multi-queue scheduler? Again, due to a separate kernel build configuration error, the multiqueue schedulers were compiled as modules rather than compiled into the kernel as builtins. When compiled as modules, they were not available when the HDD block device was registered. As a result, the noop scheduler is applied instead of the scheduler that would be more appropriate for the device (in this case, mq-deadline):</p>
|
||
</div>
|
||
|
||
</div>
|
||
<div id class="component component-githubGist" data-component-type="githubGist">
|
||
|
||
|
||
|
||
|
||
<div class="github-gist" data-github-gist data-gist-src="https://gist.github.com/runderwo/c40b873bb9b4d2abbd47dc95c9bc5ff2.js" data-gist-title="Github Gist Embed"></div>
|
||
|
||
<!---->
|
||
|
||
</div>
|
||
<div id class="component component-migratedContent" data-component-type="migratedContent">
|
||
|
||
|
||
|
||
|
||
<div class="rich-text">
|
||
<h2>Learnings</h2>
|
||
<p>We learned quite a bit from this exercise and, most importantly, were able to complete the system image upgrade on our private cloud. This ultimately benefits our members by enabling our engineers to safely ship new products and features.</p>
|
||
<p>One thing that Linux users need to know is that <i>ext4 performance on kernel versions above 4.5 and below 5.6 suffers severely in the presence of concurrent sequential I/O on rotating disks</i>. NVMe devices are not affected, nor are other filesystems. <b>We highly recommend not using those kernels for ext4 on rotating storage media.</b> Since the kernel 4.x series is EOL as of 2020, the only realistic option for most people is to upgrade to kernel 5.6 or above.</p>
|
||
<p>Vendor configuration options can also play a role in bisecting. Always ensure that you’re starting from a clean upstream release when tackling an upstream bug, and be prepared to find secondary regressions in vendor-internal configs and in-house patches that mask the root cause.</p>
|
||
<p>For LinkedIn, we incorporated this experience into our golden image qualification process by making our suite of I/O stress tests more comprehensive. This incident also spurred us to explore investing in automation around lightweight filesystem reprovisioning without reimaging, i.e., so that we could convert an in-service host from one filesystem to another without rebooting by merely evacuating its applications and reformatting the partition where application data is stored.</p>
|
||
<h2>Acknowledgements</h2>
|
||
<p>In no particular order, these individuals each unblocked at least one key aspect of this investigation: <a href="https://www.linkedin.com/in/sergiyz" target="_blank" data-tracking-control-name="_anchor_link">Sergiy Zhuk</a>, <a href="https://www.linkedin.com/in/kyleareid/" target="_blank" data-tracking-control-name="_anchor_link">Kyle Reid</a>, <a href="https://www.linkedin.com/in/pradeepsanders/" target="_blank" data-tracking-control-name="_anchor_link">Pradeep Sanders</a>, <a href="https://www.linkedin.com/in/timcrofts/" target="_blank" data-tracking-control-name="_anchor_link">Tim Crofts</a>, <a href="https://www.linkedin.com/in/mislam/" target="_blank" data-tracking-control-name="_anchor_link">Mir Islam</a>, <a href="https://www.linkedin.com/in/adamdebus/" target="_blank" data-tracking-control-name="_anchor_link">Adam Debus</a>, <a href="https://www.linkedin.com/in/hengyang-hu-4934ba57/" target="_blank" data-tracking-control-name="_anchor_link">Hengyang Hu</a>, <a href="https://www.linkedin.com/in/chris-stufflebeam-66276694/" target="_blank" data-tracking-control-name="_anchor_link">Chris Stufflebeam</a>, <a href="https://www.linkedin.com/in/navoday-tomar-48a52719/" target="_blank" data-tracking-control-name="_anchor_link">Navoday Tomar</a>, and <a href="https://www.linkedin.com/in/sasha-levin-9861662/" target="_blank" data-tracking-control-name="_anchor_link">Sasha Levin</a> at Microsoft.</p>
|
||
</div>
|
||
|
||
</div>
|
||
<div id class="component component-postList" data-component-type="postList">
|
||
|
||
|
||
|
||
|
||
<section id="postList0FocusPoint" class="component__content-container list-layout" data-post-list>
|
||
<!---->
|
||
<hr class="artdeco-divider">
|
||
|
||
<section class="list-layout__headline-container">
|
||
|
||
|
||
<p class="list-layout__headline t-32 t-sans">Related articles</p>
|
||
|
||
|
||
|
||
<!---->
|
||
</section>
|
||
|
||
<!---->
|
||
<!----> </section>
|
||
|
||
|
||
</div>
|
||
|
||
|
||
<!----> </section>
|
||
</div>
|
||
|
||
<script src="https://static.licdn.com/aero-v1/sc/h/8hfbuq1ftcvnnx4dd5067pi0t" async></script>
|
||
<script src="https://static.licdn.com/aero-v1/sc/h/bknx7jzs0ylu540sn4wuo6bue" async></script>
|
||
<script src="https://static.licdn.com/aero-v1/sc/h/egqq1vs0784qd3h3801e2w9lr" async></script>
|
||
|
||
</main>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<footer class="li-footer bg-transparent w-full ">
|
||
<ul class="li-footer__list flex flex-wrap flex-row items-start justify-start w-full h-auto min-h-[50px] my-[0px] mx-auto py-3 px-2 papabear:p-0">
|
||
|
||
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
|
||
|
||
<span class="sr-only">LinkedIn</span>
|
||
<icon class="li-footer__copy-logo text-color-logo-brand-alt inline-block self-center h-[14px] w-[56px] mr-1" data-delayed-url="https://static.licdn.com/aero-v1/sc/h/5mebydpuuijm3uhv1q375inqh"></icon>
|
||
<span class="li-footer__copy-text flex items-center">© 2026</span>
|
||
|
||
</li>
|
||
|
||
|
||
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
|
||
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://about.linkedin.com?trk=content_footer-about" data-tracking-control-name="content_footer-about" data-tracking-will-navigate>
|
||
|
||
About
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
|
||
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/accessibility?trk=content_footer-accessibility" data-tracking-control-name="content_footer-accessibility" data-tracking-will-navigate>
|
||
|
||
Accessibility
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
|
||
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/user-agreement?trk=content_footer-user-agreement" data-tracking-control-name="content_footer-user-agreement" data-tracking-will-navigate>
|
||
|
||
User Agreement
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
|
||
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/privacy-policy?trk=content_footer-privacy-policy" data-tracking-control-name="content_footer-privacy-policy" data-tracking-will-navigate>
|
||
|
||
Privacy Policy
|
||
|
||
</a>
|
||
</li>
|
||
|
||
<!---->
|
||
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
|
||
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/cookie-policy?trk=content_footer-cookie-policy" data-tracking-control-name="content_footer-cookie-policy" data-tracking-will-navigate>
|
||
|
||
Cookie Policy
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
|
||
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/copyright-policy?trk=content_footer-copyright-policy" data-tracking-control-name="content_footer-copyright-policy" data-tracking-will-navigate>
|
||
|
||
Copyright Policy
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
|
||
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://brand.linkedin.com/policies?trk=content_footer-brand-policy" data-tracking-control-name="content_footer-brand-policy" data-tracking-will-navigate>
|
||
|
||
Brand Policy
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
|
||
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/psettings/guest-controls?trk=content_footer-guest-controls" data-tracking-control-name="content_footer-guest-controls" data-tracking-will-navigate>
|
||
|
||
Guest Controls
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
|
||
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/help/linkedin/answer/34593?trk=content_footer-community-guide" data-tracking-control-name="content_footer-community-guide" data-tracking-will-navigate>
|
||
|
||
Community Guidelines
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
<!---->
|
||
|
||
|
||
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="collapsible-dropdown collapsible-dropdown--footer collapsible-dropdown--up flex items-center relative hyphens-auto language-selector z-2">
|
||
<!---->
|
||
<ul class="collapsible-dropdown__list hidden container-raised absolute w-auto overflow-y-auto flex-col items-stretch z-[9999] bottom-[100%] top-auto" role="menu" tabindex="-1">
|
||
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="العربية (Arabic)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-ar_AE" data-locale="ar_AE" role="menuitem" lang="ar_AE">
|
||
العربية (Arabic)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="বাংলা (Bangla)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-bn_IN" data-locale="bn_IN" role="menuitem" lang="bn_IN">
|
||
বাংলা (Bangla)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Čeština (Czech)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-cs_CZ" data-locale="cs_CZ" role="menuitem" lang="cs_CZ">
|
||
Čeština (Czech)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Dansk (Danish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-da_DK" data-locale="da_DK" role="menuitem" lang="da_DK">
|
||
Dansk (Danish)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Deutsch (German)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-de_DE" data-locale="de_DE" role="menuitem" lang="de_DE">
|
||
Deutsch (German)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Ελληνικά (Greek)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-el_GR" data-locale="el_GR" role="menuitem" lang="el_GR">
|
||
Ελληνικά (Greek)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="English (English) selected" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link--selected" data-tracking-control-name="language-selector-en_US" data-locale="en_US" role="menuitem" lang="en_US">
|
||
<strong>English (English)</strong>
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Español (Spanish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-es_ES" data-locale="es_ES" role="menuitem" lang="es_ES">
|
||
Español (Spanish)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="فارسی (Persian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-fa_IR" data-locale="fa_IR" role="menuitem" lang="fa_IR">
|
||
فارسی (Persian)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Suomi (Finnish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-fi_FI" data-locale="fi_FI" role="menuitem" lang="fi_FI">
|
||
Suomi (Finnish)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Français (French)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-fr_FR" data-locale="fr_FR" role="menuitem" lang="fr_FR">
|
||
Français (French)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="हिंदी (Hindi)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-hi_IN" data-locale="hi_IN" role="menuitem" lang="hi_IN">
|
||
हिंदी (Hindi)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Magyar (Hungarian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-hu_HU" data-locale="hu_HU" role="menuitem" lang="hu_HU">
|
||
Magyar (Hungarian)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Bahasa Indonesia (Indonesian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-in_ID" data-locale="in_ID" role="menuitem" lang="in_ID">
|
||
Bahasa Indonesia (Indonesian)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Italiano (Italian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-it_IT" data-locale="it_IT" role="menuitem" lang="it_IT">
|
||
Italiano (Italian)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="עברית (Hebrew)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-iw_IL" data-locale="iw_IL" role="menuitem" lang="iw_IL">
|
||
עברית (Hebrew)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="日本語 (Japanese)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-ja_JP" data-locale="ja_JP" role="menuitem" lang="ja_JP">
|
||
日本語 (Japanese)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="한국어 (Korean)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-ko_KR" data-locale="ko_KR" role="menuitem" lang="ko_KR">
|
||
한국어 (Korean)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="मराठी (Marathi)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-mr_IN" data-locale="mr_IN" role="menuitem" lang="mr_IN">
|
||
मराठी (Marathi)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Bahasa Malaysia (Malay)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-ms_MY" data-locale="ms_MY" role="menuitem" lang="ms_MY">
|
||
Bahasa Malaysia (Malay)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Nederlands (Dutch)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-nl_NL" data-locale="nl_NL" role="menuitem" lang="nl_NL">
|
||
Nederlands (Dutch)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Norsk (Norwegian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-no_NO" data-locale="no_NO" role="menuitem" lang="no_NO">
|
||
Norsk (Norwegian)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="ਪੰਜਾਬੀ (Punjabi)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-pa_IN" data-locale="pa_IN" role="menuitem" lang="pa_IN">
|
||
ਪੰਜਾਬੀ (Punjabi)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Polski (Polish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-pl_PL" data-locale="pl_PL" role="menuitem" lang="pl_PL">
|
||
Polski (Polish)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Português (Portuguese)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-pt_BR" data-locale="pt_BR" role="menuitem" lang="pt_BR">
|
||
Português (Portuguese)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Română (Romanian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-ro_RO" data-locale="ro_RO" role="menuitem" lang="ro_RO">
|
||
Română (Romanian)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Русский (Russian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-ru_RU" data-locale="ru_RU" role="menuitem" lang="ru_RU">
|
||
Русский (Russian)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Svenska (Swedish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-sv_SE" data-locale="sv_SE" role="menuitem" lang="sv_SE">
|
||
Svenska (Swedish)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="తెలుగు (Telugu)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-te_IN" data-locale="te_IN" role="menuitem" lang="te_IN">
|
||
తెలుగు (Telugu)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="ภาษาไทย (Thai)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-th_TH" data-locale="th_TH" role="menuitem" lang="th_TH">
|
||
ภาษาไทย (Thai)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Tagalog (Tagalog)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-tl_PH" data-locale="tl_PH" role="menuitem" lang="tl_PH">
|
||
Tagalog (Tagalog)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Türkçe (Turkish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-tr_TR" data-locale="tr_TR" role="menuitem" lang="tr_TR">
|
||
Türkçe (Turkish)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Українська (Ukrainian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-uk_UA" data-locale="uk_UA" role="menuitem" lang="uk_UA">
|
||
Українська (Ukrainian)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="Tiếng Việt (Vietnamese)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-vi_VN" data-locale="vi_VN" role="menuitem" lang="vi_VN">
|
||
Tiếng Việt (Vietnamese)
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="简体中文 (Chinese (Simplified))" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-zh_CN" data-locale="zh_CN" role="menuitem" lang="zh_CN">
|
||
简体中文 (Chinese (Simplified))
|
||
</button>
|
||
</li>
|
||
<li class="language-selector__item" role="presentation">
|
||
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
|
||
<button aria-label="正體中文 (Chinese (Traditional))" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
|
||
language-selector__link !font-regular" data-tracking-control-name="language-selector-zh_TW" data-locale="zh_TW" role="menuitem" lang="zh_TW">
|
||
正體中文 (Chinese (Traditional))
|
||
</button>
|
||
</li>
|
||
<!---->
|
||
</ul>
|
||
|
||
|
||
<button class="language-selector__button select-none relative pr-2 font-sans text-xs font-bold text-color-text-low-emphasis hover:text-color-link-hover hover:cursor-pointer focus:text-color-link-focus focus:outline-dotted focus:outline-1" aria-expanded="false" data-tracking-control-name="footer-lang-dropdown_trigger">
|
||
<span class="language-selector__label-text mr-0.5 break-words">
|
||
Language
|
||
</span>
|
||
<icon class="language-selector__label-chevron w-2 h-2 absolute top-0 right-0" data-delayed-url="https://static.licdn.com/aero-v1/sc/h/cyolgscd0imw2ldqppkrb84vo"></icon>
|
||
</button>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
<!----> </footer>
|
||
|
||
|
||
|
||
<script src="https://static.licdn.com/aero-v1/sc/h/e69gtg8c61dzqklxridy0htl9" async></script>
|
||
<!---->
|
||
|
||
<!----><!---->
|
||
</body>
|
||
</html>
|
||
|
||
|
||
|