Files
nexus/sreweekly/articles/235/06-production-testing-with-dark-canaries.html
2026-09-12 17:23:01 +08:00

1557 lines
94 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en">
<head>
<meta name="pageKey" content="d_lithograph_content">
<!----><!----><!----> <meta name="locale" content="en_US">
<!----> <meta id="config" data-app-id="lithograph" data-app-version="0.0.2138" data-call-tree-id="AAZbPpo1F/xN8fsSFzNq3Q==" data-multiproduct-name="lithograph-publish-frontend" data-service-name="lithograph-publish-frontend" data-browser-id="78ab2b8c-d48b-4647-8ee9-6a2bfc3100cf" data-is-bot="false" data-page-instance="urn:li:page:d_lithograph_content;Hvmrw6Q0STuEScKnh7C/VA==" data-disable-jsbeacon-pagekey-suffix="false" data-dna-member-lix-treatment="control" data-human-member-lix-treatment="control" data-dfp-member-lix-treatment="control" data-sync-apfc-headers-lix-treatment="control" data-sync-apfc-cb-lix-treatment="control" data-recaptcha-v3-integration-lix-value="control" data-network-interceptor-lix-value="control" data-is-epd-audit-event-enabled="false" data-is-feed-sponsored-tracking-kill-switch-enabled="false" data-sequence-auto-redirect-before-request-enabled="true">
<link rel="canonical" href="https://www.linkedin.com/blog/engineering/infrastructure/production-testing-with-dark-canaries">
<!----><!---->
<!---->
<!---->
<!---->
<!---->
<link rel="icon" href="https://static.licdn.com/aero-v1/sc/h/al2o9zrvru7aqj8e1x2rzsrca">
<script>
function getDfd() {let yFn,nFn;const p=new Promise(function(y, n){yFn=y;nFn=n;});p.resolve=yFn;p.reject=nFn;return p;}
window.lazyloader = getDfd();
window.tracking = getDfd();
window.impressionTracking = getDfd();
window.ingraphTracking = getDfd();
window.appDetection = getDfd();
window.pemTracking = getDfd();
window.appRedirectCompleted = getDfd();
</script>
<!---->
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Production testing with dark canaries</title>
<!---->
<meta property="og:type" content="website">
<meta property="og:title" content="Production testing with dark canaries">
<!---->
<meta property="og:url" content="https://www.linkedin.com/blog/engineering/infrastructure/production-testing-with-dark-canaries">
<meta property="og:image" content="https://media.licdn.com/dms/image/v2/D4D08AQG1FC7Xg2p10Q/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700688304467?e=2147483647&amp;v=beta&amp;t=f-FtYT7sniMePoLLbhBQu4yFpGs86fQJDoYmQsQBUJ8">
<meta name="twitter:title" content="Production testing with dark canaries">
<meta name="twitter:site" content="@linkedin">
<!---->
<!---->
<meta name="robots" content="noarchive">
<meta name="linkedin:pageTag" content="/blog/engineering/infrastructure/production-testing-with-dark-canaries">
<meta name="litmsProfileName" content="lithograph">
<!----> <meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="asset-url" id="artdeco/static/images/icons.svg" content="https://static.licdn.com/aero-v1/sc/h/6sz1e821vgereyxaakx87tt4z ">
<link rel="stylesheet" href="https://static.licdn.com/aero-v1/sc/h/lbz6744ddd0wsc02ey2a3dhx">
<!---->
<!----> </head>
<body dir="ltr">
<!---->
<a href="#lithograph-app" class="skip-link btn-md btn-primary absolute z-11 -top-[100vh] focus:top-0">
Skip to main content
</a>
<!---->
<div class="page-header__wrapper page-header__wrapper--margin" data-custom-nav-wrapper="true">
<div class="page-header__container">
<header id="page-header" class="page-header page-header--hide-dropshadow">
<div class="page-header__content-container">
<div class="header__logo">
<a href="https://www.linkedin.com/blog/engineering" class="custom-nav__logo-link">
<img class="header__linkedin-logo header__full-logo" src="https://static.licdn.com/aero-v1/sc/h/6zm111mce7vohqze950ilreo4" alt="LinkedIn Logo">
<img class="header__linkedin-logo header__in-bug" src="https://static.licdn.com/aero-v1/sc/h/5g0hjlcng3j5pgn50n2et1ca2" alt="LinkedIn Logo">
<span class="t-17 t-black t-bold default-header-title">
Engineering Blog
</span>
</a>
</div>
<!---->
<!---->
<button class="stripped header__expand-nav-button" data-mobile-header-navigation-button data-header-expand-navigation-button aria-expanded="false">
<li-icon class="header__expand-nav-icon" a11y-text="Open navigation" type="hamburger-icon"></li-icon>
</button>
<button class="stripped header__expand-nav-button hidden" data-mobile-header-navigation-button data-header-collapse-navigation-button aria-expanded="false">
<li-icon class="header__expand-nav-icon" a11y-text="Close navigation" type="cancel-icon"></li-icon>
</button>
</div>
</header>
</div>
<nav class="header-nav fixed" aria-label="main" data-header-navigation>
<ul class="header-nav__list">
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/data" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Data
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Data" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Graph System</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/economic-graph" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Economic Graph</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/skills-graph" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Skills Graph</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/knowledge" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Knowledge Graph</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/data-management" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Management</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/data-streaming-processing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Streams/Processing</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/data-science" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Science</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/research" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Research</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/ab-testing-experimentation" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">A/B Testing/Experimentation</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/artificial-intelligence" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
AI
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for AI" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/machine-learning" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Machine Learning</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/natural-language-processing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Natural Language Processing</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/generative-ai" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Generative AI</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/data-modeling" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Data Modeling</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/responsible-ai" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Responsible AI</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/trust-and-safety" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Trust & Safety
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Trust &amp;amp; Safety" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/fairness" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Fairness</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/verification" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Verification </a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/security" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Security</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/product-design" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Product Design
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Product Design" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/marketing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Marketing </a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/sales" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Sales</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/learning" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Learning</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/hiring" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Hiring</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Profile</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/messaging-notifications" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Messaging/Notifications</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/feed" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Feed</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Profile</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/groups" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Groups</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/accessibility" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Accessibility</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/member-customer-experience" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Member/Customer Experience</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/search" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Search</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/recommendations" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Recommendations</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/skills" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Skills</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/infrastructure" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Infrastructure
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Infrastructure" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/code" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Code</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/scalability" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Scalability</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/distributed-systems" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Distributed Systems</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Graph Systems</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/architecture" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Architecture</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/analytics" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Analytics</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/automation" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Automation</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/platform-platformization" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Platforms/Platformization</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/cloud-computing" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Cloud Computing</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/open-source" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Open Source</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/additional-innovations" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Additional Innovations</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
<li>
<div class="artdeco-dropdown">
<div class="artdeco-dropdown__trigger">
<a href="https://www.linkedin.com/blog/engineering/talent" class="t-sans t-black--light t-semibold header-nav__toplink" target="_self">
Talent
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Talent" aria-expanded="false" data-trigger-dropdown-menu type="button">
<li-icon size="small" type="caret-filled-down-icon"></li-icon>
</button>
<div class="artdeco-dropdown__content header-nav__dropdown-offset">
<ul>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/culture" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Culture</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/developer-experience-productivity" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Developer Experience/Productivity</a>
</div>
</li>
<li>
<div class="artdeco-dropdown__item">
<a href="https://www.linkedin.com/blog/engineering/leadership" class="t-14 t-sans t-black t-bold header-nav__link" target="_self">Leadership</a>
</div>
</li>
</ul>
</div>
</div>
</div>
</li>
</ul>
</nav>
<nav class="custom-header__nav--mobile invisible" data-mobile-navigation="true" aria-label="main">
<ul class="custom-header__list--mobile">
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/data" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Data
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Data" aria-expanded="false" data-label-expand="Expand to show more links for Data" data-label-collapse="Collapse to show less links for Data" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-sans t-black t-normal header-nav__link" target="_self">Graph System</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/economic-graph" class="t-sans t-black t-normal header-nav__link" target="_self">Economic Graph</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/skills-graph" class="t-sans t-black t-normal header-nav__link" target="_self">Skills Graph</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/knowledge" class="t-sans t-black t-normal header-nav__link" target="_self">Knowledge Graph</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/data-management" class="t-sans t-black t-normal header-nav__link" target="_self">Data Management</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/data-streaming-processing" class="t-sans t-black t-normal header-nav__link" target="_self">Data Streams/Processing</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/data-science" class="t-sans t-black t-normal header-nav__link" target="_self">Data Science</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/research" class="t-sans t-black t-normal header-nav__link" target="_self">Research</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/ab-testing-experimentation" class="t-sans t-black t-normal header-nav__link" target="_self">A/B Testing/Experimentation</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/artificial-intelligence" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
AI
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for AI" aria-expanded="false" data-label-expand="Expand to show more links for AI" data-label-collapse="Collapse to show less links for AI" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/machine-learning" class="t-sans t-black t-normal header-nav__link" target="_self">Machine Learning</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/natural-language-processing" class="t-sans t-black t-normal header-nav__link" target="_self">Natural Language Processing</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/generative-ai" class="t-sans t-black t-normal header-nav__link" target="_self">Generative AI</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/data-modeling" class="t-sans t-black t-normal header-nav__link" target="_self">Data Modeling</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/responsible-ai" class="t-sans t-black t-normal header-nav__link" target="_self">Responsible AI</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/trust-and-safety" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Trust &amp; Safety
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Trust &amp;amp; Safety" aria-expanded="false" data-label-expand="Expand to show more links for Trust &amp;amp; Safety" data-label-collapse="Collapse to show less links for Trust &amp;amp; Safety" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/fairness" class="t-sans t-black t-normal header-nav__link" target="_self">Fairness</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/verification" class="t-sans t-black t-normal header-nav__link" target="_self">Verification </a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/security" class="t-sans t-black t-normal header-nav__link" target="_self">Security</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/product-design" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Product Design
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Product Design" aria-expanded="false" data-label-expand="Expand to show more links for Product Design" data-label-collapse="Collapse to show less links for Product Design" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/marketing" class="t-sans t-black t-normal header-nav__link" target="_self">Marketing </a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/sales" class="t-sans t-black t-normal header-nav__link" target="_self">Sales</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/learning" class="t-sans t-black t-normal header-nav__link" target="_self">Learning</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/hiring" class="t-sans t-black t-normal header-nav__link" target="_self">Hiring</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-sans t-black t-normal header-nav__link" target="_self">Profile</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/messaging-notifications" class="t-sans t-black t-normal header-nav__link" target="_self">Messaging/Notifications</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/feed" class="t-sans t-black t-normal header-nav__link" target="_self">Feed</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/profile" class="t-sans t-black t-normal header-nav__link" target="_self">Profile</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/groups" class="t-sans t-black t-normal header-nav__link" target="_self">Groups</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/accessibility" class="t-sans t-black t-normal header-nav__link" target="_self">Accessibility</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/member-customer-experience" class="t-sans t-black t-normal header-nav__link" target="_self">Member/Customer Experience</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/search" class="t-sans t-black t-normal header-nav__link" target="_self">Search</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/recommendations" class="t-sans t-black t-normal header-nav__link" target="_self">Recommendations</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/skills" class="t-sans t-black t-normal header-nav__link" target="_self">Skills</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/infrastructure" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Infrastructure
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Infrastructure" aria-expanded="false" data-label-expand="Expand to show more links for Infrastructure" data-label-collapse="Collapse to show less links for Infrastructure" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/code" class="t-sans t-black t-normal header-nav__link" target="_self">Code</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/scalability" class="t-sans t-black t-normal header-nav__link" target="_self">Scalability</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/distributed-systems" class="t-sans t-black t-normal header-nav__link" target="_self">Distributed Systems</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/graph-systems" class="t-sans t-black t-normal header-nav__link" target="_self">Graph Systems</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/architecture" class="t-sans t-black t-normal header-nav__link" target="_self">Architecture</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/analytics" class="t-sans t-black t-normal header-nav__link" target="_self">Analytics</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/automation" class="t-sans t-black t-normal header-nav__link" target="_self">Automation</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/platform-platformization" class="t-sans t-black t-normal header-nav__link" target="_self">Platforms/Platformization</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/cloud-computing" class="t-sans t-black t-normal header-nav__link" target="_self">Cloud Computing</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/open-source" class="t-sans t-black t-normal header-nav__link" target="_self">Open Source</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/additional-innovations" class="t-sans t-black t-normal header-nav__link" target="_self">Additional Innovations</a>
</li>
</ul>
</li>
<li class="custom-header__list-item--mobile" data-mobile-navigation-list-item="true">
<a href="https://www.linkedin.com/blog/engineering/talent" class="t-16 t-sans t-black t-semibold header-nav__toplink" target="_self">
Talent
</a>
<button class="header-navigation__button" aria-label="Expand to show more links for Talent" aria-expanded="false" data-label-expand="Expand to show more links for Talent" data-label-collapse="Collapse to show less links for Talent" data-mobile-dropdown-trigger type="button">
<li-icon size="small" type="chevron-down-icon"></li-icon>
</button>
<ul class="custom-header__dropdownlink-list--mobile hidden" data-dropdownlinks-list="true">
<li>
<a href="https://www.linkedin.com/blog/engineering/culture" class="t-sans t-black t-normal header-nav__link" target="_self">Culture</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/developer-experience-productivity" class="t-sans t-black t-normal header-nav__link" target="_self">Developer Experience/Productivity</a>
</li>
<li>
<a href="https://www.linkedin.com/blog/engineering/leadership" class="t-sans t-black t-normal header-nav__link" target="_self">Leadership</a>
</li>
</ul>
</li>
</ul>
<div class="custom-header__button-share-container--mobile">
<!---->
</div>
</nav>
</div>
<main role="main">
<div id="lithograph-app">
<section id="component-container" data-attach-collapse-behavior="true">
<div id class="component component-articleHeadline" data-component-type="articleHeadline">
<div class="article-headline-container">
<p class="article-headline__topic">
<a class="t-14" href="https://www.linkedin.com/blog/engineering/infrastructure">Infrastructure</a>
</p>
<h1 class="article-headline t-40" data-article-title="Production testing with dark canaries">Production testing with dark canaries</h1>
<div class="article-meta t-14 article-meta__hairline">
<div class="article-meta__byline-container">
<div class="author-profile__author-container">
<img class="author-profile__author-image" srcset="https://media.licdn.com/dms/image/v2/C4D03AQGCrOuwkc_yqQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1515626518922?e=2147483647&amp;v=beta&amp;t=vlfjaNR80Id2hos7PUvIQ4E3B9R_Y7V8PMdszG0MAsk 100w,https://media.licdn.com/dms/image/v2/C4D03AQGCrOuwkc_yqQ/profile-displayphoto-shrink_200_200/profile-displayphoto-shrink_200_200/0/1515626518922?e=2147483647&amp;v=beta&amp;t=QkRrVkjfG2-qjjRE8sUI3SrZExG6EZBj5l-Wr3azYkE 115w,https://media.licdn.com/dms/image/v2/C4D03AQGCrOuwkc_yqQ/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1515626518843?e=2147483647&amp;v=beta&amp;t=aFFtWdJxezpDJi1pu4AX6AMgPTaz0gIMLsDiJ6Tfb6E 115w,https://media.licdn.com/dms/image/v2/C4D03AQGCrOuwkc_yqQ/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1515626518848?e=2147483647&amp;v=beta&amp;t=xWh5yHNhKbEgBQRlPZeezNvfAn7uGTHd3H5M8KWLD5k 115w," sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" src="https://media.licdn.com/dms/image/v2/C4D03AQGCrOuwkc_yqQ/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1515626518848?e=2147483647&amp;v=beta&amp;t=xWh5yHNhKbEgBQRlPZeezNvfAn7uGTHd3H5M8KWLD5k" alt="David Hoa">
<div class="author-profile__author-text-container">
<a href="https://www.linkedin.com/in/davidhoa" target="_blank" class="t-14 t-bold"><span class="screen-reader">Authored by</span>David Hoa</a>
<p class="t-14">Staff Software Engineer at Databricks</p>
<p class="t-14 t-black--light" data-published-date="2020-9-10">
September 10, 2020
</p>
</div>
</div>
<!---->
</div>
<div class="social-share">
<ul class="social-share__list share-list" title="Share via" data-share-list>
<li class="share-list__item share-item">
<a href="#" class="share-item__link medium-round-muted-tertiary-button" role="button" data-tracking-control-name="_linkedin_share" data-share-type="linkedin" aria-label="Share on LinkedIn, New window will open">
<li-icon class="share-item__icon" type="linkedin-icon"></li-icon>
</a>
</li>
<li class="share-list__item share-item">
<a href="#" class="share-item__link medium-round-muted-tertiary-button" role="button" data-tracking-control-name="_facebook_share" data-share-type="facebook" aria-label="Share on Facebook, New window will open">
<li-icon class="share-item__icon" type="facebook-icon"></li-icon>
</a>
</li>
<li class="share-list__item share-item">
<a href="#" class="share-item__link medium-round-muted-tertiary-button" role="button" data-tracking-control-name="_twitter_share" data-share-type="twitter" aria-label="Share on X, New window will open">
<svg class="share-item__icon share-item__icon--twitter" viewbox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" fill="currentColor" focusable="false" aria-hidden="true" role="none">
<path d="M13.9 10.47L21.35 2h-1.76l-6.46 7.35L7.96 2H2l7.81 11.12L2 22h1.76l6.83-7.77L16.04 22H22l-8.1-11.53zm-2.42 2.75l-.79-1.11L4.4 3.3h2.71l5.08 7.11.79 1.11 6.6 9.24h-2.71l-5.39-7.54z"></path>
</svg>
</a>
</li>
</ul>
</div>
</div>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
The internet software industry has moved away from long development cycles and dedicated quality assurance (QA) stages, toward a fast-paced continuous-integration/continuous-delivery (CI/CD) pipeline, where new code is quickly written, committed, and pushed to user-facing applications and services. Doing so has dramatically increased iteration speeds, and at LinkedIn, it is not uncommon for new code to be committed and pushed to production within an hour, multiple times a day. While this has been incredibly successful—it has improved developer productivity by allowing us to make changes more quickly—it has also caused problems such as site or service outages when bad code, configuration, or AI models have been pushed to production. Even with complete unit and integration tests, sometimes there is no substitute for production-level testing using both production data and production volume, especially with regards to system performance metrics, such as memory consumption, CPU/GPU usage, concurrency, and latency.
But to do production-level testing safely without negatively impacting customers, teams often need both production requests and production data. Frameworks such as record-and-replay offer a solution, but oftentimes are not suitable for quickly changing scenarios, such as testing new AI models, which require different input features to produce an output. Gathering new input at production volume requires a potentially expensive and time-consuming step to re-record production requests and responses that is infeasible with multiple new models being A/B tested daily. This blog post introduces dark canary clusters as a way to detect problems before they hit production.
What is a dark canary?
The canary terminology comes from mining, where miners would lower an unfortunate canary in a cage to detect the presence of dangerous gases poisonous for miners to breathe. These canaries would serve as an early warning signal to the miners that it wasn’t safe to go into the mines. In software engineering, a canary is an instance of a service that runs new code, configuration, or AI models at production level so that we can verify that the new code is safe before deploying it to more instances. If the new code is bad (e.g., has a performance degradation or increased error rate), requests to that canary are potentially “sacrificed” until the bad code is rolled back. This may mean a small fraction of user requests will fail. A “dark” canary is an instance of a service that takes duplicated traffic from a real service instance, but where the response from the dark canary is discarded by default. This means the end user is never impacted even if something goes wrong in the dark canary, such as errors, higher latency, higher CPU/memory consumption, etc. Only read requests without side effects, such as tracking events that might pollute business metrics, should be duplicated. For instance, if there is a request to read a profile, a side effect would be to write to a database counting the number of profile reads. You could mitigate this side effect by detecting that this was a dark canary request and using a separate database counter, or not writing to the database in this case. Calls to downstream services also need to be without side effects or else their impact needs to be mitigated. Other downstream effects also need to be carefully considered before using dark canaries. If a request to a service results in hundreds of requests to downstream services, then there might be a non-negligible monetary cost for these “extra and non-essential” requests because it might require more server capacity across all those downstream services.
History of dark canaries at LinkedIn
Back in 2013, one of our large backend services wanted support in Rest.li for dark canaries. The service, at the time, involved duplicating requests from one host machine and sending it to another host machine. This was added via a Python tool to populate the host-to-host mapping in Apache ZooKeeper along with a filter to read this mapping and multiply traffic. As operational complexity grew (due to additional data centers, dark canaries being used in midtier and even frontend services, and dynamic scale up-down of instances), this became more complex to maintain. More teams were using dark canaries, but our developers and SREs were still hindered by how difficult it was to onboard and maintain dark canaries. For example, when dark canaries suddenly stopped receiving traffic or disappeared because hosts were swapped out from underneath them, engineers had to recreate the tedious host-to-host mapping in every data center. It was clear we needed a new solution.
Introducing dark canary clusters
A dark canary cluster is just like a dark canary except that there are multiple dark canary instances. Sending traffic to dark canary clusters requires the user to discover and spread traffic among the members of the cluster. LinkedIn uses the Dynamic Discovery (D2) service discovery mechanism in Rest.li to send requests from service to service, so this was the logical place to add first-class support for dark clusters because D2 already had a way to send requests to clusters. This integration enabled an entire cluster of regular service instances to send duplicated-but-ignored requests to a dark cluster of instances. The benefits of doing this were that:
We could easily send a substantial level of traffic to dark canaries while minimizing any impact on sending instances because the additional QPS on any instance would be low.
LinkedIn developers were familiar with the D2 service discovery mechanism and had a clear supported path for reviewing code changes, and promoting changes across staging areas and data centers.
The forking mechanism (where we duplicate the request, send it to the dark service, and ignore the response) could be aware of both the source and target cluster sizes so that we could maintain comparable levels of traffic between regular service instances and dark instances, such as with EKG.
Our teams at LinkedIn have found that dark canary clusters are an easy way to validate changes, especially for testing out new online AI models, where the model’s system performance at production level traffic is unknown. Common across all use cases is that this helps keep the LinkedIn site more stable and developers more productive by freeing our engineers from fearing new changes.
Architecture
In a typical service-oriented architecture, services will call other services to fulfill the user request. In the below diagram, Service A has multiple instances, takes inbound requests, and calls other services (including Service B) to retrieve necessary information. This is a typical layered system in which Service A is a mid-tier service and Service B is a back-end service. If Service A is the service that we want to validate, then we can set up a cluster of Service A dark canaries.
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQERvX7iyu1N6Q/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700688304731?e=2147483647&amp;v=beta&amp;t=Kf69B4SEGmCWnBsYy4tfHRdMyQ1BrwJVg6w2PkKQ6Tk" alt="dark-cluster-architecture-diagram">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p><i>Dark cluster architecture diagram<br> </i>&nbsp;</p>
<p>While there are many ways to dispatch traffic to the Service A dark canary cluster, LinkedIn uses a client-side library that is able to discover the dark cluster(s) that Service A should fork traffic to.&nbsp;</p>
<p>In this case, we store a self-updating mapping from a source cluster (“ServiceACluster”) to a set of corresponding Dark Clusters ([“DarkServiceACluster”]) in Apache ZooKeeper. In addition, we can store other metadata such as how to multiply traffic to the Dark Cluster and other configurations we need for forking traffic. Now, whenever Service A receives a request, an inbound request filter will read this ZooKeeper data, detect that it needs to send dark traffic to DarkServiceACluster, and fork the request appropriately.</p>
</div>
</div>
<div id class="component component-githubGist" data-component-type="githubGist">
<div class="github-gist" data-github-gist data-gist-src="https://gist.github.com/davidhoa/9fa01d4a3a092636c21b9c11dd8d83dd.js" data-gist-title="Github Gist Embed"></div>
<!---->
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p><i>Sample D2 Configuration, simplified for brevity. Configs can be structured in XML/JSON/etc.<br> &nbsp;</i></p>
<p>Using Apache ZooKeeper in this case is not strictly necessary, but it does help because it allows us to know the exact count of both regular clusters and dark clusters. This enables us to scale up and down both regular and dark clusters independently with the dispatching instances adjusted automatically. One strategy for sending dark traffic is to make the queries per second (QPS) between a dark cluster instance and a regular cluster instance identical (or proportional according to some multiplier), so they can be compared by validation tooling. If we add instances to DarkServiceACluster, then we would expect the total outbound dark requests from ServiceACluster to increase, but the QPS on each dark cluster instance to stay the same. Conversely, if we added instances to ServiceACluster, then we’d expect the per-instance QPS to ServiceACluster to drop (as the total QPS remains the same and becomes spread over more instances). As a result, we’d want the per-instance DarkServiceACluster QPS to drop a corresponding amount.&nbsp;</p>
<p>To follow this strategy, let’s illustrate how a Service A instance can calculate the ratio of requests it should send.</p>
<p>First, a definition: <br> <b>Multiplier:</b> number used to proportionally scale the traffic a dark cluster instance receives compared to a regular cluster instance. Setting the multiplier as 1.0 means the dark cluster per-instance QPS should be the same as the regular per-instance QPS, while 1.2 means the dark cluster per-instance QPS should receive 20% more QPS than the regular per-instance QPS.</p>
<p>All QPS fields below are per-instance unless explicitly specified otherwise.</p>
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQGW1igHnAfYEw/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700688304547?e=2147483647&amp;v=beta&amp;t=PQoqpQgUGcy_YxCjibPRhC7Q3GQv8N7gDl_0mCLJxe8" alt="ratio-of-dark-requests-equals-number-of-dark-service-a-cluster-instances-divided-by-number-of-service-a-cluster-instances-multiplied-by-multiplier">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>And</p>
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQFwpKkR1hRpYQ/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700688304205?e=2147483647&amp;v=beta&amp;t=zD4MOtIlyxIR0ofC4vB_5SFm8NYFOqbAaB6b8C0PEnY" alt="outbound-dark-request-QPS-is-equal-to-ratio-of-dark-requests-multiplied-by-inbound-QPS">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>This means that if the inbound QPS is 100, the multiplier is 1, and there are 2 DarkServiceACluster instances and 100 regular ServiceACluster instances, then the outbound DarkRequest QPS per regular ServiceA instance is:</p>
</div>
</div>
<div id class="component component-standaloneImage" data-component-type="standaloneImage">
<div class="standalone-image-component standalone-image-component--article-width">
<figure>
<img class="standalone-image-component__image" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 1920px" data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQGBl0zgPawZpA/croft-frontend-shrinkToFit1024/croft-frontend-shrinkToFit1024/0/1700688304276?e=2147483647&amp;v=beta&amp;t=dOeDoHB43Ed1gWg53zn_ZH7TVAXcXD6S4Zxiq5En5YI" alt="two-QPS-equals-two-instances-divided-by-one-hundred-instances-multiplied-by-one-multiplied-by-one-hundred-QPS">
<!----> </figure>
</div>
</div>
<div id class="component component-migratedContent" data-component-type="migratedContent">
<div class="rich-text">
<p>With this formula, the traffic on the dark cluster instances will stay proportional to your regular service instances, and you don’t have to worry about overloading them just by scaling the respective clusters up or down.</p>
<h2>Ongoing work</h2>
<p>Many teams at LinkedIn are working to realize the full potential of dark canary clusters, including approaches to automate the onboarding and spinning up of dark canary clusters, orchestrate the testing of AI models on dark clusters, and build better validation metrics and test suites that weren’t practical earlier to run in production.</p>
<h2>Conclusion</h2>
<p>Dark canary clusters are something we believe many companies can benefit from, regardless of whether the cluster management is done using D2. Not only is both onboarding and maintenance of dark clusters easy with this approach, but the strategy to compare regular instances against dark cluster instances can also be used in automated validation, helping to speed up the development process by removing manual validation steps. By removing error-prone manual validation steps with automated validation, you remove the expertise required to correctly validate code (especially against graphs) and build the confidence needed to speed up development velocity. We hope that this can help others establish best practices for validation and deployment within their companies.</p>
<h2>Acknowledgements</h2>
<p>Many thanks to <a href="https://www.linkedin.com/in/ssheng" target="_blank" data-tracking-control-name="_anchor_link">Sean Sheng</a> and <a href="https://www.linkedin.com/in/chrisyzhang" target="_blank" data-tracking-control-name="_anchor_link">Chris Zhang</a> from the Service Infrastructure team for their development support and brainstorming, to <a href="https://www.linkedin.com/in/xkrogen" target="_blank" data-tracking-control-name="_anchor_link">Erik Krogen</a> and <a href="https://www.linkedin.com/in/sumitrangwala" target="_blank" data-tracking-control-name="_anchor_link">Sumit Rangwala</a> for reviewing this blog post, and to the brave first adopters: <a href="https://www.linkedin.com/in/srividyakrishnamurthy" target="_blank" data-tracking-control-name="_anchor_link">Srividya Krishnamurthy</a>, <a href="https://www.linkedin.com/in/ramonfierro" target="_blank" data-tracking-control-name="_anchor_link">Ramon Garcia</a>, <a href="https://www.linkedin.com/in/peterchng" target="_blank" data-tracking-control-name="_anchor_link">Peter Chng</a>, and <a href="https://www.linkedin.com/in/yafeiw" target="_blank" data-tracking-control-name="_anchor_link">Yafei Wang</a>. Thanks to the <a href="https://engineering.linkedin.com/blog/2019/01/scaling-machine-learning-productivity-at-linkedin" target="_blank" data-tracking-control-name="_anchor_link">Pro-ML</a> Leadership team for their continued support and investment in this cross discipline area: <a href="https://www.linkedin.com/in/eingong" target="_blank" data-tracking-control-name="_anchor_link">Eing Ong</a>, <a href="https://www.linkedin.com/in/joshuahartman" target="_blank" data-tracking-control-name="_anchor_link">Josh Hartman</a>, <a href="https://www.linkedin.com/in/joeldyoung" target="_blank" data-tracking-control-name="_anchor_link">Joel Young</a>, and <a href="https://www.linkedin.com/in/joshowalker" target="_blank" data-tracking-control-name="_anchor_link">Josh Walker</a>.</p>
</div>
</div>
<div id class="component component-postList" data-component-type="postList">
<section id="postList0FocusPoint" class="component__content-container list-layout" data-post-list>
<section class="simple-topic-list">
<p class="t-14 t-bold">Topics:
<a class="simple-topic-list__topic" href="https://www.linkedin.com/blog/engineering/developer-experience-productivity">
Developer Experience/Productivity
</a>
<a class="simple-topic-list__topic" href="https://www.linkedin.com/blog/engineering/artificial-intelligence">
Artificial intelligence
</a>
<a class="simple-topic-list__topic" href="https://www.linkedin.com/blog/engineering/ab-testing-experimentation">
A/B Testing/Experimentation
</a>
<a class="simple-topic-list__topic" href="https://www.linkedin.com/blog/engineering/automation">
Automation
</a>
<a class="simple-topic-list__topic" href="https://www.linkedin.com/blog/engineering/infrastructure">
Infrastructure
</a>
</p>
</section>
<hr class="artdeco-divider">
<section class="list-layout__headline-container">
<p class="list-layout__headline t-32 t-sans">Related articles</p>
<!---->
</section>
<ul class="list-layout__posts">
<li class="post-list__item list-post">
<div class="list-post__content-container">
<p class="list-post__content-container__topic">
<a class="t-14 t-bold" href="https://www.linkedin.com/blog/engineering/infrastructure">
Infrastructure
</a>
</p>
<div class="list-post__content-container__title">
<a class="list-post__link t-20 t-black" href="https://www.linkedin.com/blog/engineering/infrastructure/the-training-infrastructure-behind-ai-powered-job-search-eight-x-faster-multi-teacher-distillation">
The Training Infrastructure Behind AI-Powered Job Search: 8X F...
</a>
</div>
<div class="list-post__content-container__byline">
<p class="list-post__content-container__author t-14 t-bold t-black">
Jaideep Ray
</p>
<!---->
<p class="list-post__content-container__date t-14 t-black--light">
Aug 6, 2026
</p>
</div>
</div>
<div class="list-post__image-container">
<figure class="post__image-figure">
<img class="post__image" alt data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQEctZ-EpMCamw/croft-frontend-shrinkToFit600/B4DZ_Ti6GaGYAQ-/0/1785960571979?e=2147483647&amp;v=beta&amp;t=gpPNPoM7OkjDl_PzMT4Tub1yA6bII-BPfO1uqIYeHRI" data-ghost-url="https://static.licdn.com/aero-v1/sc/h/8pouyb1ly746xtru0x7p2axm0" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 480px">
</figure>
</div>
</li>
<li class="post-list__item list-post">
<div class="list-post__content-container">
<p class="list-post__content-container__topic">
<a class="t-14 t-bold" href="https://www.linkedin.com/blog/engineering/ai">
AI
</a>
</p>
<div class="list-post__content-container__title">
<a class="list-post__link t-20 t-black" href="https://www.linkedin.com/blog/engineering/ai/qa-agent-reimagining-software-quality-with-ai-driven-autonomous-testing">
Quality Assurance Agent: Reimagining Software Quality with AI-...
</a>
</div>
<div class="list-post__content-container__byline">
<p class="list-post__content-container__author t-14 t-bold t-black">
Asa Kusuma
</p>
<!---->
<p class="list-post__content-container__date t-14 t-black--light">
Jun 18, 2026
</p>
</div>
</div>
<div class="list-post__image-container">
<figure class="post__image-figure">
<img class="post__image" alt data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQFWxUw04ihjYQ/croft-frontend-shrinkToFit600/B4DZ7bHNPnGkAQ-/0/1781792560756?e=2147483647&amp;v=beta&amp;t=KiyIbEztZVaRRSzY53ezbvpjY4EYCksSDoMR7R8--Rk" data-ghost-url="https://static.licdn.com/aero-v1/sc/h/8pouyb1ly746xtru0x7p2axm0" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 480px">
</figure>
</div>
</li>
<li class="post-list__item list-post">
<div class="list-post__content-container">
<!---->
<div class="list-post__content-container__title">
<a class="list-post__link t-20 t-black" href="https://www.linkedin.com/blog/engineering/infrastructure/faster-than-light-optimizing-generative-recommender-training-efficiency-at-linkedin">
Faster than Light: Optimizing Generative Recommender Training ...
</a>
</div>
<div class="list-post__content-container__byline">
<p class="list-post__content-container__author t-14 t-bold t-black">
Yitong Zhou
</p>
<!----><!----><!----><!----><!---->
<p class="list-post__content-container__date t-14 t-black--light">
May 28, 2026
</p>
</div>
</div>
<div class="list-post__image-container">
<figure class="post__image-figure">
<img class="post__image" alt data-delayed-url="https://media.licdn.com/dms/image/v2/D4D08AQF8sldepd_9KQ/croft-frontend-shrinkToFit1200/B4DZ5v8sMMKsAg-/0/1779994641420?e=2147483647&amp;v=beta&amp;t=QTrTmYC_08QGkiHzNXCv7C_c-gfVEF4NdLnG4nqgugk" data-ghost-url="https://static.licdn.com/aero-v1/sc/h/8pouyb1ly746xtru0x7p2axm0" sizes="(max-width: 480px) 480px, (max-width: 767px) 767px, (max-width: 1024px) 1024px, 480px">
</figure>
</div>
</li>
</ul>
<!----> </section>
</div>
<!----> </section>
</div>
<script src="https://static.licdn.com/aero-v1/sc/h/8hfbuq1ftcvnnx4dd5067pi0t" async></script>
<script src="https://static.licdn.com/aero-v1/sc/h/bknx7jzs0ylu540sn4wuo6bue" async></script>
<script src="https://static.licdn.com/aero-v1/sc/h/egqq1vs0784qd3h3801e2w9lr" async></script>
</main>
<footer class="li-footer bg-transparent w-full ">
<ul class="li-footer__list flex flex-wrap flex-row items-start justify-start w-full h-auto min-h-[50px] my-[0px] mx-auto py-3 px-2 papabear:p-0">
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<span class="sr-only">LinkedIn</span>
<icon class="li-footer__copy-logo text-color-logo-brand-alt inline-block self-center h-[14px] w-[56px] mr-1" data-delayed-url="https://static.licdn.com/aero-v1/sc/h/5mebydpuuijm3uhv1q375inqh"></icon>
<span class="li-footer__copy-text flex items-center">&copy; 2026</span>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://about.linkedin.com?trk=content_footer-about" data-tracking-control-name="content_footer-about" data-tracking-will-navigate>
About
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/accessibility?trk=content_footer-accessibility" data-tracking-control-name="content_footer-accessibility" data-tracking-will-navigate>
Accessibility
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/user-agreement?trk=content_footer-user-agreement" data-tracking-control-name="content_footer-user-agreement" data-tracking-will-navigate>
User Agreement
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/privacy-policy?trk=content_footer-privacy-policy" data-tracking-control-name="content_footer-privacy-policy" data-tracking-will-navigate>
Privacy Policy
</a>
</li>
<!---->
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/cookie-policy?trk=content_footer-cookie-policy" data-tracking-control-name="content_footer-cookie-policy" data-tracking-will-navigate>
Cookie Policy
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/legal/copyright-policy?trk=content_footer-copyright-policy" data-tracking-control-name="content_footer-copyright-policy" data-tracking-will-navigate>
Copyright Policy
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://brand.linkedin.com/policies?trk=content_footer-brand-policy" data-tracking-control-name="content_footer-brand-policy" data-tracking-will-navigate>
Brand Policy
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/psettings/guest-controls?trk=content_footer-guest-controls" data-tracking-control-name="content_footer-guest-controls" data-tracking-will-navigate>
Guest Controls
</a>
</li>
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<a class="li-footer__item-link flex items-center font-sans text-xs font-bold text-color-text-solid-secondary hover:text-color-link-hover focus:text-color-link-focus" href="https://www.linkedin.com/help/linkedin/answer/34593?trk=content_footer-community-guide" data-tracking-control-name="content_footer-community-guide" data-tracking-will-navigate>
Community Guidelines
</a>
</li>
<!---->
<li class="li-footer__item font-sans text-xs text-color-text-solid-secondary flex flex-shrink-0 justify-start p-1 relative w-50% papabear:justify-center papabear:w-auto">
<div class="collapsible-dropdown collapsible-dropdown--footer collapsible-dropdown--up flex items-center relative hyphens-auto language-selector z-2">
<!---->
<ul class="collapsible-dropdown__list hidden container-raised absolute w-auto overflow-y-auto flex-col items-stretch z-[9999] bottom-[100%] top-auto" role="menu" tabindex="-1">
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="العربية (Arabic)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ar_AE" data-locale="ar_AE" role="menuitem" lang="ar_AE">
العربية (Arabic)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="বাংলা (Bangla)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-bn_IN" data-locale="bn_IN" role="menuitem" lang="bn_IN">
বাংলা (Bangla)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Čeština (Czech)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-cs_CZ" data-locale="cs_CZ" role="menuitem" lang="cs_CZ">
Čeština (Czech)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Dansk (Danish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-da_DK" data-locale="da_DK" role="menuitem" lang="da_DK">
Dansk (Danish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Deutsch (German)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-de_DE" data-locale="de_DE" role="menuitem" lang="de_DE">
Deutsch (German)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Ελληνικά (Greek)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-el_GR" data-locale="el_GR" role="menuitem" lang="el_GR">
Ελληνικά (Greek)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="English (English) selected" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link--selected" data-tracking-control-name="language-selector-en_US" data-locale="en_US" role="menuitem" lang="en_US">
<strong>English (English)</strong>
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Español (Spanish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-es_ES" data-locale="es_ES" role="menuitem" lang="es_ES">
Español (Spanish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="فارسی (Persian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-fa_IR" data-locale="fa_IR" role="menuitem" lang="fa_IR">
فارسی (Persian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Suomi (Finnish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-fi_FI" data-locale="fi_FI" role="menuitem" lang="fi_FI">
Suomi (Finnish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Français (French)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-fr_FR" data-locale="fr_FR" role="menuitem" lang="fr_FR">
Français (French)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="हिंदी (Hindi)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-hi_IN" data-locale="hi_IN" role="menuitem" lang="hi_IN">
हिंदी (Hindi)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Magyar (Hungarian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-hu_HU" data-locale="hu_HU" role="menuitem" lang="hu_HU">
Magyar (Hungarian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Bahasa Indonesia (Indonesian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-in_ID" data-locale="in_ID" role="menuitem" lang="in_ID">
Bahasa Indonesia (Indonesian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Italiano (Italian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-it_IT" data-locale="it_IT" role="menuitem" lang="it_IT">
Italiano (Italian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="עברית (Hebrew)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-iw_IL" data-locale="iw_IL" role="menuitem" lang="iw_IL">
עברית (Hebrew)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="日本語 (Japanese)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ja_JP" data-locale="ja_JP" role="menuitem" lang="ja_JP">
日本語 (Japanese)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="한국어 (Korean)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ko_KR" data-locale="ko_KR" role="menuitem" lang="ko_KR">
한국어 (Korean)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="मराठी (Marathi)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-mr_IN" data-locale="mr_IN" role="menuitem" lang="mr_IN">
मराठी (Marathi)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Bahasa Malaysia (Malay)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ms_MY" data-locale="ms_MY" role="menuitem" lang="ms_MY">
Bahasa Malaysia (Malay)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Nederlands (Dutch)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-nl_NL" data-locale="nl_NL" role="menuitem" lang="nl_NL">
Nederlands (Dutch)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Norsk (Norwegian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-no_NO" data-locale="no_NO" role="menuitem" lang="no_NO">
Norsk (Norwegian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="ਪੰਜਾਬੀ (Punjabi)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-pa_IN" data-locale="pa_IN" role="menuitem" lang="pa_IN">
ਪੰਜਾਬੀ (Punjabi)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Polski (Polish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-pl_PL" data-locale="pl_PL" role="menuitem" lang="pl_PL">
Polski (Polish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Português (Portuguese)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-pt_BR" data-locale="pt_BR" role="menuitem" lang="pt_BR">
Português (Portuguese)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Română (Romanian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ro_RO" data-locale="ro_RO" role="menuitem" lang="ro_RO">
Română (Romanian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Русский (Russian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-ru_RU" data-locale="ru_RU" role="menuitem" lang="ru_RU">
Русский (Russian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Svenska (Swedish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-sv_SE" data-locale="sv_SE" role="menuitem" lang="sv_SE">
Svenska (Swedish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="తెలుగు (Telugu)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-te_IN" data-locale="te_IN" role="menuitem" lang="te_IN">
తెలుగు (Telugu)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="ภาษาไทย (Thai)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-th_TH" data-locale="th_TH" role="menuitem" lang="th_TH">
ภาษาไทย (Thai)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Tagalog (Tagalog)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-tl_PH" data-locale="tl_PH" role="menuitem" lang="tl_PH">
Tagalog (Tagalog)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Türkçe (Turkish)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-tr_TR" data-locale="tr_TR" role="menuitem" lang="tr_TR">
Türkçe (Turkish)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Українська (Ukrainian)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-uk_UA" data-locale="uk_UA" role="menuitem" lang="uk_UA">
Українська (Ukrainian)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="Tiếng Việt (Vietnamese)" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-vi_VN" data-locale="vi_VN" role="menuitem" lang="vi_VN">
Tiếng Việt (Vietnamese)
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="简体中文 (Chinese (Simplified))" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-zh_CN" data-locale="zh_CN" role="menuitem" lang="zh_CN">
简体中文 (Chinese (Simplified))
</button>
</li>
<li class="language-selector__item" role="presentation">
<!-- Adding aria-label to both the li and the button because screen reader focus goes to button on desktop and li on mobile-->
<button aria-label="正體中文 (Chinese (Traditional))" class="font-sans text-xs link block py-[5px] px-2 w-full hover:cursor-pointer hover:bg-color-action hover:text-color-text-on-dark focus:bg-color-action focus:text-color-text-on-dark
language-selector__link !font-regular" data-tracking-control-name="language-selector-zh_TW" data-locale="zh_TW" role="menuitem" lang="zh_TW">
正體中文 (Chinese (Traditional))
</button>
</li>
<!---->
</ul>
<button class="language-selector__button select-none relative pr-2 font-sans text-xs font-bold text-color-text-low-emphasis hover:text-color-link-hover hover:cursor-pointer focus:text-color-link-focus focus:outline-dotted focus:outline-1" aria-expanded="false" data-tracking-control-name="footer-lang-dropdown_trigger">
<span class="language-selector__label-text mr-0.5 break-words">
Language
</span>
<icon class="language-selector__label-chevron w-2 h-2 absolute top-0 right-0" data-delayed-url="https://static.licdn.com/aero-v1/sc/h/cyolgscd0imw2ldqppkrb84vo"></icon>
</button>
</div>
</li>
</ul>
<!----> </footer>
<script src="https://static.licdn.com/aero-v1/sc/h/e69gtg8c61dzqklxridy0htl9" async></script>
<!---->
<!----><!---->
</body>
</html>