Files
nexus/sreweekly/articles/195/04-debugging-network-stalls-on-kubernetes.html
2026-09-12 17:23:01 +08:00

1113 lines
194 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!doctype html>
<html lang="en-US" class="mt-0">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="profile" href="https://gmpg.org/xfn/11">
<link rel="icon" type="image/x-icon" href="https://github.githubassets.com/favicon.ico">
<meta name='robots' content='index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1' />
<!-- This site is optimized with the Yoast SEO Premium plugin v28.4 (Yoast SEO v28.4) - https://yoast.com/product/yoast-seo-premium-wordpress/ -->
<title>Debugging network stalls on Kubernetes - The GitHub Blog</title>
<meta name="description" content="In this deep-dive, we identified and worked through sporadic latency issues with services running on Kubernetes in our environment." />
<link rel="canonical" href="https://github.blog/engineering/infrastructure/debugging-network-stalls-on-kubernetes/" />
<meta property="og:locale" content="en_US" />
<meta property="og:type" content="article" />
<meta property="og:title" content="Debugging network stalls on Kubernetes" />
<meta property="og:description" content="In this deep-dive, we identified and worked through sporadic latency issues with services running on Kubernetes in our environment." />
<meta property="og:url" content="https://github.blog/engineering/infrastructure/debugging-network-stalls-on-kubernetes/" />
<meta property="og:site_name" content="The GitHub Blog" />
<meta property="article:published_time" content="2019-11-22T07:14:39+00:00" />
<meta property="article:modified_time" content="2020-03-23T01:46:04+00:00" />
<meta property="og:image" content="https://github.blog/wp-content/uploads/2019/03/engineering-social.png?fit=1201%2C630" />
<meta property="og:image:width" content="1201" />
<meta property="og:image:height" content="630" />
<meta property="og:image:type" content="image/png" />
<meta name="author" content="Theo Julienne" />
<meta name="twitter:card" content="summary_large_image" />
<meta name="twitter:title" content="Debugging network stalls on Kubernetes" />
<meta name="twitter:description" content="In this deep-dive, we identified and worked through sporadic latency issues with services running on Kubernetes in our environment." />
<meta name="twitter:image" content="https://github.blog/wp-content/uploads/2019/03/engineering-social.png?fit=1201%2C630" />
<meta name="twitter:label1" content="Written by" />
<meta name="twitter:data1" content="Theo Julienne" />
<meta name="twitter:label2" content="Est. reading time" />
<meta name="twitter:data2" content="23 minutes" />
<script type='application/ld+json'>{"@context":"https:\/\/schema.org","headline":"Debugging network stalls on Kubernetes","author":{"@type":"Person","name":"Theo Julienne"},"datePublished":"2019-11-21T23:14:39-08:00","abstract":"In this deep-dive, we identified and worked through sporadic latency issues with services running on Kubernetes in our environment.","mainEntityOfPage":{"@type":"WebPage","@id":"https:\/\/github.blog\/engineering\/infrastructure\/debugging-network-stalls-on-kubernetes\/"},"image":"https:\/\/github.blog\/wp-content\/uploads\/2019\/03\/engineering-social.png?resize=150%2C150","@type":"TechArticle"}</script>
<script type='application/ld+json'>{"@context":"https:\/\/schema.org","headline":"Debugging network stalls on Kubernetes","author":{"@type":"Person","name":"Theo Julienne"},"datePublished":"2019-11-21T23:14:39-08:00","abstract":"In this deep-dive, we identified and worked through sporadic latency issues with services running on Kubernetes in our environment.","mainEntityOfPage":{"@type":"WebPage","@id":"https:\/\/github.blog\/engineering\/infrastructure\/debugging-network-stalls-on-kubernetes\/"},"image":"https:\/\/github.blog\/wp-content\/uploads\/2019\/03\/engineering-social.png?resize=150%2C150","@type":"BlogPosting"}</script>
<script type="application/ld+json" class="yoast-schema-graph">{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/github.blog\/engineering\/infrastructure\/debugging-network-stalls-on-kubernetes\/#article","isPartOf":{"@id":"https:\/\/github.blog\/engineering\/infrastructure\/debugging-network-stalls-on-kubernetes\/"},"author":[{"@id":"https:\/\/github.blog\/#\/schema\/person\/373f13da0b3ffe713b1e43c1051b667d"}],"headline":"Debugging network stalls on Kubernetes","datePublished":"2019-11-22T07:14:39+00:00","dateModified":"2020-03-23T01:46:04+00:00","mainEntityOfPage":{"@id":"https:\/\/github.blog\/engineering\/infrastructure\/debugging-network-stalls-on-kubernetes\/"},"wordCount":3349,"image":{"@id":"https:\/\/github.blog\/engineering\/infrastructure\/debugging-network-stalls-on-kubernetes\/#primaryimage"},"thumbnailUrl":"https:\/\/github.blog\/wp-content\/uploads\/2019\/03\/engineering-social.png?fit=1201%2C630","articleSection":["Engineering","Infrastructure"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/github.blog\/engineering\/infrastructure\/debugging-network-stalls-on-kubernetes\/","url":"https:\/\/github.blog\/engineering\/infrastructure\/debugging-network-stalls-on-kubernetes\/","name":"Debugging network stalls on Kubernetes - The GitHub Blog","isPartOf":{"@id":"https:\/\/github.blog\/#website"},"primaryImageOfPage":{"@id":"https:\/\/github.blog\/engineering\/infrastructure\/debugging-network-stalls-on-kubernetes\/#primaryimage"},"image":{"@id":"https:\/\/github.blog\/engineering\/infrastructure\/debugging-network-stalls-on-kubernetes\/#primaryimage"},"thumbnailUrl":"https:\/\/github.blog\/wp-content\/uploads\/2019\/03\/engineering-social.png?fit=1201%2C630","datePublished":"2019-11-22T07:14:39+00:00","dateModified":"2020-03-23T01:46:04+00:00","author":{"@id":"https:\/\/github.blog\/#\/schema\/person\/373f13da0b3ffe713b1e43c1051b667d"},"description":"In this deep-dive, we identified and worked through sporadic latency issues with services running on Kubernetes in our environment.","breadcrumb":{"@id":"https:\/\/github.blog\/engineering\/infrastructure\/debugging-network-stalls-on-kubernetes\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/github.blog\/engineering\/infrastructure\/debugging-network-stalls-on-kubernetes\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/github.blog\/engineering\/infrastructure\/debugging-network-stalls-on-kubernetes\/#primaryimage","url":"https:\/\/github.blog\/wp-content\/uploads\/2019\/03\/engineering-social.png?fit=1201%2C630","contentUrl":"https:\/\/github.blog\/wp-content\/uploads\/2019\/03\/engineering-social.png?fit=1201%2C630","width":1201,"height":630},{"@type":"BreadcrumbList","@id":"https:\/\/github.blog\/engineering\/infrastructure\/debugging-network-stalls-on-kubernetes\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/github.blog\/"},{"@type":"ListItem","position":2,"name":"Engineering","item":"https:\/\/github.blog\/engineering\/"},{"@type":"ListItem","position":3,"name":"Infrastructure","item":"https:\/\/github.blog\/engineering\/infrastructure\/"},{"@type":"ListItem","position":4,"name":"Debugging network stalls on Kubernetes"}]},{"@type":"WebSite","@id":"https:\/\/github.blog\/#website","url":"https:\/\/github.blog\/","name":"The GitHub Blog","description":"Updates, ideas, and inspiration from GitHub to help developers build and design software.","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/github.blog\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Person","@id":"https:\/\/github.blog\/#\/schema\/person\/373f13da0b3ffe713b1e43c1051b667d","name":"Theo Julienne","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/secure.gravatar.com\/avatar\/1a1be34c1481a40e8b25cad5116609e8a48873ff335e5c0f9555926bdc21d5fe?s=96&d=mm&r=gd608bcda23d68f75ba1a982489a40dae","url":"https:\/\/secure.gravatar.com\/avatar\/1a1be34c1481a40e8b25cad5116609e8a48873ff335e5c0f9555926bdc21d5fe?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/1a1be34c1481a40e8b25cad5116609e8a48873ff335e5c0f9555926bdc21d5fe?s=96&d=mm&r=g","caption":"Theo Julienne"},"sameAs":["http:\/\/theo.julienne.org\/"],"url":"https:\/\/github.blog\/author\/theojulienne\/"}]}</script>
<!-- / Yoast SEO Premium plugin. -->
<link rel='dns-prefetch' href='//ghcc.githubassets.com' />
<link rel='dns-prefetch' href='//js.monitor.azure.com' />
<link rel='dns-prefetch' href='//cdnjs.cloudflare.com' />
<link rel='dns-prefetch' href='//analytics.githubassets.com' />
<link rel='dns-prefetch' href='//v0.wordpress.com' />
<link rel="alternate" type="application/rss+xml" title="The GitHub Blog &raquo; Feed" href="https://github.blog/feed/" />
<link rel="alternate" type="application/rss+xml" title="The GitHub Blog &raquo; Comments Feed" href="https://github.blog/comments/feed/" />
<link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://github.blog/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fgithub.blog%2Fengineering%2Finfrastructure%2Fdebugging-network-stalls-on-kubernetes%2F" />
<link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://github.blog/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fgithub.blog%2Fengineering%2Finfrastructure%2Fdebugging-network-stalls-on-kubernetes%2F&#038;format=xml" />
<style id="wp-img-auto-sizes-contain-inline-css">
img:is([sizes=auto i],[sizes^="auto," i]){contain-intrinsic-size:3000px 1500px}
/*# sourceURL=wp-img-auto-sizes-contain-inline-css */
</style>
<style id="wp-emoji-styles-inline-css">
img.wp-smiley, img.emoji {
display: inline !important;
border: none !important;
box-shadow: none !important;
height: 1em !important;
width: 1em !important;
margin: 0 0.07em !important;
vertical-align: -0.1em !important;
background: none !important;
padding: 0 !important;
}
/*# sourceURL=wp-emoji-styles-inline-css */
</style>
<style id="wp-block-library-inline-css">
:root{--wp-block-synced-color:#7a00df;--wp-block-synced-color--rgb:122,0,223;--wp-bound-block-color:var(--wp-block-synced-color);--wp-editor-canvas-background:#ddd;--wp-admin-theme-color:#007cba;--wp-admin-theme-color--rgb:0,124,186;--wp-admin-theme-color-darker-10:#006ba1;--wp-admin-theme-color-darker-10--rgb:0,107,160.5;--wp-admin-theme-color-darker-20:#005a87;--wp-admin-theme-color-darker-20--rgb:0,90,135;--wp-admin-border-width-focus:2px}@media (min-resolution:192dpi){:root{--wp-admin-border-width-focus:1.5px}}.wp-element-button{cursor:pointer}:root .has-very-light-gray-background-color{background-color:#eee}:root .has-very-dark-gray-background-color{background-color:#313131}:root .has-very-light-gray-color{color:#eee}:root .has-very-dark-gray-color{color:#313131}:root .has-vivid-green-cyan-to-vivid-cyan-blue-gradient-background{background:linear-gradient(135deg,#00d084,#0693e3)}:root .has-purple-crush-gradient-background{background:linear-gradient(135deg,#34e2e4,#4721fb 50%,#ab1dfe)}:root .has-hazy-dawn-gradient-background{background:linear-gradient(135deg,#faaca8,#dad0ec)}:root .has-subdued-olive-gradient-background{background:linear-gradient(135deg,#fafae1,#67a671)}:root .has-atomic-cream-gradient-background{background:linear-gradient(135deg,#fdd79a,#004a59)}:root .has-nightshade-gradient-background{background:linear-gradient(135deg,#330968,#31cdcf)}:root .has-midnight-gradient-background{background:linear-gradient(135deg,#020381,#2874fc)}:root{--wp--preset--font-size--normal:16px;--wp--preset--font-size--huge:42px}.has-regular-font-size{font-size:1em}.has-larger-font-size{font-size:2.625em}.has-normal-font-size{font-size:var(--wp--preset--font-size--normal)}.has-huge-font-size{font-size:var(--wp--preset--font-size--huge)}:root .has-text-align-center{text-align:center}:root .has-text-align-left{text-align:left}:root .has-text-align-right{text-align:right}.has-fit-text{white-space:nowrap!important}#end-resizable-editor-section{display:none}.aligncenter{clear:both}.items-justified-left{justify-content:flex-start}.items-justified-center{justify-content:center}.items-justified-right{justify-content:flex-end}.items-justified-space-between{justify-content:space-between}.screen-reader-text{word-wrap:normal!important;border:0;clip-path:inset(50%);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px;word-break:normal!important}.screen-reader-text:focus{background-color:#ddd;clip-path:none;color:#444;display:block;font-size:1em;height:auto;left:5px;line-height:normal;padding:15px 23px 14px;text-decoration:none;top:5px;width:auto;z-index:100000}html :where(.has-border-color){border-style:solid}html :where([style^=border-color],[style*=";border-color"],[style*="; border-color"]){border-style:solid}html :where([style^=border-top-color],[style*=";border-top-color"],[style*="; border-top-color"]){border-top-style:solid}html :where([style^=border-right-color],[style*=";border-right-color"],[style*="; border-right-color"]){border-right-style:solid}html :where([style^=border-bottom-color],[style*=";border-bottom-color"],[style*="; border-bottom-color"]){border-bottom-style:solid}html :where([style^=border-left-color],[style*=";border-left-color"],[style*="; border-left-color"]){border-left-style:solid}html :where([style^=border-width],[style*=";border-width"],[style*="; border-width"]){border-style:solid}html :where([style^=border-top-width],[style*=";border-top-width"],[style*="; border-top-width"]){border-top-style:solid}html :where([style^=border-right-width],[style*=";border-right-width"],[style*="; border-right-width"]){border-right-style:solid}html :where([style^=border-bottom-width],[style*=";border-bottom-width"],[style*="; border-bottom-width"]){border-bottom-style:solid}html :where([style^=border-left-width],[style*=";border-left-width"],[style*="; border-left-width"]){border-left-style:solid}html :where(img[class*=wp-image-]){height:auto;max-width:100%}:where(figure){margin:0 0 1em}html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:var(--wp-admin--admin-bar--height,0px)}@media screen and (max-width:600px){html :where(.is-position-sticky){--wp-admin--admin-bar--position-offset:0px}}
/*# sourceURL=/wp-includes/css/dist/block-library/common.min.css */
</style>
<style id="github-typography-presets-inline-css">
.is-typography-preset-h1{font-size:var(--base-size-36, 36px);font-weight:700;line-height:1.25;margin-block-end:20px;margin-block-start:0;}.is-typography-preset-h2{font-size:var(--base-size-32, 32px);font-weight:700;line-height:1.25;margin-block-end:20px;margin-block-start:40px;}.is-typography-preset-h3{font-size:var(--base-size-28, 28px);font-weight:700;line-height:1.25;margin-block-end:12px;margin-block-start:20px;}.is-typography-preset-h4{font-size:var(--base-size-24, 24px);font-weight:700;line-height:1.25;margin-block-end:12px;margin-block-start:20px;}.is-typography-preset-h5{font-size:var(--base-size-20, 20px);font-weight:700;line-height:1.25;margin-block-end:12px;margin-block-start:20px;}.is-typography-preset-h6{font-size:var(--base-size-18, 18px);font-weight:700;line-height:1.25;margin-block-end:0;margin-block-start:20px;}.is-typography-preset-display{font:var(--text-display-shorthand);}.is-typography-preset-title-large{font:var(--text-title-shorthand-large);}.is-typography-preset-title-medium{font:var(--text-title-shorthand-medium);}.is-typography-preset-title-small{font:var(--text-title-shorthand-small);}.is-typography-preset-subtitle{font:var(--text-subtitle-shorthand);}.is-typography-preset-body-large{font:var(--text-body-shorthand-large);}.is-typography-preset-body-medium{font:var(--text-body-shorthand-medium);}.is-typography-preset-body-small{font:var(--text-body-shorthand-small);}
/*# sourceURL=github-typography-presets-inline-css */
</style>
<style id="global-styles-inline-css">
:root{--wp--preset--aspect-ratio--square: 1;--wp--preset--aspect-ratio--4-3: 4/3;--wp--preset--aspect-ratio--3-4: 3/4;--wp--preset--aspect-ratio--3-2: 3/2;--wp--preset--aspect-ratio--2-3: 2/3;--wp--preset--aspect-ratio--16-9: 16/9;--wp--preset--aspect-ratio--9-16: 9/16;--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgb(6,147,227) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgb(252,185,0) 0%,rgb(255,105,0) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgb(255,105,0) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--spacing--20: var(--base-size-20, 1.25rem);--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: var(--base-size-40, 2.5rem);--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: var(--base-size-80, 5rem);--wp--preset--spacing--4: var(--base-size-4, 0.25rem);--wp--preset--spacing--6: var(--base-size-6, 0.375rem);--wp--preset--spacing--8: var(--base-size-8, 0.5rem);--wp--preset--spacing--2: var(--base-size-2, 0.125rem);--wp--preset--spacing--12: var(--base-size-12, 0.75rem);--wp--preset--spacing--16: var(--base-size-16, 1rem);--wp--preset--spacing--24: var(--base-size-24, 1.5rem);--wp--preset--spacing--28: var(--base-size-28, 1.75rem);--wp--preset--spacing--32: var(--base-size-32, 2rem);--wp--preset--spacing--36: var(--base-size-36, 2.25rem);--wp--preset--spacing--44: var(--base-size-44, 2.75rem);--wp--preset--spacing--48: var(--base-size-48, 3rem);--wp--preset--spacing--64: var(--base-size-64, 4rem);--wp--preset--spacing--96: var(--base-size-96, 6rem);--wp--preset--spacing--112: var(--base-size-112, 7rem);--wp--preset--spacing--128: var(--base-size-128, 8rem);--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgb(255, 255, 255), 6px 6px rgb(0, 0, 0);--wp--preset--shadow--crisp: 6px 6px 0px rgb(0, 0, 0);--wp--custom--typography--presets--0--name: Like h1;--wp--custom--typography--presets--0--slug: h1;--wp--custom--typography--presets--0--font-size: var(--base-size-36, 36px);--wp--custom--typography--presets--0--font-weight: 700;--wp--custom--typography--presets--0--line-height: 1.25;--wp--custom--typography--presets--0--margin-block-end: 20px;--wp--custom--typography--presets--0--margin-block-start: 0;--wp--custom--typography--presets--1--name: Like h2;--wp--custom--typography--presets--1--slug: h2;--wp--custom--typography--presets--1--font-size: var(--base-size-32, 32px);--wp--custom--typography--presets--1--font-weight: 700;--wp--custom--typography--presets--1--line-height: 1.25;--wp--custom--typography--presets--1--margin-block-end: 20px;--wp--custom--typography--presets--1--margin-block-start: 40px;--wp--custom--typography--presets--2--name: Like h3;--wp--custom--typography--presets--2--slug: h3;--wp--custom--typography--presets--2--font-size: var(--base-size-28, 28px);--wp--custom--typography--presets--2--font-weight: 700;--wp--custom--typography--presets--2--line-height: 1.25;--wp--custom--typography--presets--2--margin-block-end: 12px;--wp--custom--typography--presets--2--margin-block-start: 20px;--wp--custom--typography--presets--3--name: Like h4;--wp--custom--typography--presets--3--slug: h4;--wp--custom--typography--presets--3--font-size: var(--base-size-24, 24px);--wp--custom--typography--presets--3--font-weight: 700;--wp--custom--typography--presets--3--line-height: 1.25;--wp--custom--typography--presets--3--margin-block-end: 12px;--wp--custom--typography--presets--3--margin-block-start: 20px;--wp--custom--typography--presets--4--name: Like h5;--wp--custom--typography--presets--4--slug: h5;--wp--custom--typography--presets--4--font-size: var(--base-size-20, 20px);--wp--custom--typography--presets--4--font-weight: 700;--wp--custom--typography--presets--4--line-height: 1.25;--wp--custom--typography--presets--4--margin-block-end: 12px;--wp--custom--typography--presets--4--margin-block-start: 20px;--wp--custom--typography--presets--5--name: Like h6;--wp--custom--typography--presets--5--slug: h6;--wp--custom--typography--presets--5--font-size: var(--base-size-18, 18px);--wp--custom--typography--presets--5--font-weight: 700;--wp--custom--typography--presets--5--line-height: 1.25;--wp--custom--typography--presets--5--margin-block-end: 0;--wp--custom--typography--presets--5--margin-block-start: 20px;--wp--custom--typography--presets--6--name: Display;--wp--custom--typography--presets--6--slug: display;--wp--custom--typography--presets--6--font: var(--text-display-shorthand);--wp--custom--typography--presets--7--name: Title Large;--wp--custom--typography--presets--7--slug: title-large;--wp--custom--typography--presets--7--font: var(--text-title-shorthand-large);--wp--custom--typography--presets--8--name: Title Medium;--wp--custom--typography--presets--8--slug: title-medium;--wp--custom--typography--presets--8--font: var(--text-title-shorthand-medium);--wp--custom--typography--presets--9--name: Title Small;--wp--custom--typography--presets--9--slug: title-small;--wp--custom--typography--presets--9--font: var(--text-title-shorthand-small);--wp--custom--typography--presets--10--name: Subtitle;--wp--custom--typography--presets--10--slug: subtitle;--wp--custom--typography--presets--10--font: var(--text-subtitle-shorthand);--wp--custom--typography--presets--11--name: Body Large;--wp--custom--typography--presets--11--slug: body-large;--wp--custom--typography--presets--11--font: var(--text-body-shorthand-large);--wp--custom--typography--presets--12--name: Body Medium;--wp--custom--typography--presets--12--slug: body-medium;--wp--custom--typography--presets--12--font: var(--text-body-shorthand-medium);--wp--custom--typography--presets--13--name: Body Small;--wp--custom--typography--presets--13--slug: body-small;--wp--custom--typography--presets--13--font: var(--text-body-shorthand-small);}.wp-block-button{--wp--preset--dimension--25: 25%;--wp--preset--dimension--50: 50%;--wp--preset--dimension--75: 75%;--wp--preset--dimension--100: 100%;}:root { --wp--style--global--content-size: calc(1280px - var(--p-responsive-blog) * 2);--wp--style--global--wide-size: calc(1280px - var(--p-responsive-blog) * 2); }:where(body) { margin: 0; }.wp-site-blocks { padding-top: var(--wp--style--root--padding-top); padding-bottom: var(--wp--style--root--padding-bottom); }.has-global-padding { padding-right: var(--wp--style--root--padding-right); padding-left: var(--wp--style--root--padding-left); }.has-global-padding > .alignfull { margin-right: calc(var(--wp--style--root--padding-right) * -1); margin-left: calc(var(--wp--style--root--padding-left) * -1); }.has-global-padding :where(:not(.alignfull.is-layout-flow) > .has-global-padding:not(.wp-block-block, .alignfull)) { padding-right: 0; padding-left: 0; }.has-global-padding :where(:not(.alignfull.is-layout-flow) > .has-global-padding:not(.wp-block-block, .alignfull)) > .alignfull { margin-left: 0; margin-right: 0; }.wp-site-blocks > .alignleft { float: left; margin-right: 2em; }.wp-site-blocks > .alignright { float: right; margin-left: 2em; }.wp-site-blocks > .aligncenter { justify-content: center; margin-left: auto; margin-right: auto; }:where(.wp-site-blocks) > * { margin-block-start: 24px; margin-block-end: 0; }:where(.wp-site-blocks) > :first-child { margin-block-start: 0; }:where(.wp-site-blocks) > :last-child { margin-block-end: 0; }:root { --wp--style--block-gap: 24px; }:root :where(.is-layout-flow) > :first-child{margin-block-start: 0;}:root :where(.is-layout-flow) > :last-child{margin-block-end: 0;}:root :where(.is-layout-flow) > *{margin-block-start: 24px;margin-block-end: 0;}:root :where(.is-layout-constrained) > :first-child{margin-block-start: 0;}:root :where(.is-layout-constrained) > :last-child{margin-block-end: 0;}:root :where(.is-layout-constrained) > *{margin-block-start: 24px;margin-block-end: 0;}:root :where(.is-layout-flex){gap: 24px;}:root :where(.is-layout-grid){gap: 24px;}.is-layout-flow > .alignleft{float: left;margin-inline-start: 0;margin-inline-end: 2em;}.is-layout-flow > .alignright{float: right;margin-inline-start: 2em;margin-inline-end: 0;}.is-layout-flow > .aligncenter{margin-left: auto !important;margin-right: auto !important;}.is-layout-constrained > .alignleft{float: left;margin-inline-start: 0;margin-inline-end: 2em;}.is-layout-constrained > .alignright{float: right;margin-inline-start: 2em;margin-inline-end: 0;}.is-layout-constrained > .aligncenter{margin-left: auto !important;margin-right: auto !important;}.is-layout-constrained > :where(:not(.alignleft):not(.alignright):not(.alignfull)){max-width: var(--wp--style--global--content-size);margin-left: auto !important;margin-right: auto !important;}.is-layout-constrained > .alignwide{max-width: var(--wp--style--global--wide-size);}body .is-layout-flex{display: flex;}.is-layout-flex{flex-wrap: wrap;align-items: center;}.is-layout-flex > :is(*, div){margin: 0;}body .is-layout-grid{display: grid;}.is-layout-grid > :is(*, div){margin: 0;}body{font-family: var(--brand-fontStack-sansSerif, sans-serif);--wp--style--root--padding-top: 0px;--wp--style--root--padding-right: var(--p-responsive-blog);--wp--style--root--padding-bottom: 0px;--wp--style--root--padding-left: var(--p-responsive-blog);}:root :where(.wp-element-button, .wp-block-button__link){background-color: #32373c;border-width: 0;color: #fff;font-family: inherit;font-size: inherit;font-style: inherit;font-weight: inherit;letter-spacing: inherit;line-height: inherit;padding-top: calc(0.667em + 2px);padding-right: calc(1.333em + 2px);padding-bottom: calc(0.667em + 2px);padding-left: calc(1.333em + 2px);text-decoration: none;text-transform: inherit;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
/*# sourceURL=global-styles-inline-css */
</style>
<link rel='stylesheet' id='all-css-12' href='https://github.blog/_static/??-eJyNjEEOwjAMBD+EMalEWw6It7SpSSzFaVQ7KvyewAWOXGd3BvcCfs1G2bCkGjgrNiSUK7BMgXBSJVP02gZhuG/tDWrPRMfGDvgTsEhCioEt1hm6U+fAR04LLqz2KSQO0eb18Yf6lZSNQP3Gxd7eTa5uGMfLuXf98ALe40UN' type='text/css' media='all' />
<link rel='stylesheet' id='highlightjs-css-css' href='https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.11.1/styles/default.min.css?ver=11.4.0' media='all' />
<link rel="https://api.w.org/" href="https://github.blog/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://github.blog/wp-json/wp/v2/posts/51321" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://github.blog/xmlrpc.php?rsd" />
<meta name="generator" content="WordPress 7.1" />
<link rel='shortlink' href='https://wp.me/pamS32-dlL' />
<!-- Stream WordPress user activity plugin v4.4.0 -->
<style>img#wpstats{display:none}</style>
<link rel="llms-sitemap" href="https://github.blog/llms.txt" />
<meta name="ha-url" content="https://collector.githubapp.com/github-blog/collect"><link rel="preload" href="https://github.blog/wp-content/themes/github-2021/dist/fonts/mona-sans.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://github.blog/wp-content/themes/github-2021/dist/fonts/MonaspaceNeon-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="icon" href="https://github.blog/wp-content/uploads/2019/01/cropped-github-favicon-512.png?fit=32%2C32" sizes="32x32" />
<link rel="icon" href="https://github.blog/wp-content/uploads/2019/01/cropped-github-favicon-512.png?fit=192%2C192" sizes="192x192" />
<link rel="apple-touch-icon" href="https://github.blog/wp-content/uploads/2019/01/cropped-github-favicon-512.png?fit=180%2C180" />
<meta name="msapplication-TileImage" content="https://github.blog/wp-content/uploads/2019/01/cropped-github-favicon-512.png?fit=270%2C270" />
<style id="wp-style-engine-github-typography-presets-inline-css">
.is-typography-preset-h1{font-size:var(--base-size-36, 36px);font-weight:700;line-height:1.25;margin-block-end:20px;margin-block-start:0;}.is-typography-preset-h2{font-size:var(--base-size-32, 32px);font-weight:700;line-height:1.25;margin-block-end:20px;margin-block-start:40px;}.is-typography-preset-h3{font-size:var(--base-size-28, 28px);font-weight:700;line-height:1.25;margin-block-end:12px;margin-block-start:20px;}.is-typography-preset-h4{font-size:var(--base-size-24, 24px);font-weight:700;line-height:1.25;margin-block-end:12px;margin-block-start:20px;}.is-typography-preset-h5{font-size:var(--base-size-20, 20px);font-weight:700;line-height:1.25;margin-block-end:12px;margin-block-start:20px;}.is-typography-preset-h6{font-size:var(--base-size-18, 18px);font-weight:700;line-height:1.25;margin-block-end:0;margin-block-start:20px;}.is-typography-preset-display{font:var(--text-display-shorthand);}.is-typography-preset-title-large{font:var(--text-title-shorthand-large);}.is-typography-preset-title-medium{font:var(--text-title-shorthand-medium);}.is-typography-preset-title-small{font:var(--text-title-shorthand-small);}.is-typography-preset-subtitle{font:var(--text-subtitle-shorthand);}.is-typography-preset-body-large{font:var(--text-body-shorthand-large);}.is-typography-preset-body-medium{font:var(--text-body-shorthand-medium);}.is-typography-preset-body-small{font:var(--text-body-shorthand-small);}
/*# sourceURL=wp-style-engine-github-typography-presets-inline-css */
</style>
</head>
<body class="wp-singular post-template-default single single-post postid-51321 single-format-standard wp-embed-responsive wp-theme-github-2021 wp-child-theme-github-2021-child font-mktg no-sidebar" data-color-mode="light" data-light-theme="light" data-dark-theme="dark">
<div data-color-mode="dark" data-light-theme="light" data-dark-theme="dark_dimmed" class="pt-header pt-lg-0">
<header id="header" class="header position-fixed position-lg-static pb-lg-header z-4 top-0 left-0 right-0 d-flex flex-column flex-items-stretch color-bg-default">
<a href="#start-of-content" class="p-3 color-bg-accent-emphasis color-fg-on-emphasis show-on-focus">
Skip to content </a>
<a href="#sidebar" id="skip-to-sidebar" class="p-3 color-bg-accent-emphasis color-fg-on-emphasis show-on-focus">
Skip to sidebar </a>
<div class="position-relative container-xl width-full mx-auto p-responsive-blog">
<div class="d-flex flex-items-center flex-justify-between pt-3 pb-3 color-fg-default">
<a href="https://github.com" target="_blank" rel="noreferrer" aria-label="GitHub homepage" class="Header-link position-relative d-flex flex-items-center color-fg-default">
<svg aria-hidden="true" role="img" class="octicon octicon-mark-github d-block" viewBox="0 0 98 96" width="32" height="32" fill="currentColor"><g clip-path="url(#a)"><path d="M41.44 69.385C28.807 67.853 19.906 58.762 19.906 46.99c0-4.785 1.723-9.953 4.594-13.398-1.244-3.158-1.053-9.858.383-12.633 3.828-.479 8.996 1.531 12.058 4.307 3.637-1.149 7.465-1.723 12.155-1.723 4.69 0 8.517.574 11.963 1.627 2.966-2.68 8.23-4.69 12.058-4.211 1.34 2.584 1.531 9.283.287 12.537 3.063 3.637 4.69 8.518 4.69 13.494 0 11.772-8.9 20.672-21.725 22.3 3.254 2.104 5.455 6.698 5.455 11.962v9.953c0 2.871 2.393 4.498 5.264 3.35C84.41 87.95 98 70.629 98 49.19 98 22.107 75.988 0 48.904 0 21.82 0 0 22.107 0 49.191c0 21.246 13.494 38.856 31.678 45.46 2.584.956 5.072-.766 5.072-3.35v-7.657c-1.34.575-3.063.958-4.594.958-6.316 0-10.049-3.446-12.728-9.858-1.053-2.584-2.201-4.115-4.403-4.402-1.148-.096-1.53-.574-1.53-1.149 0-1.148 1.913-2.01 3.827-2.01 2.776 0 5.168 1.723 7.657 5.264 1.914 2.776 3.923 4.02 6.316 4.02 2.392 0 3.924-.861 6.125-3.063 1.627-1.627 2.871-3.062 4.02-4.02z" fill="#fff"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h98v96H0z"/></clipPath></defs></svg>
</a>
<span class="d-inline-block ml-2 f1-mktg f2-md-mktg" style="opacity: 0.3;">/</span>
<a class="d-inline-block Header-link font-weight-semibold ml-2 f2 color-fg-default" href="https://github.blog/">
Blog</a>
<nav class="d-none d-lg-block" aria-label="Secondary navigation">
<ul id="secondary-navigation" class="secondary-navigation flex-items-center flex-nowrap list-style-none ml-4" aria-hidden="false"><li id="menu-item-78809"><a href="https://github.blog/changelog/" class="position-relative d-flex flex-items-center flex-start no-wrap py-2 px-4 f4 lh-condensed-ultra Link--secondary color-fg-default text-medium"><span class="menu-item-label">Changelog</span></a></li>
<li id="menu-item-78810"><a href="https://docs.github.com/" class="position-relative d-flex flex-items-center flex-start no-wrap py-2 px-4 f4 lh-condensed-ultra Link--secondary color-fg-default text-medium"><span class="menu-item-label">Docs</span><svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-down position-absolute right-0 d-block ml-1" role="presentation"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li>
<li id="menu-item-78811"><a href="https://github.com/customer-stories" class="position-relative d-flex flex-items-center flex-start no-wrap py-2 px-4 f4 lh-condensed-ultra Link--secondary color-fg-default text-medium"><span class="menu-item-label">Customer stories</span><svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-down position-absolute right-0 d-block ml-1" role="presentation"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li>
</ul></nav>
<div class="d-none d-lg-flex flex-1">
<form id="desktop-search" class="desktop-search position-relative ml-lg-4 flex-1" action="https://github.blog" method="get" aria-hidden="true" aria-label="Search form" role="search">
<div class="position-relative d-flex flex-1 height-full color-bg-transparent" data-color-mode="light" data-light-theme="light" data-dark-theme="dark" >
<input aria-label="Search the blog" type="search" class="p-2 pl-3 pr-6 border-0 rounded-2 flex-1" placeholder="Search the blog…" value="" name="s" id="search-input">
<button type="submit" class="position-absolute right-0 z-3 d-flex flex-items-center flex-justify-center flex-self-center mr-2 p-2 border-0 rounded-2 color-bg-transparent color-fg-subtle" aria-label="Search">
<svg viewBox="0 0 16 16" width="20" height="20" class="octicon octicon-search" role="presentation"><path fill-rule="evenodd" d="M11.5 7a4.499 4.499 0 11-8.998 0A4.499 4.499 0 0111.5 7zm-.82 4.74a6 6 0 111.06-1.06l3.04 3.04a.75.75 0 11-1.06 1.06l-3.04-3.04z"></path></svg>
</button>
</div>
</form>
<button aria-label="Toggle search" class="flex-self-center ml-auto p-2 border-0 color-bg-transparent color-fg-default rounded-3 js-toggle" aria-controls="desktop-search" aria-expanded="false" >
<svg viewBox="0 0 24 24" width="24" height="24" class="octicon octicon-search" role="presentation"><path d="M10.25 2a8.25 8.25 0 0 1 6.34 13.53l5.69 5.69a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215l-5.69-5.69A8.25 8.25 0 1 1 10.25 2ZM3.5 10.25a6.75 6.75 0 1 0 13.5 0 6.75 6.75 0 0 0-13.5 0Z"></path></svg>
<svg viewBox="2 2 20 20" width="24" height="24" class="octicon octicon-x" role="presentation"><path d="M5.72 5.72a.75.75 0 0 1 1.06 0L12 10.94l5.22-5.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L13.06 12l5.22 5.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L12 13.06l-5.22 5.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L10.94 12 5.72 6.78a.75.75 0 0 1 0-1.06Z"></path></svg>
</button>
<a class="Button Button--size-medium Button--primary ml-3"
href="https://github.com/features/ai/github-app?utm_source=blog-header-button-cta&#038;utm_medium=blog&#038;utm_campaign=github-app-cli-Q1-push" target="_blank" data-analytics-click="Blog, click on button, text: Use Copilot for free; ref_location:top nav;" >
<span class="Button__text">
<span class="Text Text--200 Text--antialiased Text--weight-semibold Button--label Button--label-medium Button--label-primary">
Try GitHub Copilot app </span>
</span>
</a>
<a class="Button Button--size-medium Button--secondary ml-3"
href="https://githubuniverse.com/?utm_source=Blog-button&#038;utm_medium=GitHub&#038;utm_campaign=blog_button_seb_uni_26" target="_blank" data-analytics-click="Blog, click on button, text: Attend Universe; ref_location:top nav;" >
<span class="Button__text">
<span class="Text Text--200 Text--antialiased Text--weight-semibold Button--label Button--label-medium Button--label-secondary">
Attend GitHub Universe </span>
</span>
</a>
</div>
<div class="d-flex d-lg-none flex-items-center flex-1 mr-n2">
<button aria-label="Toggle search" class="ml-auto p-2 border-0 color-bg-transparent color-fg-default rounded-3 js-toggle" aria-controls="mobile-search" aria-expanded="false" >
<svg viewBox="0 0 24 24" width="24" height="24" class="octicon octicon-search" role="presentation"><path d="M10.25 2a8.25 8.25 0 0 1 6.34 13.53l5.69 5.69a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215l-5.69-5.69A8.25 8.25 0 1 1 10.25 2ZM3.5 10.25a6.75 6.75 0 1 0 13.5 0 6.75 6.75 0 0 0-13.5 0Z"></path></svg>
<svg viewBox="2 2 20 20" width="24" height="24" class="octicon octicon-x" role="presentation"><path d="M5.72 5.72a.75.75 0 0 1 1.06 0L12 10.94l5.22-5.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L13.06 12l5.22 5.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L12 13.06l-5.22 5.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L10.94 12 5.72 6.78a.75.75 0 0 1 0-1.06Z"></path></svg>
</button>
<button class="mobile-nav-focus-trap js-mobile-nav-focus-trap" data-target="#mobile-menu" data-focus-order="last"></button>
<button id="mobile-menu-toggle" aria-label="Toggle menu" class="ml-2 p-2 border-0 color-bg-transparent color-fg-default rounded-3 js-toggle" aria-controls="mobile-menu" aria-expanded="false" data-trap-focus="#header">
<svg viewBox="0 0 16 16" width="24" height="24" class="octicon octicon-three-bars" role="presentation"><path d="M1 2.75A.75.75 0 0 1 1.75 2h12.5a.75.75 0 0 1 0 1.5H1.75A.75.75 0 0 1 1 2.75Zm0 5A.75.75 0 0 1 1.75 7h12.5a.75.75 0 0 1 0 1.5H1.75A.75.75 0 0 1 1 7.75ZM1.75 12h12.5a.75.75 0 0 1 0 1.5H1.75a.75.75 0 0 1 0-1.5Z"></path></svg>
<svg viewBox="2 2 20 20" width="24" height="24" class="octicon octicon-x " role="presentation"><path d="M5.72 5.72a.75.75 0 0 1 1.06 0L12 10.94l5.22-5.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L13.06 12l5.22 5.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L12 13.06l-5.22 5.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L10.94 12 5.72 6.78a.75.75 0 0 1 0-1.06Z"></path></svg>
</button>
</div>
</div>
</div>
<div class="position-relative pb-lg-navigation" data-sticky-navigation=".sticky-navigation">
<nav aria-label="Primary navigation" class="sticky-navigation position-absolute z-4 top-0 left-0 d-none d-lg-block width-full color-border-subtle color-bg-default">
<div class="position-relative container-xl width-full mx-auto p-responsive-blog">
<ul id="primary-navigation" class="primary-navigation position-relative d-flex flex-row flex-nowrap flex-grow-0 flex-justify-between list-style-none"><li id="menu-item-78814"><a href="https://github.blog/ai-and-ml/" class="position-relative d-flex flex-items-center flex-start no-wrap py-3 f4-mktg text-bold js-toggle" aria-controls="primary-78814-dropdown" aria-expanded="false" aria-haspopup="true" role="button">AI &amp; ML<svg viewBox="0 0 16 16" width="16" height="16" class="octicon d-block ml-1 color-fg-muted" role="presentation"><path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path></svg></a>
<ul class="dropdown d-flex flex-wrap width-full position-absolute left-0 list-style-none mt-2 rounded-3 z-3 overflow-hidden color-border-subtle" aria-label="AI &amp; ML sub-menu" aria-hidden="true" data-color-mode="light" data-dark-theme="dark" data-light-theme="light" id="primary-78814-dropdown" tabindex="-1">
<li id="menu-item-78863" class="flex-1 p-5 py-xl-7 px-xl-7"><div class="col-9 mb-4 mb-lg-7"><a href="https://github.blog/ai-and-ml/" class="d-flex flex-items-center flex-start f3 lh-condensed color-fg-default text-bold">AI &amp; ML<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right ml-1" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a><p class="mt-1 my-0 f4 color-fg-muted">Learn about artificial intelligence and machine learning across the GitHub ecosystem and the wider industry.</p></div>
<ul class="d-flex flex-wrap list-style-none" aria-label="AI &amp; ML sub-menu">
<li id="menu-item-78815" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right"><a href="https://github.blog/ai-and-ml/generative-ai/" class="d-block f4 color-fg-default text-bold">Generative AI</a><p class="mb-0 f5 color-fg-muted">Learn how to build with generative AI.</p></li>
<li id="menu-item-78816" class="col-6 d-block f4 color-fg-default color-border-subtle pl-4 pl-lg-8"><a href="https://github.blog/ai-and-ml/github-copilot/" class="d-block f4 color-fg-default text-bold">GitHub Copilot</a><p class="mb-0 f5 color-fg-muted">Change how you work with GitHub Copilot.</p></li>
<li id="menu-item-78817" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right pt-2 pt-lg-4"><a href="https://github.blog/ai-and-ml/llms/" class="d-block f4 color-fg-default text-bold">LLMs</a><p class="mb-0 f5 color-fg-muted">Everything developers need to know about LLMs.</p></li>
<li id="menu-item-78818" class="col-6 d-block f4 color-fg-default color-border-subtle pl-4 pl-lg-8 pt-2 pt-lg-4"><a href="https://github.blog/ai-and-ml/machine-learning/" class="d-block f4 color-fg-default text-bold">Machine learning</a><p class="mb-0 f5 color-fg-muted">Machine learning tips, tricks, and best practices.</p></li>
</ul>
</li>
<li id="menu-item-78864" class="p-5 py-xl-7 px-xl-8 col-4 color-bg-subtle post-card"><div class="d-block position-relative mb-3 rounded-2 tease-thumbnail overflow-hidden"><img width="800" height="425" src="https://github.blog/wp-content/uploads/2024/06/AI-DarkMode-4.png?resize=800%2C425" class="tease-thumbnail__img d-block width-full height-auto aspect-ratio-1032-548 object-fit-cover" alt="" decoding="async" fetchpriority="high" srcset="https://github.blog/wp-content/uploads/2024/06/AI-DarkMode-4.png?w=300 300w, https://github.blog/wp-content/uploads/2024/06/AI-DarkMode-4.png?w=800 800w, https://github.blog/wp-content/uploads/2024/06/AI-DarkMode-4.png?w=400 400w, https://github.blog/wp-content/uploads/2024/06/AI-DarkMode-4.png?w=1032 1032w, https://github.blog/wp-content/uploads/2024/06/AI-DarkMode-4.png?w=516 516w" sizes="(max-width: 800px) 100vw, 800px" /></div><a href="https://github.blog/ai-and-ml/generative-ai/how-ai-code-generation-works/" class="d-block mb-2 f3 lh-condensed color-fg-default text-bold post-card__link">How AI code generation works</a><p class="mb-3 f5 color-fg-muted">Explore the capabilities and benefits of AI code generation and how it can improve your developer experience.</p><span class="Link d-inline-flex flex-items-center" role="presentation">Learn more<svg xmlns="http://www.w3.org/2000/svg" class="octicon d-block ml-1" viewBox="0 0 12 12" width="12" height="12"><path d="M4.7 10c-.2 0-.4-.1-.5-.2-.3-.3-.3-.8 0-1.1L6.9 6 4.2 3.3c-.3-.3-.3-.8 0-1.1.3-.3.8-.3 1.1 0l3.3 3.2c.3.3.3.8 0 1.1L5.3 9.7c-.2.2-.4.3-.6.3Z"></path></svg></span></li>
</ul>
</li>
<li id="menu-item-78819"><a href="https://github.blog/developer-skills/" class="position-relative d-flex flex-items-center flex-start no-wrap py-3 f4-mktg text-bold js-toggle" aria-controls="primary-78819-dropdown" aria-expanded="false" aria-haspopup="true" role="button">Developer skills<svg viewBox="0 0 16 16" width="16" height="16" class="octicon d-block ml-1 color-fg-muted" role="presentation"><path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path></svg></a>
<ul class="dropdown d-flex flex-wrap width-full position-absolute left-0 list-style-none mt-2 rounded-3 z-3 overflow-hidden color-border-subtle" aria-label="Developer skills sub-menu" aria-hidden="true" data-color-mode="light" data-dark-theme="dark" data-light-theme="light" id="primary-78819-dropdown" tabindex="-1">
<li id="menu-item-78865" class="flex-1 p-5 py-xl-7 px-xl-7"><div class="col-9 mb-4 mb-lg-7"><a href="https://github.blog/developer-skills/" class="d-flex flex-items-center flex-start f3 lh-condensed color-fg-default text-bold">Developer skills<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right ml-1" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a><p class="mt-1 my-0 f4 color-fg-muted">Resources for developers to grow in their skills and careers.</p></div>
<ul class="d-flex flex-wrap list-style-none" aria-label="Developer skills sub-menu">
<li id="menu-item-78820" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right"><a href="https://github.blog/developer-skills/application-development/" class="d-block f4 color-fg-default text-bold">Application development</a><p class="mb-0 f5 color-fg-muted">Insights and best practices for building apps.</p></li>
<li id="menu-item-78821" class="col-6 d-block f4 color-fg-default color-border-subtle pl-4 pl-lg-8"><a href="https://github.blog/developer-skills/career-growth/" class="d-block f4 color-fg-default text-bold">Career growth</a><p class="mb-0 f5 color-fg-muted">Tips &amp; tricks to grow as a professional developer.</p></li>
<li id="menu-item-78822" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right pt-2 pt-lg-4"><a href="https://github.blog/developer-skills/github/" class="d-block f4 color-fg-default text-bold">GitHub</a><p class="mb-0 f5 color-fg-muted">Improve how you use GitHub at work.</p></li>
<li id="menu-item-78823" class="col-6 d-block f4 color-fg-default color-border-subtle pl-4 pl-lg-8 pt-2 pt-lg-4"><a href="https://github.blog/developer-skills/github-education/" class="d-block f4 color-fg-default text-bold">GitHub Education</a><p class="mb-0 f5 color-fg-muted">Learn how to move into your first professional role.</p></li>
<li id="menu-item-78824" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right pt-2 pt-lg-4"><a href="https://github.blog/developer-skills/programming-languages-and-frameworks/" class="d-block f4 color-fg-default text-bold">Programming languages &amp; frameworks</a><p class="mb-0 f5 color-fg-muted">Stay current on what’s new (or new again).</p></li>
</ul>
</li>
<li id="menu-item-78866" class="p-5 py-xl-7 px-xl-8 col-4 color-bg-subtle post-card"><div class="d-block position-relative mb-3 rounded-2 tease-thumbnail overflow-hidden"><img width="800" height="425" src="https://github.blog/wp-content/uploads/2024/05/Enterprise-DarkMode-3.png?resize=800%2C425" class="tease-thumbnail__img d-block width-full height-auto aspect-ratio-1032-548 object-fit-cover" alt="" decoding="async" srcset="https://github.blog/wp-content/uploads/2024/05/Enterprise-DarkMode-3.png?w=300 300w, https://github.blog/wp-content/uploads/2024/05/Enterprise-DarkMode-3.png?w=800 800w, https://github.blog/wp-content/uploads/2024/05/Enterprise-DarkMode-3.png?w=400 400w, https://github.blog/wp-content/uploads/2024/05/Enterprise-DarkMode-3.png?w=1032 1032w, https://github.blog/wp-content/uploads/2024/05/Enterprise-DarkMode-3.png?w=516 516w" sizes="(max-width: 800px) 100vw, 800px" /></div><a href="https://docs.github.com/en/get-started" class="d-block mb-2 f3 lh-condensed color-fg-default text-bold post-card__link">Get started with GitHub documentation</a><p class="mb-3 f5 color-fg-muted">Learn how to start building, shipping, and maintaining software with GitHub.</p><span class="Link d-inline-flex flex-items-center" role="presentation">Learn more<svg xmlns="http://www.w3.org/2000/svg" class="octicon d-block ml-1" viewBox="0 0 12 12" width="12" height="12"><path d="M4.7 10c-.2 0-.4-.1-.5-.2-.3-.3-.3-.8 0-1.1L6.9 6 4.2 3.3c-.3-.3-.3-.8 0-1.1.3-.3.8-.3 1.1 0l3.3 3.2c.3.3.3.8 0 1.1L5.3 9.7c-.2.2-.4.3-.6.3Z"></path></svg></span></li>
</ul>
</li>
<li id="menu-item-78825"><a href="https://github.blog/engineering/" class="position-relative d-flex flex-items-center flex-start no-wrap py-3 f4-mktg text-bold js-toggle" aria-controls="primary-78825-dropdown" aria-expanded="false" aria-haspopup="true" role="button">Engineering<svg viewBox="0 0 16 16" width="16" height="16" class="octicon d-block ml-1 color-fg-muted" role="presentation"><path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path></svg></a>
<ul class="dropdown d-flex flex-wrap width-full position-absolute left-0 list-style-none mt-2 rounded-3 z-3 overflow-hidden color-border-subtle" aria-label="Engineering sub-menu" aria-hidden="true" data-color-mode="light" data-dark-theme="dark" data-light-theme="light" id="primary-78825-dropdown" tabindex="-1">
<li id="menu-item-78867" class="flex-1 p-5 py-xl-7 px-xl-7"><div class="col-9 mb-4 mb-lg-7"><a href="https://github.blog/engineering/" class="d-flex flex-items-center flex-start f3 lh-condensed color-fg-default text-bold">Engineering<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right ml-1" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a><p class="mt-1 my-0 f4 color-fg-muted">Get an inside look at how we’re building the home for all developers.</p></div>
<ul class="d-flex flex-wrap list-style-none" aria-label="Engineering sub-menu">
<li id="menu-item-78827" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right"><a href="https://github.blog/engineering/architecture-optimization/" class="d-block f4 color-fg-default text-bold">Architecture &amp; optimization</a><p class="mb-0 f5 color-fg-muted">Discover how we deliver a performant and highly available experience across the GitHub platform.</p></li>
<li id="menu-item-78828" class="col-6 d-block f4 color-fg-default color-border-subtle pl-4 pl-lg-8"><a href="https://github.blog/engineering/engineering-principles/" class="d-block f4 color-fg-default text-bold">Engineering principles</a><p class="mb-0 f5 color-fg-muted">Explore best practices for building software at scale with a majority remote team.</p></li>
<li id="menu-item-78829" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right pt-2 pt-lg-4"><a href="https://github.blog/engineering/infrastructure/" class="d-block f4 color-fg-default text-bold">Infrastructure</a><p class="mb-0 f5 color-fg-muted">Get a glimpse at the technology underlying the world’s leading AI-powered developer platform.</p></li>
<li id="menu-item-78830" class="col-6 d-block f4 color-fg-default color-border-subtle pl-4 pl-lg-8 pt-2 pt-lg-4"><a href="https://github.blog/engineering/platform-security/" class="d-block f4 color-fg-default text-bold">Platform security</a><p class="mb-0 f5 color-fg-muted">Learn how we build security into everything we do across the developer lifecycle.</p></li>
<li id="menu-item-78858" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right pt-2 pt-lg-4"><a href="https://github.blog/engineering/user-experience/" class="d-block f4 color-fg-default text-bold">User experience</a><p class="mb-0 f5 color-fg-muted">Find out what goes into making GitHub the home for all developers.</p></li>
</ul>
</li>
<li id="menu-item-78868" class="p-5 py-xl-7 px-xl-8 col-4 color-bg-subtle post-card"><div class="d-block position-relative mb-3 rounded-2 tease-thumbnail overflow-hidden"><img width="800" height="425" src="https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.42.45 AM.png?resize=800%2C425" class="tease-thumbnail__img d-block width-full height-auto aspect-ratio-1032-548 object-fit-cover" alt="" decoding="async" srcset="https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.42.45 AM.png?w=800 800w, https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.42.45 AM.png?w=400 400w, https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.42.45 AM.png?w=1032 1032w, https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.42.45 AM.png?w=516 516w" sizes="(max-width: 800px) 100vw, 800px" /></div><a href="https://github.blog/engineering/how-we-use-github-to-be-more-productive-collaborative-and-secure/" class="d-block mb-2 f3 lh-condensed color-fg-default text-bold post-card__link">How we use GitHub to be more productive, collaborative, and secure</a><p class="mb-3 f5 color-fg-muted">Our engineering and security teams do some incredible work. Let’s take a look at how we use GitHub to be more productive, build collaboratively, and shift security left.</p><span class="Link d-inline-flex flex-items-center" role="presentation">Learn more<svg xmlns="http://www.w3.org/2000/svg" class="octicon d-block ml-1" viewBox="0 0 12 12" width="12" height="12"><path d="M4.7 10c-.2 0-.4-.1-.5-.2-.3-.3-.3-.8 0-1.1L6.9 6 4.2 3.3c-.3-.3-.3-.8 0-1.1.3-.3.8-.3 1.1 0l3.3 3.2c.3.3.3.8 0 1.1L5.3 9.7c-.2.2-.4.3-.6.3Z"></path></svg></span></li>
</ul>
</li>
<li id="menu-item-78832"><a href="https://github.blog/enterprise-software/" class="position-relative d-flex flex-items-center flex-start no-wrap py-3 f4-mktg text-bold js-toggle" aria-controls="primary-78832-dropdown" aria-expanded="false" aria-haspopup="true" role="button">Enterprise software<svg viewBox="0 0 16 16" width="16" height="16" class="octicon d-block ml-1 color-fg-muted" role="presentation"><path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path></svg></a>
<ul class="dropdown d-flex flex-wrap width-full position-absolute left-0 list-style-none mt-2 rounded-3 z-3 overflow-hidden color-border-subtle" aria-label="Enterprise software sub-menu" aria-hidden="true" data-color-mode="light" data-dark-theme="dark" data-light-theme="light" id="primary-78832-dropdown" tabindex="-1">
<li id="menu-item-78869" class="flex-1 p-5 py-xl-7 px-xl-7"><div class="col-9 mb-4 mb-lg-7"><a href="https://github.blog/enterprise-software/" class="d-flex flex-items-center flex-start f3 lh-condensed color-fg-default text-bold">Enterprise software<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right ml-1" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a><p class="mt-1 my-0 f4 color-fg-muted">Explore how to write, build, and deploy enterprise software at scale.</p></div>
<ul class="d-flex flex-wrap list-style-none" aria-label="Enterprise software sub-menu">
<li id="menu-item-78833" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right"><a href="https://github.blog/enterprise-software/automation/" class="d-block f4 color-fg-default text-bold">Automation</a><p class="mb-0 f5 color-fg-muted">Automating your way to faster and more secure ships.</p></li>
<li id="menu-item-78834" class="col-6 d-block f4 color-fg-default color-border-subtle pl-4 pl-lg-8"><a href="https://github.blog/enterprise-software/ci-cd/" class="d-block f4 color-fg-default text-bold">CI/CD</a><p class="mb-0 f5 color-fg-muted">Guides on continuous integration and delivery.</p></li>
<li id="menu-item-78835" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right pt-2 pt-lg-4"><a href="https://github.blog/enterprise-software/collaboration/" class="d-block f4 color-fg-default text-bold">Collaboration</a><p class="mb-0 f5 color-fg-muted">Tips, tools, and tricks to improve developer collaboration.</p></li>
<li id="menu-item-78836" class="col-6 d-block f4 color-fg-default color-border-subtle pl-4 pl-lg-8 pt-2 pt-lg-4"><a href="https://github.blog/enterprise-software/devops/" class="d-block f4 color-fg-default text-bold">DevOps</a><p class="mb-0 f5 color-fg-muted">DevOps resources for enterprise engineering teams.</p></li>
<li id="menu-item-78837" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right pt-2 pt-lg-4"><a href="https://github.blog/enterprise-software/devsecops/" class="d-block f4 color-fg-default text-bold">DevSecOps</a><p class="mb-0 f5 color-fg-muted">How to integrate security into the SDLC.</p></li>
<li id="menu-item-78838" class="col-6 d-block f4 color-fg-default color-border-subtle pl-4 pl-lg-8 pt-2 pt-lg-4"><a href="https://github.blog/enterprise-software/governance-and-compliance/" class="d-block f4 color-fg-default text-bold">Governance &amp; compliance</a><p class="mb-0 f5 color-fg-muted">Ensuring your builds stay clean.</p></li>
</ul>
</li>
<li id="menu-item-78870" class="p-5 py-xl-7 px-xl-8 col-4 color-bg-subtle post-card"><div class="d-block position-relative mb-3 rounded-2 tease-thumbnail overflow-hidden"><img width="800" height="425" src="https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.43.47 AM.png?resize=800%2C425" class="tease-thumbnail__img d-block width-full height-auto aspect-ratio-1032-548 object-fit-cover" alt="" decoding="async" srcset="https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.43.47 AM.png?w=800 800w, https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.43.47 AM.png?w=400 400w, https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.43.47 AM.png?w=1032 1032w, https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.43.47 AM.png?w=516 516w" sizes="(max-width: 800px) 100vw, 800px" /></div><a href="https://github.com/resources/whitepapers/gartner-magic-quadrant-and-critical-capabilities-for-ai-code-assistants" class="d-block mb-2 f3 lh-condensed color-fg-default text-bold post-card__link">GitHub recognized as a Leader in the Gartner® Magic Quadrant™ for AI Code Assistants</a><p class="mb-3 f5 color-fg-muted">Learn why Gartner positioned GitHub as a Leader for the second year in a row.</p><span class="Link d-inline-flex flex-items-center" role="presentation">Learn more<svg xmlns="http://www.w3.org/2000/svg" class="octicon d-block ml-1" viewBox="0 0 12 12" width="12" height="12"><path d="M4.7 10c-.2 0-.4-.1-.5-.2-.3-.3-.3-.8 0-1.1L6.9 6 4.2 3.3c-.3-.3-.3-.8 0-1.1.3-.3.8-.3 1.1 0l3.3 3.2c.3.3.3.8 0 1.1L5.3 9.7c-.2.2-.4.3-.6.3Z"></path></svg></span></li>
</ul>
</li>
<li id="menu-item-78840"><a href="https://github.blog/news-insights/" class="position-relative d-flex flex-items-center flex-start no-wrap py-3 f4-mktg text-bold js-toggle" aria-controls="primary-78840-dropdown" aria-expanded="false" aria-haspopup="true" role="button">News &amp; insights<svg viewBox="0 0 16 16" width="16" height="16" class="octicon d-block ml-1 color-fg-muted" role="presentation"><path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path></svg></a>
<ul class="dropdown d-flex flex-wrap width-full position-absolute left-0 list-style-none mt-2 rounded-3 z-3 overflow-hidden color-border-subtle" aria-label="News &amp; insights sub-menu" aria-hidden="true" data-color-mode="light" data-dark-theme="dark" data-light-theme="light" id="primary-78840-dropdown" tabindex="-1">
<li id="menu-item-78871" class="flex-1 p-5 py-xl-7 px-xl-7"><div class="col-9 mb-4 mb-lg-7"><a href="https://github.blog/news-insights/" class="d-flex flex-items-center flex-start f3 lh-condensed color-fg-default text-bold">News &amp; insights<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right ml-1" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a><p class="mt-1 my-0 f4 color-fg-muted">Keep up with what’s new and notable from inside GitHub.</p></div>
<ul class="d-flex flex-wrap list-style-none" aria-label="News &amp; insights sub-menu">
<li id="menu-item-78841" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right"><a href="https://github.blog/news-insights/company-news/" class="d-block f4 color-fg-default text-bold">Company news</a><p class="mb-0 f5 color-fg-muted">An inside look at news and product updates from GitHub.</p></li>
<li id="menu-item-78844" class="col-6 d-block f4 color-fg-default color-border-subtle pl-4 pl-lg-8"><a href="https://github.blog/news-insights/product-news/" class="d-block f4 color-fg-default text-bold">Product</a><p class="mb-0 f5 color-fg-muted">The latest on GitHub’s platform, products, and tools.</p></li>
<li id="menu-item-78842" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right pt-2 pt-lg-4"><a href="https://github.blog/news-insights/octoverse/" class="d-block f4 color-fg-default text-bold">Octoverse</a><p class="mb-0 f5 color-fg-muted">Insights into the state of open source on GitHub.</p></li>
<li id="menu-item-78843" class="col-6 d-block f4 color-fg-default color-border-subtle pl-4 pl-lg-8 pt-2 pt-lg-4"><a href="https://github.blog/news-insights/policy-news-and-insights/" class="d-block f4 color-fg-default text-bold">Policy</a><p class="mb-0 f5 color-fg-muted">The latest policy and regulatory changes in software.</p></li>
<li id="menu-item-78845" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right pt-2 pt-lg-4"><a href="https://github.blog/news-insights/research/" class="d-block f4 color-fg-default text-bold">Research</a><p class="mb-0 f5 color-fg-muted">Data-driven insights around the developer ecosystem.</p></li>
<li id="menu-item-78847" class="col-6 d-block f4 color-fg-default color-border-subtle pl-4 pl-lg-8 pt-2 pt-lg-4"><a href="https://github.blog/news-insights/the-library/" class="d-block f4 color-fg-default text-bold">The library</a><p class="mb-0 f5 color-fg-muted">Older news and updates from GitHub.</p></li>
</ul>
</li>
<li id="menu-item-78872" class="p-5 py-xl-7 px-xl-8 col-4 color-bg-subtle post-card"><div class="d-block position-relative mb-3 rounded-2 tease-thumbnail overflow-hidden"><img width="800" height="425" src="https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.44.23 AM.png?resize=800%2C425" class="tease-thumbnail__img d-block width-full height-auto aspect-ratio-1032-548 object-fit-cover" alt="" decoding="async" srcset="https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.44.23 AM.png?w=300 300w, https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.44.23 AM.png?w=800 800w, https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.44.23 AM.png?w=400 400w, https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.44.23 AM.png?w=1032 1032w, https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.44.23 AM.png?w=516 516w" sizes="(max-width: 800px) 100vw, 800px" /></div><a href="https://github.blog/ai-and-ml/llms/unlocking-the-power-of-unstructured-data-with-rag/" class="d-block mb-2 f3 lh-condensed color-fg-default text-bold post-card__link">Unlocking the power of unstructured data with RAG</a><p class="mb-3 f5 color-fg-muted">Learn how to use retrieval-augmented generation (RAG) to capture more insights.</p><span class="Link d-inline-flex flex-items-center" role="presentation">Learn more<svg xmlns="http://www.w3.org/2000/svg" class="octicon d-block ml-1" viewBox="0 0 12 12" width="12" height="12"><path d="M4.7 10c-.2 0-.4-.1-.5-.2-.3-.3-.3-.8 0-1.1L6.9 6 4.2 3.3c-.3-.3-.3-.8 0-1.1.3-.3.8-.3 1.1 0l3.3 3.2c.3.3.3.8 0 1.1L5.3 9.7c-.2.2-.4.3-.6.3Z"></path></svg></span></li>
</ul>
</li>
<li id="menu-item-78848"><a href="https://github.blog/open-source/" class="position-relative d-flex flex-items-center flex-start no-wrap py-3 f4-mktg text-bold js-toggle" aria-controls="primary-78848-dropdown" aria-expanded="false" aria-haspopup="true" role="button">Open Source<svg viewBox="0 0 16 16" width="16" height="16" class="octicon d-block ml-1 color-fg-muted" role="presentation"><path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path></svg></a>
<ul class="dropdown d-flex flex-wrap width-full position-absolute left-0 list-style-none mt-2 rounded-3 z-3 overflow-hidden color-border-subtle" aria-label="Open Source sub-menu" aria-hidden="true" data-color-mode="light" data-dark-theme="dark" data-light-theme="light" id="primary-78848-dropdown" tabindex="-1">
<li id="menu-item-78873" class="flex-1 p-5 py-xl-7 px-xl-7"><div class="col-9 mb-4 mb-lg-7"><a href="https://github.blog/open-source/" class="d-flex flex-items-center flex-start f3 lh-condensed color-fg-default text-bold">Open Source<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right ml-1" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a><p class="mt-1 my-0 f4 color-fg-muted">Everything open source on GitHub.</p></div>
<ul class="d-flex flex-wrap list-style-none" aria-label="Open Source sub-menu">
<li id="menu-item-78851" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right"><a href="https://github.blog/open-source/git/" class="d-block f4 color-fg-default text-bold">Git</a><p class="mb-0 f5 color-fg-muted">The latest Git updates.</p></li>
<li id="menu-item-78853" class="col-6 d-block f4 color-fg-default color-border-subtle pl-4 pl-lg-8"><a href="https://github.blog/open-source/maintainers/" class="d-block f4 color-fg-default text-bold">Maintainers</a><p class="mb-0 f5 color-fg-muted">Spotlighting open source maintainers.</p></li>
<li id="menu-item-78854" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right pt-2 pt-lg-4"><a href="https://github.blog/open-source/social-impact/" class="d-block f4 color-fg-default text-bold">Social impact</a><p class="mb-0 f5 color-fg-muted">How open source is driving positive change.</p></li>
<li id="menu-item-78850" class="col-6 d-block f4 color-fg-default color-border-subtle pl-4 pl-lg-8 pt-2 pt-lg-4"><a href="https://github.blog/open-source/gaming/" class="d-block f4 color-fg-default text-bold">Gaming</a><p class="mb-0 f5 color-fg-muted">Explore open source games on GitHub.</p></li>
</ul>
</li>
<li id="menu-item-78874" class="p-5 py-xl-7 px-xl-8 col-4 color-bg-subtle post-card"><div class="d-block position-relative mb-3 rounded-2 tease-thumbnail overflow-hidden"><img width="800" height="425" src="https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.46.23 AM.png?resize=800%2C425" class="tease-thumbnail__img d-block width-full height-auto aspect-ratio-1032-548 object-fit-cover" alt="" decoding="async" srcset="https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.46.23 AM.png?w=800 800w, https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.46.23 AM.png?w=400 400w, https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.46.23 AM.png?w=1032 1032w, https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.46.23 AM.png?w=516 516w" sizes="(max-width: 800px) 100vw, 800px" /></div><a href="https://resources.github.com/software-development/innersource/" class="d-block mb-2 f3 lh-condensed color-fg-default text-bold post-card__link">An introduction to innersource</a><p class="mb-3 f5 color-fg-muted">Organizations worldwide are incorporating open source methodologies into the way they build and ship their own software.</p><span class="Link d-inline-flex flex-items-center" role="presentation">Learn more<svg xmlns="http://www.w3.org/2000/svg" class="octicon d-block ml-1" viewBox="0 0 12 12" width="12" height="12"><path d="M4.7 10c-.2 0-.4-.1-.5-.2-.3-.3-.3-.8 0-1.1L6.9 6 4.2 3.3c-.3-.3-.3-.8 0-1.1.3-.3.8-.3 1.1 0l3.3 3.2c.3.3.3.8 0 1.1L5.3 9.7c-.2.2-.4.3-.6.3Z"></path></svg></span></li>
</ul>
</li>
<li id="menu-item-78859"><a href="https://github.blog/security/" class="position-relative d-flex flex-items-center flex-start no-wrap py-3 f4-mktg text-bold js-toggle" aria-controls="primary-78859-dropdown" aria-expanded="false" aria-haspopup="true" role="button">Security<svg viewBox="0 0 16 16" width="16" height="16" class="octicon d-block ml-1 color-fg-muted" role="presentation"><path d="M12.78 5.22a.749.749 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.06 0L3.22 6.28a.749.749 0 1 1 1.06-1.06L8 8.939l3.72-3.719a.749.749 0 0 1 1.06 0Z"></path></svg></a>
<ul class="dropdown d-flex flex-wrap width-full position-absolute left-0 list-style-none mt-2 rounded-3 z-3 overflow-hidden color-border-subtle" aria-label="Security sub-menu" aria-hidden="true" data-color-mode="light" data-dark-theme="dark" data-light-theme="light" id="primary-78859-dropdown" tabindex="-1">
<li id="menu-item-78875" class="flex-1 p-5 py-xl-7 px-xl-7"><div class="col-9 mb-4 mb-lg-7"><a href="https://github.blog/security/" class="d-flex flex-items-center flex-start f3 lh-condensed color-fg-default text-bold">Security<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right ml-1" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a><p class="mt-1 my-0 f4 color-fg-muted">Stay up to date on everything security.</p></div>
<ul class="d-flex flex-wrap list-style-none" aria-label="Security sub-menu">
<li id="menu-item-78860" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right"><a href="https://github.blog/security/application-security/" class="d-block f4 color-fg-default text-bold">Application security</a><p class="mb-0 f5 color-fg-muted">Application security, explained.</p></li>
<li id="menu-item-78861" class="col-6 d-block f4 color-fg-default color-border-subtle pl-4 pl-lg-8"><a href="https://github.blog/security/supply-chain-security/" class="d-block f4 color-fg-default text-bold">Supply chain security</a><p class="mb-0 f5 color-fg-muted">Demystifying supply chain security.</p></li>
<li id="menu-item-78855" class="col-6 d-block f4 color-fg-default color-border-subtle pr-4 pr-lg-8 border-right pt-2 pt-lg-4"><a href="https://github.blog/security/vulnerability-research/" class="d-block f4 color-fg-default text-bold">Vulnerability research</a><p class="mb-0 f5 color-fg-muted">Updates from the GitHub Security Lab.</p></li>
<li id="menu-item-78862" class="col-6 d-block f4 color-fg-default color-border-subtle pl-4 pl-lg-8 pt-2 pt-lg-4"><a href="https://github.blog/security/web-application-security/" class="d-block f4 color-fg-default text-bold">Web application security</a><p class="mb-0 f5 color-fg-muted">Helpful tips on securing web applications.</p></li>
</ul>
</li>
<li id="menu-item-78876" class="p-5 py-xl-7 px-xl-8 col-4 color-bg-subtle post-card"><div class="d-block position-relative mb-3 rounded-2 tease-thumbnail overflow-hidden"><img width="800" height="425" src="https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.47.04 AM.png?resize=800%2C425" class="tease-thumbnail__img d-block width-full height-auto aspect-ratio-1032-548 object-fit-cover" alt="" decoding="async" srcset="https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.47.04 AM.png?w=800 800w, https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.47.04 AM.png?w=400 400w, https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.47.04 AM.png?w=1032 1032w, https://github.blog/wp-content/uploads/2024/07/Screenshot-2024-07-23-at-8.47.04 AM.png?w=516 516w" sizes="(max-width: 800px) 100vw, 800px" /></div><a href="https://resources.github.com/security/the-enterprise-guide-to-ai-powered-devsecops/" class="d-block mb-2 f3 lh-condensed color-fg-default text-bold post-card__link">The enterprise guide to AI-powered DevSecOps</a><p class="mb-3 f5 color-fg-muted">Learn about core challenges in DevSecOps, and how you can start addressing them with AI and automation.</p><span class="Link d-inline-flex flex-items-center" role="presentation">Learn more<svg xmlns="http://www.w3.org/2000/svg" class="octicon d-block ml-1" viewBox="0 0 12 12" width="12" height="12"><path d="M4.7 10c-.2 0-.4-.1-.5-.2-.3-.3-.3-.8 0-1.1L6.9 6 4.2 3.3c-.3-.3-.3-.8 0-1.1.3-.3.8-.3 1.1 0l3.3 3.2c.3.3.3.8 0 1.1L5.3 9.7c-.2.2-.4.3-.6.3Z"></path></svg></span></li>
</ul>
</li>
</ul> </div>
</nav>
</div>
<form id="mobile-search" role="search" method="get" class="mobile-search" action="https://github.blog" aria-hidden="true" aria-label="Search form">
<div class="d-flex flex-1 p-3 color-bg-inset">
<div class="d-flex flex-1 position-relative color-bg-transparent" data-color-mode="light" data-light-theme="light" data-dark-theme="dark" >
<svg height="20" class="d-flex position-absolute z-3 octicon height-full ml-2 color-fg-subtle" aria-hidden="true" viewBox="0 0 16 16" version="1.1" width="20" role="img"><path fill-rule="evenodd" d="M11.5 7a4.499 4.499 0 11-8.998 0A4.499 4.499 0 0111.5 7zm-.82 4.74a6 6 0 111.06-1.06l3.04 3.04a.75.75 0 11-1.06 1.06l-3.04-3.04z"></path></svg>
<input aria-label="Search the blog" type="search" class="pl-6 search-field form-control p-2 flex-1" placeholder="Search the blog…" value="" name="s" id="search-input">
</div>
<button class="Button Button--size-medium Button--secondary ml-2"
type="submit" >
<span class="Button__text">
<span class="Text Text--200 Text--antialiased Text--weight-semibold Button--label Button--label-medium Button--label-secondary">
Search </span>
</span>
</button>
</div>
</form>
<nav id="mobile-menu" class="mobile-menu position-relative overflow-y-auto flex-1 width-full rounded-top-3" aria-label="Navigation menu" aria-hidden="true" data-color-mode="light" data-light-theme="light" data-dark-theme="dark_dimmed">
<div class="p-5">
<h2 class="mb-5 text-bold color-fg-subtle">Categories</h2>
<ul id="menu-new-primary-navigation" class="list-style-none"><li class="mb-5"><a href="https://github.blog/ai-and-ml/" class="d-flex flex-items-center flex-justify-between lh-condensed-ultra text-bold color-fg-default js-toggle" aria-controls="primary-mobile-78814-dropdown" aria-expanded="false" aria-haspopup="true" role="button">AI &amp; ML<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right d-block color-fg-subtle" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a>
<ul aria-hidden="true" aria-label="AI &amp; ML sub-menu" class="dropdown overflow-y-auto position-absolute top-0 left-0 right-0 bottom-0 flex-wrap p-5 list-style-none rounded-3 z-3 color-bg-default color-border-subtle" id="primary-mobile-78814-dropdown">
<li class="mb-5"><div class="mb-4"><button type="button" class="d-flex flex-items-center mb-4 p-0 border-0 text-semibold color-bg-transparent color-fg-subtle" aria-controls="primary-mobile-78814-dropdown" aria-expanded="true"><svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-left mr-1"><path d="M9.78 12.78a.75.75 0 0 1-1.06 0L4.47 8.53a.75.75 0 0 1 0-1.06l4.25-4.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L6.06 8l3.72 3.72a.75.75 0 0 1 0 1.06Z"></path></svg>Back</button><a href="https://github.blog/ai-and-ml/" class="d-flex flex-items-center flex-justify-start lh-condensed-ultra text-bold color-fg-default">AI &amp; ML<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right ml-1" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a><p class="mt-2 mb-0 color-fg-muted">Learn about artificial intelligence and machine learning across the GitHub ecosystem and the wider industry.</p></div>
<ul class="border-top list-style-none">
<li class="mt-4"><a href="https://github.blog/ai-and-ml/generative-ai/" class="text-bold lh-condensed-ultra color-fg-default">Generative AI</a><p class="mb-0 f5 color-fg-muted">Learn how to build with generative AI.</p></li>
<li class="mt-4"><a href="https://github.blog/ai-and-ml/github-copilot/" class="text-bold lh-condensed-ultra color-fg-default">GitHub Copilot</a><p class="mb-0 f5 color-fg-muted">Change how you work with GitHub Copilot.</p></li>
<li class="mt-4"><a href="https://github.blog/ai-and-ml/llms/" class="text-bold lh-condensed-ultra color-fg-default">LLMs</a><p class="mb-0 f5 color-fg-muted">Everything developers need to know about LLMs.</p></li>
<li class="mt-4"><a href="https://github.blog/ai-and-ml/machine-learning/" class="text-bold lh-condensed-ultra color-fg-default">Machine learning</a><p class="mb-0 f5 color-fg-muted">Machine learning tips, tricks, and best practices.</p></li>
</ul>
</li>
<li class="d-none"><div class="mb-4"><a href="https://github.blog/ai-and-ml/generative-ai/how-ai-code-generation-works/" class="d-flex flex-items-center flex-justify-start lh-condensed-ultra text-bold color-fg-default">How AI code generation works</a><p class="mb-3 f5 color-fg-muted">Explore the capabilities and benefits of AI code generation and how it can improve your developer experience.</p><a href="https://github.blog/ai-and-ml/generative-ai/how-ai-code-generation-works/" target="" class="Link d-inline-flex flex-items-center">Learn more<svg xmlns="http://www.w3.org/2000/svg" class="octicon d-block ml-1" viewBox="0 0 12 12" width="12" height="12"><path d="M4.7 10c-.2 0-.4-.1-.5-.2-.3-.3-.3-.8 0-1.1L6.9 6 4.2 3.3c-.3-.3-.3-.8 0-1.1.3-.3.8-.3 1.1 0l3.3 3.2c.3.3.3.8 0 1.1L5.3 9.7c-.2.2-.4.3-.6.3Z"></path></svg></a></div></li>
</ul>
</li>
<li class="mb-5"><a href="https://github.blog/developer-skills/" class="d-flex flex-items-center flex-justify-between lh-condensed-ultra text-bold color-fg-default js-toggle" aria-controls="primary-mobile-78819-dropdown" aria-expanded="false" aria-haspopup="true" role="button">Developer skills<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right d-block color-fg-subtle" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a>
<ul aria-hidden="true" aria-label="Developer skills sub-menu" class="dropdown overflow-y-auto position-absolute top-0 left-0 right-0 bottom-0 flex-wrap p-5 list-style-none rounded-3 z-3 color-bg-default color-border-subtle" id="primary-mobile-78819-dropdown">
<li class="mb-5"><div class="mb-4"><button type="button" class="d-flex flex-items-center mb-4 p-0 border-0 text-semibold color-bg-transparent color-fg-subtle" aria-controls="primary-mobile-78819-dropdown" aria-expanded="true"><svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-left mr-1"><path d="M9.78 12.78a.75.75 0 0 1-1.06 0L4.47 8.53a.75.75 0 0 1 0-1.06l4.25-4.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L6.06 8l3.72 3.72a.75.75 0 0 1 0 1.06Z"></path></svg>Back</button><a href="https://github.blog/developer-skills/" class="d-flex flex-items-center flex-justify-start lh-condensed-ultra text-bold color-fg-default">Developer skills<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right ml-1" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a><p class="mt-2 mb-0 color-fg-muted">Resources for developers to grow in their skills and careers.</p></div>
<ul class="border-top list-style-none">
<li class="mt-4"><a href="https://github.blog/developer-skills/application-development/" class="text-bold lh-condensed-ultra color-fg-default">Application development</a><p class="mb-0 f5 color-fg-muted">Insights and best practices for building apps.</p></li>
<li class="mt-4"><a href="https://github.blog/developer-skills/career-growth/" class="text-bold lh-condensed-ultra color-fg-default">Career growth</a><p class="mb-0 f5 color-fg-muted">Tips &amp; tricks to grow as a professional developer.</p></li>
<li class="mt-4"><a href="https://github.blog/developer-skills/github/" class="text-bold lh-condensed-ultra color-fg-default">GitHub</a><p class="mb-0 f5 color-fg-muted">Improve how you use GitHub at work.</p></li>
<li class="mt-4"><a href="https://github.blog/developer-skills/github-education/" class="text-bold lh-condensed-ultra color-fg-default">GitHub Education</a><p class="mb-0 f5 color-fg-muted">Learn how to move into your first professional role.</p></li>
<li class="mt-4"><a href="https://github.blog/developer-skills/programming-languages-and-frameworks/" class="text-bold lh-condensed-ultra color-fg-default">Programming languages &amp; frameworks</a><p class="mb-0 f5 color-fg-muted">Stay current on what’s new (or new again).</p></li>
</ul>
</li>
<li class="d-none"><div class="mb-4"><a href="https://docs.github.com/en/get-started" class="d-flex flex-items-center flex-justify-start lh-condensed-ultra text-bold color-fg-default">Get started with GitHub documentation</a><p class="mb-3 f5 color-fg-muted">Learn how to start building, shipping, and maintaining software with GitHub.</p><a href="https://docs.github.com/en/get-started" target="" class="Link d-inline-flex flex-items-center">Learn more<svg xmlns="http://www.w3.org/2000/svg" class="octicon d-block ml-1" viewBox="0 0 12 12" width="12" height="12"><path d="M4.7 10c-.2 0-.4-.1-.5-.2-.3-.3-.3-.8 0-1.1L6.9 6 4.2 3.3c-.3-.3-.3-.8 0-1.1.3-.3.8-.3 1.1 0l3.3 3.2c.3.3.3.8 0 1.1L5.3 9.7c-.2.2-.4.3-.6.3Z"></path></svg></a></div></li>
</ul>
</li>
<li class="mb-5"><a href="https://github.blog/engineering/" class="d-flex flex-items-center flex-justify-between lh-condensed-ultra text-bold color-fg-default js-toggle" aria-controls="primary-mobile-78825-dropdown" aria-expanded="false" aria-haspopup="true" role="button">Engineering<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right d-block color-fg-subtle" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a>
<ul aria-hidden="true" aria-label="Engineering sub-menu" class="dropdown overflow-y-auto position-absolute top-0 left-0 right-0 bottom-0 flex-wrap p-5 list-style-none rounded-3 z-3 color-bg-default color-border-subtle" id="primary-mobile-78825-dropdown">
<li class="mb-5"><div class="mb-4"><button type="button" class="d-flex flex-items-center mb-4 p-0 border-0 text-semibold color-bg-transparent color-fg-subtle" aria-controls="primary-mobile-78825-dropdown" aria-expanded="true"><svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-left mr-1"><path d="M9.78 12.78a.75.75 0 0 1-1.06 0L4.47 8.53a.75.75 0 0 1 0-1.06l4.25-4.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L6.06 8l3.72 3.72a.75.75 0 0 1 0 1.06Z"></path></svg>Back</button><a href="https://github.blog/engineering/" class="d-flex flex-items-center flex-justify-start lh-condensed-ultra text-bold color-fg-default">Engineering<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right ml-1" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a><p class="mt-2 mb-0 color-fg-muted">Get an inside look at how we’re building the home for all developers.</p></div>
<ul class="border-top list-style-none">
<li class="mt-4"><a href="https://github.blog/engineering/architecture-optimization/" class="text-bold lh-condensed-ultra color-fg-default">Architecture &amp; optimization</a><p class="mb-0 f5 color-fg-muted">Discover how we deliver a performant and highly available experience across the GitHub platform.</p></li>
<li class="mt-4"><a href="https://github.blog/engineering/engineering-principles/" class="text-bold lh-condensed-ultra color-fg-default">Engineering principles</a><p class="mb-0 f5 color-fg-muted">Explore best practices for building software at scale with a majority remote team.</p></li>
<li class="mt-4"><a href="https://github.blog/engineering/infrastructure/" class="text-bold lh-condensed-ultra color-fg-default">Infrastructure</a><p class="mb-0 f5 color-fg-muted">Get a glimpse at the technology underlying the world’s leading AI-powered developer platform.</p></li>
<li class="mt-4"><a href="https://github.blog/engineering/platform-security/" class="text-bold lh-condensed-ultra color-fg-default">Platform security</a><p class="mb-0 f5 color-fg-muted">Learn how we build security into everything we do across the developer lifecycle.</p></li>
<li class="mt-4"><a href="https://github.blog/engineering/user-experience/" class="text-bold lh-condensed-ultra color-fg-default">User experience</a><p class="mb-0 f5 color-fg-muted">Find out what goes into making GitHub the home for all developers.</p></li>
</ul>
</li>
<li class="d-none"><div class="mb-4"><a href="https://github.blog/engineering/how-we-use-github-to-be-more-productive-collaborative-and-secure/" class="d-flex flex-items-center flex-justify-start lh-condensed-ultra text-bold color-fg-default">How we use GitHub to be more productive, collaborative, and secure</a><p class="mb-3 f5 color-fg-muted">Our engineering and security teams do some incredible work. Let’s take a look at how we use GitHub to be more productive, build collaboratively, and shift security left.</p><a href="https://github.blog/engineering/how-we-use-github-to-be-more-productive-collaborative-and-secure/" target="" class="Link d-inline-flex flex-items-center">Learn more<svg xmlns="http://www.w3.org/2000/svg" class="octicon d-block ml-1" viewBox="0 0 12 12" width="12" height="12"><path d="M4.7 10c-.2 0-.4-.1-.5-.2-.3-.3-.3-.8 0-1.1L6.9 6 4.2 3.3c-.3-.3-.3-.8 0-1.1.3-.3.8-.3 1.1 0l3.3 3.2c.3.3.3.8 0 1.1L5.3 9.7c-.2.2-.4.3-.6.3Z"></path></svg></a></div></li>
</ul>
</li>
<li class="mb-5"><a href="https://github.blog/enterprise-software/" class="d-flex flex-items-center flex-justify-between lh-condensed-ultra text-bold color-fg-default js-toggle" aria-controls="primary-mobile-78832-dropdown" aria-expanded="false" aria-haspopup="true" role="button">Enterprise software<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right d-block color-fg-subtle" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a>
<ul aria-hidden="true" aria-label="Enterprise software sub-menu" class="dropdown overflow-y-auto position-absolute top-0 left-0 right-0 bottom-0 flex-wrap p-5 list-style-none rounded-3 z-3 color-bg-default color-border-subtle" id="primary-mobile-78832-dropdown">
<li class="mb-5"><div class="mb-4"><button type="button" class="d-flex flex-items-center mb-4 p-0 border-0 text-semibold color-bg-transparent color-fg-subtle" aria-controls="primary-mobile-78832-dropdown" aria-expanded="true"><svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-left mr-1"><path d="M9.78 12.78a.75.75 0 0 1-1.06 0L4.47 8.53a.75.75 0 0 1 0-1.06l4.25-4.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L6.06 8l3.72 3.72a.75.75 0 0 1 0 1.06Z"></path></svg>Back</button><a href="https://github.blog/enterprise-software/" class="d-flex flex-items-center flex-justify-start lh-condensed-ultra text-bold color-fg-default">Enterprise software<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right ml-1" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a><p class="mt-2 mb-0 color-fg-muted">Explore how to write, build, and deploy enterprise software at scale.</p></div>
<ul class="border-top list-style-none">
<li class="mt-4"><a href="https://github.blog/enterprise-software/automation/" class="text-bold lh-condensed-ultra color-fg-default">Automation</a><p class="mb-0 f5 color-fg-muted">Automating your way to faster and more secure ships.</p></li>
<li class="mt-4"><a href="https://github.blog/enterprise-software/ci-cd/" class="text-bold lh-condensed-ultra color-fg-default">CI/CD</a><p class="mb-0 f5 color-fg-muted">Guides on continuous integration and delivery.</p></li>
<li class="mt-4"><a href="https://github.blog/enterprise-software/collaboration/" class="text-bold lh-condensed-ultra color-fg-default">Collaboration</a><p class="mb-0 f5 color-fg-muted">Tips, tools, and tricks to improve developer collaboration.</p></li>
<li class="mt-4"><a href="https://github.blog/enterprise-software/devops/" class="text-bold lh-condensed-ultra color-fg-default">DevOps</a><p class="mb-0 f5 color-fg-muted">DevOps resources for enterprise engineering teams.</p></li>
<li class="mt-4"><a href="https://github.blog/enterprise-software/devsecops/" class="text-bold lh-condensed-ultra color-fg-default">DevSecOps</a><p class="mb-0 f5 color-fg-muted">How to integrate security into the SDLC.</p></li>
<li class="mt-4"><a href="https://github.blog/enterprise-software/governance-and-compliance/" class="text-bold lh-condensed-ultra color-fg-default">Governance &amp; compliance</a><p class="mb-0 f5 color-fg-muted">Ensuring your builds stay clean.</p></li>
</ul>
</li>
<li class="d-none"><div class="mb-4"><a href="https://github.com/resources/whitepapers/gartner-magic-quadrant-and-critical-capabilities-for-ai-code-assistants" class="d-flex flex-items-center flex-justify-start lh-condensed-ultra text-bold color-fg-default">GitHub recognized as a Leader in the Gartner® Magic Quadrant™ for AI Code Assistants</a><p class="mb-3 f5 color-fg-muted">Learn why Gartner positioned GitHub as a Leader for the second year in a row.</p><a href="https://github.com/resources/whitepapers/gartner-magic-quadrant-and-critical-capabilities-for-ai-code-assistants" target="" class="Link d-inline-flex flex-items-center">Learn more<svg xmlns="http://www.w3.org/2000/svg" class="octicon d-block ml-1" viewBox="0 0 12 12" width="12" height="12"><path d="M4.7 10c-.2 0-.4-.1-.5-.2-.3-.3-.3-.8 0-1.1L6.9 6 4.2 3.3c-.3-.3-.3-.8 0-1.1.3-.3.8-.3 1.1 0l3.3 3.2c.3.3.3.8 0 1.1L5.3 9.7c-.2.2-.4.3-.6.3Z"></path></svg></a></div></li>
</ul>
</li>
<li class="mb-5"><a href="https://github.blog/news-insights/" class="d-flex flex-items-center flex-justify-between lh-condensed-ultra text-bold color-fg-default js-toggle" aria-controls="primary-mobile-78840-dropdown" aria-expanded="false" aria-haspopup="true" role="button">News &amp; insights<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right d-block color-fg-subtle" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a>
<ul aria-hidden="true" aria-label="News &amp; insights sub-menu" class="dropdown overflow-y-auto position-absolute top-0 left-0 right-0 bottom-0 flex-wrap p-5 list-style-none rounded-3 z-3 color-bg-default color-border-subtle" id="primary-mobile-78840-dropdown">
<li class="mb-5"><div class="mb-4"><button type="button" class="d-flex flex-items-center mb-4 p-0 border-0 text-semibold color-bg-transparent color-fg-subtle" aria-controls="primary-mobile-78840-dropdown" aria-expanded="true"><svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-left mr-1"><path d="M9.78 12.78a.75.75 0 0 1-1.06 0L4.47 8.53a.75.75 0 0 1 0-1.06l4.25-4.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L6.06 8l3.72 3.72a.75.75 0 0 1 0 1.06Z"></path></svg>Back</button><a href="https://github.blog/news-insights/" class="d-flex flex-items-center flex-justify-start lh-condensed-ultra text-bold color-fg-default">News &amp; insights<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right ml-1" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a><p class="mt-2 mb-0 color-fg-muted">Keep up with what’s new and notable from inside GitHub.</p></div>
<ul class="border-top list-style-none">
<li class="mt-4"><a href="https://github.blog/news-insights/company-news/" class="text-bold lh-condensed-ultra color-fg-default">Company news</a><p class="mb-0 f5 color-fg-muted">An inside look at news and product updates from GitHub.</p></li>
<li class="mt-4"><a href="https://github.blog/news-insights/product-news/" class="text-bold lh-condensed-ultra color-fg-default">Product</a><p class="mb-0 f5 color-fg-muted">The latest on GitHub’s platform, products, and tools.</p></li>
<li class="mt-4"><a href="https://github.blog/news-insights/octoverse/" class="text-bold lh-condensed-ultra color-fg-default">Octoverse</a><p class="mb-0 f5 color-fg-muted">Insights into the state of open source on GitHub.</p></li>
<li class="mt-4"><a href="https://github.blog/news-insights/policy-news-and-insights/" class="text-bold lh-condensed-ultra color-fg-default">Policy</a><p class="mb-0 f5 color-fg-muted">The latest policy and regulatory changes in software.</p></li>
<li class="mt-4"><a href="https://github.blog/news-insights/research/" class="text-bold lh-condensed-ultra color-fg-default">Research</a><p class="mb-0 f5 color-fg-muted">Data-driven insights around the developer ecosystem.</p></li>
<li class="mt-4"><a href="https://github.blog/news-insights/the-library/" class="text-bold lh-condensed-ultra color-fg-default">The library</a><p class="mb-0 f5 color-fg-muted">Older news and updates from GitHub.</p></li>
</ul>
</li>
<li class="d-none"><div class="mb-4"><a href="https://github.blog/ai-and-ml/llms/unlocking-the-power-of-unstructured-data-with-rag/" class="d-flex flex-items-center flex-justify-start lh-condensed-ultra text-bold color-fg-default">Unlocking the power of unstructured data with RAG</a><p class="mb-3 f5 color-fg-muted">Learn how to use retrieval-augmented generation (RAG) to capture more insights.</p><a href="https://github.blog/ai-and-ml/llms/unlocking-the-power-of-unstructured-data-with-rag/" target="" class="Link d-inline-flex flex-items-center">Learn more<svg xmlns="http://www.w3.org/2000/svg" class="octicon d-block ml-1" viewBox="0 0 12 12" width="12" height="12"><path d="M4.7 10c-.2 0-.4-.1-.5-.2-.3-.3-.3-.8 0-1.1L6.9 6 4.2 3.3c-.3-.3-.3-.8 0-1.1.3-.3.8-.3 1.1 0l3.3 3.2c.3.3.3.8 0 1.1L5.3 9.7c-.2.2-.4.3-.6.3Z"></path></svg></a></div></li>
</ul>
</li>
<li class="mb-5"><a href="https://github.blog/open-source/" class="d-flex flex-items-center flex-justify-between lh-condensed-ultra text-bold color-fg-default js-toggle" aria-controls="primary-mobile-78848-dropdown" aria-expanded="false" aria-haspopup="true" role="button">Open Source<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right d-block color-fg-subtle" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a>
<ul aria-hidden="true" aria-label="Open Source sub-menu" class="dropdown overflow-y-auto position-absolute top-0 left-0 right-0 bottom-0 flex-wrap p-5 list-style-none rounded-3 z-3 color-bg-default color-border-subtle" id="primary-mobile-78848-dropdown">
<li class="mb-5"><div class="mb-4"><button type="button" class="d-flex flex-items-center mb-4 p-0 border-0 text-semibold color-bg-transparent color-fg-subtle" aria-controls="primary-mobile-78848-dropdown" aria-expanded="true"><svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-left mr-1"><path d="M9.78 12.78a.75.75 0 0 1-1.06 0L4.47 8.53a.75.75 0 0 1 0-1.06l4.25-4.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L6.06 8l3.72 3.72a.75.75 0 0 1 0 1.06Z"></path></svg>Back</button><a href="https://github.blog/open-source/" class="d-flex flex-items-center flex-justify-start lh-condensed-ultra text-bold color-fg-default">Open Source<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right ml-1" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a><p class="mt-2 mb-0 color-fg-muted">Everything open source on GitHub.</p></div>
<ul class="border-top list-style-none">
<li class="mt-4"><a href="https://github.blog/open-source/git/" class="text-bold lh-condensed-ultra color-fg-default">Git</a><p class="mb-0 f5 color-fg-muted">The latest Git updates.</p></li>
<li class="mt-4"><a href="https://github.blog/open-source/maintainers/" class="text-bold lh-condensed-ultra color-fg-default">Maintainers</a><p class="mb-0 f5 color-fg-muted">Spotlighting open source maintainers.</p></li>
<li class="mt-4"><a href="https://github.blog/open-source/social-impact/" class="text-bold lh-condensed-ultra color-fg-default">Social impact</a><p class="mb-0 f5 color-fg-muted">How open source is driving positive change.</p></li>
<li class="mt-4"><a href="https://github.blog/open-source/gaming/" class="text-bold lh-condensed-ultra color-fg-default">Gaming</a><p class="mb-0 f5 color-fg-muted">Explore open source games on GitHub.</p></li>
</ul>
</li>
<li class="d-none"><div class="mb-4"><a href="https://resources.github.com/software-development/innersource/" class="d-flex flex-items-center flex-justify-start lh-condensed-ultra text-bold color-fg-default">An introduction to innersource</a><p class="mb-3 f5 color-fg-muted">Organizations worldwide are incorporating open source methodologies into the way they build and ship their own software.</p><a href="https://resources.github.com/software-development/innersource/" target="" class="Link d-inline-flex flex-items-center">Learn more<svg xmlns="http://www.w3.org/2000/svg" class="octicon d-block ml-1" viewBox="0 0 12 12" width="12" height="12"><path d="M4.7 10c-.2 0-.4-.1-.5-.2-.3-.3-.3-.8 0-1.1L6.9 6 4.2 3.3c-.3-.3-.3-.8 0-1.1.3-.3.8-.3 1.1 0l3.3 3.2c.3.3.3.8 0 1.1L5.3 9.7c-.2.2-.4.3-.6.3Z"></path></svg></a></div></li>
</ul>
</li>
<li class="mb-5"><a href="https://github.blog/security/" class="d-flex flex-items-center flex-justify-between lh-condensed-ultra text-bold color-fg-default js-toggle" aria-controls="primary-mobile-78859-dropdown" aria-expanded="false" aria-haspopup="true" role="button">Security<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right d-block color-fg-subtle" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a>
<ul aria-hidden="true" aria-label="Security sub-menu" class="dropdown overflow-y-auto position-absolute top-0 left-0 right-0 bottom-0 flex-wrap p-5 list-style-none rounded-3 z-3 color-bg-default color-border-subtle" id="primary-mobile-78859-dropdown">
<li class="mb-5"><div class="mb-4"><button type="button" class="d-flex flex-items-center mb-4 p-0 border-0 text-semibold color-bg-transparent color-fg-subtle" aria-controls="primary-mobile-78859-dropdown" aria-expanded="true"><svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-left mr-1"><path d="M9.78 12.78a.75.75 0 0 1-1.06 0L4.47 8.53a.75.75 0 0 1 0-1.06l4.25-4.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042L6.06 8l3.72 3.72a.75.75 0 0 1 0 1.06Z"></path></svg>Back</button><a href="https://github.blog/security/" class="d-flex flex-items-center flex-justify-start lh-condensed-ultra text-bold color-fg-default">Security<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-chevron-right ml-1" role="presentation"><path d="M6.22 3.22a.75.75 0 0 1 1.06 0l4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L9.94 8 6.22 4.28a.75.75 0 0 1 0-1.06Z"></path></svg></a><p class="mt-2 mb-0 color-fg-muted">Stay up to date on everything security.</p></div>
<ul class="border-top list-style-none">
<li class="mt-4"><a href="https://github.blog/security/application-security/" class="text-bold lh-condensed-ultra color-fg-default">Application security</a><p class="mb-0 f5 color-fg-muted">Application security, explained.</p></li>
<li class="mt-4"><a href="https://github.blog/security/supply-chain-security/" class="text-bold lh-condensed-ultra color-fg-default">Supply chain security</a><p class="mb-0 f5 color-fg-muted">Demystifying supply chain security.</p></li>
<li class="mt-4"><a href="https://github.blog/security/vulnerability-research/" class="text-bold lh-condensed-ultra color-fg-default">Vulnerability research</a><p class="mb-0 f5 color-fg-muted">Updates from the GitHub Security Lab.</p></li>
<li class="mt-4"><a href="https://github.blog/security/web-application-security/" class="text-bold lh-condensed-ultra color-fg-default">Web application security</a><p class="mb-0 f5 color-fg-muted">Helpful tips on securing web applications.</p></li>
</ul>
</li>
<li class="d-none"><div class="mb-4"><a href="https://resources.github.com/security/the-enterprise-guide-to-ai-powered-devsecops/" class="d-flex flex-items-center flex-justify-start lh-condensed-ultra text-bold color-fg-default">The enterprise guide to AI-powered DevSecOps</a><p class="mb-3 f5 color-fg-muted">Learn about core challenges in DevSecOps, and how you can start addressing them with AI and automation.</p><a href="https://resources.github.com/security/the-enterprise-guide-to-ai-powered-devsecops/" target="" class="Link d-inline-flex flex-items-center">Learn more<svg xmlns="http://www.w3.org/2000/svg" class="octicon d-block ml-1" viewBox="0 0 12 12" width="12" height="12"><path d="M4.7 10c-.2 0-.4-.1-.5-.2-.3-.3-.3-.8 0-1.1L6.9 6 4.2 3.3c-.3-.3-.3-.8 0-1.1.3-.3.8-.3 1.1 0l3.3 3.2c.3.3.3.8 0 1.1L5.3 9.7c-.2.2-.4.3-.6.3Z"></path></svg></a></div></li>
</ul>
</li>
</ul> <ul id="menu-secondary-navigation" class="pt-5 border-top list-style-none"><li class="mb-5"><a href="https://github.blog/changelog/" class="d-flex flex-items-center flex-justify-between lh-condensed-ultra text-bold color-fg-default">Changelog</a></li>
<li class="mb-5"><a href="https://docs.github.com/" class="d-flex flex-items-center flex-justify-between lh-condensed-ultra text-bold color-fg-default">Docs<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-link-external d-block color-fg-subtle" role="presentation"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li>
<li class="mb-5"><a href="https://github.com/customer-stories" class="d-flex flex-items-center flex-justify-between lh-condensed-ultra text-bold color-fg-default">Customer stories<svg viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-link-external d-block color-fg-subtle" role="presentation"><path d="M3.75 2h3.5a.75.75 0 0 1 0 1.5h-3.5a.25.25 0 0 0-.25.25v8.5c0 .138.112.25.25.25h8.5a.25.25 0 0 0 .25-.25v-3.5a.75.75 0 0 1 1.5 0v3.5A1.75 1.75 0 0 1 12.25 14h-8.5A1.75 1.75 0 0 1 2 12.25v-8.5C2 2.784 2.784 2 3.75 2Zm6.854-1h4.146a.25.25 0 0 1 .25.25v4.146a.25.25 0 0 1-.427.177L13.03 4.03 9.28 7.78a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042l3.75-3.75-1.543-1.543A.25.25 0 0 1 10.604 1Z"></path></svg></a></li>
</ul> <a class="Button Button--size-medium Button--secondary Button--block my-3"
href="https://githubuniverse.com/?utm_source=Blog-button&#038;utm_medium=GitHub&#038;utm_campaign=blog_button_seb_uni_26" target="_blank" data-analytics-click="Blog, click on button, text: Attend Universe; ref_location:top nav;" >
<span class="Button__text">
<span class="Text Text--200 Text--antialiased Text--weight-semibold Button--label Button--label-medium Button--label-secondary">
Attend GitHub Universe </span>
</span>
</a>
<a class="Button Button--size-medium Button--primary Button--block"
href="https://github.com/features/ai/github-app?utm_source=blog-header-button-cta&#038;utm_medium=blog&#038;utm_campaign=github-app-cli-Q1-push" target="_blank" data-analytics-click="Blog, click on button, text: Use Copilot for free; ref_location:top nav;" >
<span class="Button__text">
<span class="Text Text--200 Text--antialiased Text--weight-semibold Button--label Button--label-medium Button--label-primary">
Try GitHub Copilot app </span>
</span>
</a>
</div>
<button class="mobile-nav-focus-trap js-mobile-nav-focus-trap" data-target="#mobile-menu-toggle" data-focus-order="target"></button>
</nav>
</header>
</div>
<main id="start-of-content">
<header class="position-relative" data-color-mode="dark" data-light-theme="light" data-dark-theme="dark_dimmed">
<div class="container-xl p-responsive-blog">
<div class="gutter-spacious">
<div class="col-12 offset-lg-1 col-lg-10 col-xl-8 py-5 py-md-6 ">
<div class="f5 mb-5 mb-md-8"><span><span><a class="Link--primary" href="https://github.blog/">Home</a></span> <span class="mx-2 opacity-5">/</span> <span><a class="Link--primary" href="https://github.blog/engineering/">Engineering</a></span> <span class="mx-2 opacity-5">/</span> <span><a class="Link--primary" href="https://github.blog/engineering/infrastructure/">Infrastructure</a></span></span></div> <h1 class="h3-mktg lh-condensed mb-3 color-fg-default">Debugging network stalls on Kubernetes</h1> <div class="f4-mktg color-fg-muted">
<p>In this deep-dive, we identified and worked through sporadic latency issues with services running on Kubernetes in our environment.</p>
</div>
</div>
<div class="offset-lg-1 col-lg-10">
<div class="position-relative overflow-hidden rounded-2 z-1">
<svg aria-hidden="true" width="1032" height="548" class=" d-block width-full height-auto" role="presentation"></svg>
<img width="1201" height="630" src="https://github.blog/wp-content/uploads/2019/03/engineering-social.png?resize=1201%2C630" class="d-block cover-image wp-post-image" alt="" decoding="async" loading="lazy" srcset="https://github.blog/wp-content/uploads/2019/03/engineering-social.png?w=1201 1201w, https://github.blog/wp-content/uploads/2019/03/engineering-social.png?w=300 300w, https://github.blog/wp-content/uploads/2019/03/engineering-social.png?w=768 768w, https://github.blog/wp-content/uploads/2019/03/engineering-social.png?w=1024 1024w" sizes="auto, (max-width: 1201px) 100vw, 1201px" /> </div>
</div>
</div>
</div>
<div class="position-absolute bottom-0 width-full" style="background:#fff; height:80px;"></div>
</header>
<section class="container-xl mx-auto p-responsive-blog mt-4">
<div class="gutter-spacious">
<div class="col-12 offset-lg-1 col-lg-10">
<div class="d-lg-flex flex-justify-between flex-items-center">
<div class="mb-4 mb-lg-0">
<div class="mb-1">
<div class="d-flex flex-items-center mb-6px">
<span>
<span class="text-bold">
<a href="https://github.blog/author/theojulienne/" title="Posts by Theo Julienne" class="author url fn Link--primary no-wrap position-relative z-2" rel="author">Theo Julienne</a><span class="mx-2">·</span><a target="_blank" href="https://github.com/theojulienne" class="Link--primary" aria-label="GitHub profile of theojulienne">@theojulienne</a> </span>
</span>
</div>
</div>
<div data-color-mode="auto" data-light-theme="light_tritanopia" class="d-flex flex-column flex-md-row text-mono color-fg-muted">
<time datetime="2019-11-21" class="d-block">
November 21, 2019 </time>
<span class="d-none d-md-block mx-3">|</span>
<time datetime="2020-03-22" class="d-block mt-1 mt-md-0">
Updated March 22, 2020 </time>
<div class="d-flex flex-items-center mt-2 mt-md-0">
<span class="d-none d-md-block mx-3">|</span>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16" class="mr-2">
<path fill="currentcolor" d="M8 0a8 8 0 1 1 0 16A8 8 0 0 1 8 0ZM1.5 8a6.5 6.5 0 1 0 13 0 6.5 6.5 0 0 0-13 0Zm7-3.25v2.992l2.028.812a.75.75 0 0 1-.557 1.392l-2.5-1A.751.751 0 0 1 7 8.25v-3.5a.75.75 0 0 1 1.5 0Z"></path>
</svg>
23 minutes </div>
</div>
</div>
<ul class="list-style-none d-flex flex-items-center">
<li>
<span class="text-semibold">
Share: </span>
</li>
<li class="ml-3">
<a href="https://x.com/share?text=Debugging%20network%20stalls%20on%20Kubernetes&#038;url=https%3A%2F%2Fgithub.blog%2Fengineering%2Finfrastructure%2Fdebugging-network-stalls-on-kubernetes%2F" target="_blank" rel="noopener noreferrer" class="d-flex flex-justify-center flex-items-center border circle px-2 py-2" aria-label="Share on X">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1200 1227" width="10" height="10">
<path d="M714.163 519.284L1160.89 0H1055.03L667.137 450.887L357.328 0H0L468.492 681.821L0 1226.37H105.866L515.491 750.218L842.672 1226.37H1200L714.137 519.284H714.163ZM569.165 687.828L521.697 619.934L144.011 79.6944H306.615L611.412 515.685L658.88 583.579L1055.08 1150.3H892.476L569.165 687.854V687.828Z" fill="currentColor" />
</svg>
</a>
</li>
<li class="ml-3">
<a href="https://www.facebook.com/sharer/sharer.php?t=Debugging%20network%20stalls%20on%20Kubernetes&#038;u=https%3A%2F%2Fgithub.blog%2Fengineering%2Finfrastructure%2Fdebugging-network-stalls-on-kubernetes%2F" target="_blank" rel="noopener noreferrer" class="d-flex flex-justify-center flex-items-center border circle px-2 py-2" aria-label="Share on Facebook">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 15.3 15.4" width="10" height="10">
<path d="M14.5 0H.8a.88.88 0 0 0-.8.9v13.6a.88.88 0 0 0 .8.9h7.3v-6h-2V7.1h2V5.4a2.87 2.87 0 0 1 2.5-3.1h.5a10.87 10.87 0 0 1 1.8.1v2.1h-1.3c-1 0-1.1.5-1.1 1.1v1.5h2.3l-.3 2.3h-2v5.9h3.9a.88.88 0 0 0 .9-.8V.8a.86.86 0 0 0-.8-.8z" fill="currentColor" />
</svg>
</a>
</li>
<li class="ml-3">
<a href="https://www.linkedin.com/shareArticle?title=Debugging%20network%20stalls%20on%20Kubernetes&#038;url=https%3A%2F%2Fgithub.blog%2Fengineering%2Finfrastructure%2Fdebugging-network-stalls-on-kubernetes%2F" target="_blank" rel="noopener noreferrer" class="d-flex flex-justify-center flex-items-center border circle px-2 py-2" aria-label="Share on LinkedIn">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 19 18" width="10" height="10">
<path d="M3.94 2A2 2 0 1 1 2 0a2 2 0 0 1 1.94 2zM4 5.48H0V18h4zm6.32 0H6.34V18h3.94v-6.57c0-3.66 4.77-4 4.77 0V18H19v-7.93c0-6.17-7.06-5.94-8.72-2.91z" fill="currentColor" />
</svg>
</a>
</li>
</ul>
</div>
<div class="indigo-separator mt-4"></div>
</div>
</div>
</section>
<div class="container-xl mx-auto p-responsive-blog mt-4 mt-md-7 mb-7 mb-md-9">
<div class="d-flex flex-wrap flex-justify-center gutter-spacious">
<section class="col-12 col-md-8 col-lg-7 post__content is-layout-constrained post-51321 post type-post status-publish format-standard has-post-thumbnail hentry category-engineering category-infrastructure">
<p>We’ve talked about <a href="https://github.blog/2017-08-16-kubernetes-at-github/">Kubernetes</a> before, and over the last couple of years it’s become the standard deployment pattern at GitHub. We now run a large portion of both internal and public-facing services on Kubernetes. As our Kubernetes clusters have grown, and our targets on the latency of our services have become more stringent, we began to notice that certain services running on Kubernetes in our environment were experiencing sporadic latency that couldn’t be attributed to the performance characteristics of the application itself.</p>
<p>Essentially, applications running on our Kubernetes clusters would observe seemingly random latency of up to and over 100ms on connections, which would cause downstream timeouts or retries. Services were expected to be able to respond to requests in well under 100ms, which wasn’t feasible when the connection itself was taking so long. Separately, we also observed very fast MySQL queries, which we expected to take a matter of milliseconds and that MySQL observed taking only milliseconds, were being observed taking 100ms or more from the perspective of the querying application.</p>
<p>The problem was initially narrowed down to communications that involved a Kubernetes node, even if the other side of a connection was outside Kubernetes. The most simple reproduction we had was a <a href="https://github.com/tsenart/vegeta">Vegeta</a> benchmark that could be run from any internal host, targeting a Kubernetes service running on a node port, and would observe the sporadically high latency. In this post, we’ll walk through how we tracked down the underlying issue.</p>
<h2 id="removing-complexity-to-find-the-path-at-fault"><a class="heading-link" href="#removing-complexity-to-find-the-path-at-fault">Removing complexity to find the path at fault<span class="heading-hash pl-2 text-italic text-bold" aria-hidden="true"></span></a></h2>
<p>Using an example reproduction, we wanted to narrow down the problem and remove layers of complexity. Initially, there were too many moving parts in the flow between Vegeta and pods running on Kubernetes to determine if this was a deeper network problem, so we needed to rule some out.</p>
<p><img data-recalc-dims="1" decoding="async" loading="lazy" class="alignnone" src="https://i0.wp.com/user-images.githubusercontent.com/349190/66730905-2ede2800-eea0-11e9-9091-5e68bb23467d.png?resize=816%2C210&#038;ssl=1" alt="vegeta to kube pod via NodePort" width="816" height="210" /></p>
<p>The client, Vegeta, creates a TCP connection to any kube-node in the cluster. Kubernetes runs in our data centers as an <a href="https://en.wikipedia.org/wiki/Overlay_network">overlay network</a> (a network that runs on top of our existing datacenter network) that uses <a href="https://en.wikipedia.org/wiki/IP_in_IP">IPIP</a> (which encapsulates the overlay network’s IP packet inside the datacenter’s IP packet). When a connection is made to that first kube-node, it performs stateful <a href="https://en.wikipedia.org/wiki/Network_address_translation">Network Address Translation</a> (NAT) to convert the kube-node’s IP and port to an IP and port on the overlay network (specifically, of the pod running the application). On return, it undoes each of these steps. This is a complex system with a lot of state, and a lot of moving parts that are constantly updating and changing as services deploy and move around.</p>
<p>As part of running a <code>tcpdump</code> on the original Vegeta benchmark, we observed the latency during a TCP handshake (between SYN and SYN-ACK). To simplify some of the complexity of HTTP and Vegeta, we can use <code>hping3</code> to just “ping” with a SYN packet and see if we observe the latency in the response packet—then throw away the connection. We can filter it to only include packets over 100ms and get a simpler reproduction case than a full Layer 7 Vegeta benchmark or attack against the service. The following “pings” a kube-node using TCP SYN/SYN-ACK on the “node port” for the service (30927) with an interval of 10ms, filtered for slow responses:</p>
<pre><code class="shell">theojulienne@shell ~ $ sudo hping3 172.16.47.27 -S -p 30927 -i u10000 | egrep --line-buffered 'rtt=[0-9]{3}\.'
len=46 ip=172.16.47.27 ttl=59 DF id=0 sport=30927 flags=SA seq=1485 win=29200 rtt=127.1 ms
len=46 ip=172.16.47.27 ttl=59 DF id=0 sport=30927 flags=SA seq=1486 win=29200 rtt=117.0 ms
len=46 ip=172.16.47.27 ttl=59 DF id=0 sport=30927 flags=SA seq=1487 win=29200 rtt=106.2 ms
len=46 ip=172.16.47.27 ttl=59 DF id=0 sport=30927 flags=SA seq=1488 win=29200 rtt=104.1 ms
len=46 ip=172.16.47.27 ttl=59 DF id=0 sport=30927 flags=SA seq=5024 win=29200 rtt=109.2 ms
len=46 ip=172.16.47.27 ttl=59 DF id=0 sport=30927 flags=SA seq=5231 win=29200 rtt=109.2 ms</code></pre>
<p>Our first new observation from the sequence numbers and timings is that this isn’t a one-off, but is often grouped, like a backlog that eventually gets processed.</p>
<p>Next up, we want to narrow down which component(s) were potentially at fault. Is it the kube-proxy iptables NAT rules that are hundreds of rules long? Is it the IPIP tunnel and something on the network handling them poorly? One way to validate this is to test each step of the system. What happens if we remove the NAT and firewall logic and only use the IPIP part:</p>
<p><img data-recalc-dims="1" decoding="async" loading="lazy" class="alignnone" src="https://i0.wp.com/user-images.githubusercontent.com/349190/66730912-37cef980-eea0-11e9-8048-984d6c5966a3.png?resize=570%2C210&#038;ssl=1" alt="hping 3 on the IPIP tunnel alone" width="570" height="210" /></p>
<p>Linux thankfully lets you just talk directly to an overlay IP when you’re on a machine that’s part of the same network, so that’s pretty easy to do:</p>
<pre><code class="shell">theojulienne@kube-node-client ~ $ sudo hping3 10.125.20.64 -S -i u10000 | egrep --line-buffered 'rtt=[0-9]{3}\.'
len=40 ip=10.125.20.64 ttl=64 DF id=0 sport=0 flags=RA seq=7346 win=0 rtt=127.3 ms
len=40 ip=10.125.20.64 ttl=64 DF id=0 sport=0 flags=RA seq=7347 win=0 rtt=117.3 ms
len=40 ip=10.125.20.64 ttl=64 DF id=0 sport=0 flags=RA seq=7348 win=0 rtt=107.2 ms</code></pre>
<p>Based on our results, the problem still remains! That rules out iptables and NAT. Is it TCP that’s the problem? Let’s see what happens when we perform a normal ICMP ping:</p>
<pre><code class="shell">theojulienne@kube-node-client ~ $ sudo hping3 10.125.20.64 --icmp -i u10000 | egrep --line-buffered 'rtt=[0-9]{3}\.'
len=28 ip=10.125.20.64 ttl=64 id=42594 icmp_seq=104 rtt=110.0 ms
len=28 ip=10.125.20.64 ttl=64 id=49448 icmp_seq=4022 rtt=141.3 ms
len=28 ip=10.125.20.64 ttl=64 id=49449 icmp_seq=4023 rtt=131.3 ms
len=28 ip=10.125.20.64 ttl=64 id=49450 icmp_seq=4024 rtt=121.2 ms
len=28 ip=10.125.20.64 ttl=64 id=49451 icmp_seq=4025 rtt=111.2 ms
len=28 ip=10.125.20.64 ttl=64 id=49452 icmp_seq=4026 rtt=101.1 ms
len=28 ip=10.125.20.64 ttl=64 id=50023 icmp_seq=4343 rtt=126.8 ms
len=28 ip=10.125.20.64 ttl=64 id=50024 icmp_seq=4344 rtt=116.8 ms
len=28 ip=10.125.20.64 ttl=64 id=50025 icmp_seq=4345 rtt=106.8 ms
len=28 ip=10.125.20.64 ttl=64 id=59727 icmp_seq=9836 rtt=106.1 ms</code></pre>
<p>Our results show that the problem still exists. Is it the IPIP tunnel that’s causing the problem? Let’s simplify things further:</p>
<p><img data-recalc-dims="1" decoding="async" loading="lazy" class="alignnone" src="https://i0.wp.com/user-images.githubusercontent.com/349190/66730916-3c93ad80-eea0-11e9-810f-0f32da529da0.png?resize=570%2C210&#038;ssl=1" alt="hping 3 directly between hosts" width="570" height="210" /></p>
<p>Is it possible that it’s every packet between these two hosts?</p>
<pre><code class="shell">theojulienne@kube-node-client ~ $ sudo hping3 172.16.47.27 --icmp -i u10000 | egrep --line-buffered 'rtt=[0-9]{3}\.'
len=46 ip=172.16.47.27 ttl=61 id=41127 icmp_seq=12564 rtt=140.9 ms
len=46 ip=172.16.47.27 ttl=61 id=41128 icmp_seq=12565 rtt=130.9 ms
len=46 ip=172.16.47.27 ttl=61 id=41129 icmp_seq=12566 rtt=120.8 ms
len=46 ip=172.16.47.27 ttl=61 id=41130 icmp_seq=12567 rtt=110.8 ms
len=46 ip=172.16.47.27 ttl=61 id=41131 icmp_seq=12568 rtt=100.7 ms
len=46 ip=172.16.47.27 ttl=61 id=9062 icmp_seq=31443 rtt=134.2 ms
len=46 ip=172.16.47.27 ttl=61 id=9063 icmp_seq=31444 rtt=124.2 ms
len=46 ip=172.16.47.27 ttl=61 id=9064 icmp_seq=31445 rtt=114.2 ms
len=46 ip=172.16.47.27 ttl=61 id=9065 icmp_seq=31446 rtt=104.2 ms</code></pre>
<p>Behind the complexity, it’s as simple as two kube-node hosts sending any packet, even ICMP pings, to each other. They’ll still see the latency, if the target host is a “bad” one (some are worse than others).</p>
<p>Now there’s one last thing to question: we clearly don’t observe this everywhere, so why is it just on kube-node servers? And does it occur when the kube-node is the sender or the receiver? Luckily, this is also pretty easy to narrow down by using a host outside Kubernetes as a sender, but with the same “known bad” target host (from a staff shell host to the same kube-node). We can observe this is still an issue in that direction:</p>
<pre><code class="shell">theojulienne@shell ~ $ sudo hping3 172.16.47.27 -p 9876 -S -i u10000 | egrep --line-buffered 'rtt=[0-9]{3}\.'
len=46 ip=172.16.47.27 ttl=61 DF id=0 sport=9876 flags=RA seq=312 win=0 rtt=108.5 ms
len=46 ip=172.16.47.27 ttl=61 DF id=0 sport=9876 flags=RA seq=5903 win=0 rtt=119.4 ms
len=46 ip=172.16.47.27 ttl=61 DF id=0 sport=9876 flags=RA seq=6227 win=0 rtt=139.9 ms
len=46 ip=172.16.47.27 ttl=61 DF id=0 sport=9876 flags=RA seq=7929 win=0 rtt=131.2 ms</code></pre>
<p>And then perform the same from the previous source kube-node to a staff shell host (which rules out the source host, since a ping has both an RX and TX component):</p>
<pre><code class="shell">theojulienne@kube-node-client ~ $ sudo hping3 172.16.33.44 -p 9876 -S -i u10000 | egrep --line-buffered 'rtt=[0-9]{3}\.'
^C
--- 172.16.33.44 hping statistic ---
22352 packets transmitted, 22350 packets received, 1% packet loss
round-trip min/avg/max = 0.2/7.6/1010.6 ms</code></pre>
<p>Looking into the packet captures of the latency we observed, we get some more information. Specifically, that the “sender” host (bottom) observes this timeout while the “receiver” host (top) does not—see the Delta column (in seconds):</p>
<p><img data-recalc-dims="1" decoding="async" loading="lazy" class="alignnone" src="https://i0.wp.com/user-images.githubusercontent.com/349190/66730925-44ebe880-eea0-11e9-9efd-a533d2531de4.png?resize=3714%2C800&#038;ssl=1" alt="packet captures of the latency" width="3714" height="800" /></p>
<p>Additionally, by looking at the difference between the ordering of the packets (based on the sequence numbers) on the receiver side of the TCP and ICMP results above, we can observe that ICMP packets always arrive in the same sequence they were sent, but with uneven timing, while TCP packets are sometimes interleaved, but a subset of them stall. Notably, we observe that if you count the ports of the SYN packets, the ports are not in order on the receiver side, while they’re in order on the sender side.</p>
<p>There is a subtle difference between how modern server <a href="https://en.wikipedia.org/wiki/Network_interface_controller">NICs</a>—like we have in our data centers—handle packets containing TCP vs ICMP. When a packet arrives, the NIC hashes the packet “per connection” and tries to divvy up the connections across receive queues, each (approximately) delegated to a given CPU core. For TCP, this hash includes both source and destination IP and port. In other words, each connection is hashed (potentially) differently. For ICMP, just the IP source and destination are hashed, since there are no ports.</p>
<p>Another new observation is that we can tell that ICMP observes stalls on all communications between the two hosts during this period from the sequence numbers in ICMP vs TCP, while TCP does not. This tells us that the RX queue hashing is likely in play, almost certainly indicating the stall is in processing RX packets, not in sending responses.</p>
<p>This rules out kube-node transmits, so we now know that it’s a stall in processing packets, and that it’s on the receive side on some kube-node servers.</p>
<h2 id="deep-dive-into-linux-kernel-packet-processing"><a class="heading-link" href="#deep-dive-into-linux-kernel-packet-processing">Deep dive into Linux kernel packet processing<span class="heading-hash pl-2 text-italic text-bold" aria-hidden="true"></span></a></h2>
<p>To understand why the problem could be on the receiving side on some kube-node servers, let’s take a look at how the Linux kernel processes packets.</p>
<p>Going back to the simplest traditional implementation, the network card receives a packet and sends an <a href="https://en.wikipedia.org/wiki/Interrupt">interrupt</a> to the Linux kernel stating that there’s a packet that should be handled. The kernel stops other work, switches context to the interrupt handler, processes the packet, then switches back to what it was doing.</p>
<p><img data-recalc-dims="1" decoding="async" loading="lazy" class="alignnone" src="https://i0.wp.com/user-images.githubusercontent.com/349190/66730927-4ae1c980-eea0-11e9-897c-75cd39dbe286.png?resize=743%2C272&#038;ssl=1" alt="traditional interrupt-driven approach" width="743" height="272" /></p>
<p>This context switching is slow, which may have been fine on a 10Mbit NIC in the 90s, but on modern servers where the NIC is 10G and at maximal line rate can bring in around 15 million packets per second, on a smaller server with eight cores that could mean the kernel is interrupted millions of times per second per core.</p>
<p>Instead of constantly handling interrupts, many years ago Linux added <a href="https://en.wikipedia.org/wiki/New_API">NAPI</a>, the networking API that modern drivers use for improved performance at high packet rates. At low rates, the kernel still accepts interrupts from the NIC in the method we mentioned. Once enough packets arrive and cross a threshold, it disables interrupts and instead begins polling the NIC and pulling off packets in batches. This processing is done in a “softirq”, or <a href="https://www.kernel.org/doc/htmldocs/kernel-hacking/basics-softirqs.html">software interrupt context</a>. This happens at the end of syscalls and hardware interrupts, which are times that the kernel (as opposed to userspace) is already running.</p>
<p><img data-recalc-dims="1" decoding="async" loading="lazy" class="alignnone" src="https://i0.wp.com/user-images.githubusercontent.com/349190/66730932-4f0de700-eea0-11e9-927f-393690107ace.png?resize=740%2C293&#038;ssl=1" alt="NAPI polling in softirq at end of syscalls" width="740" height="293" /></p>
<p>This is much faster, but brings up another problem. What happens if we have so many packets to process that we spend all our time processing packets from the NIC, but we never have time to let the userspace processes actually drain those queues (read from TCP connections, etc.)? Eventually the queues would fill up, and we’d start dropping packets. To try and make this fair, the kernel limits the amount of packets processed in a given softirq context to a certain budget. Once this budget is exceeded, it wakes up a separate thread called <code>ksoftirqd</code> (you’ll see one of these in <code>ps</code> for each core) which processes these softirqs outside of the normal syscall/interrupt path. This thread is scheduled using the standard process scheduler, which already tries to be fair.</p>
<p><img data-recalc-dims="1" decoding="async" loading="lazy" class="alignnone" src="https://i0.wp.com/user-images.githubusercontent.com/349190/66730935-533a0480-eea0-11e9-8697-7fc52d2ea1fa.png?resize=841%2C358&#038;ssl=1" alt="NAPI polling crossing threshold in schedule ksoftirqd" width="841" height="358" /></p>
<p>With an overview of the way the kernel is processing packets, we can see there is definitely opportunity for this processing to become stalled. If the time between softirq processing calls grows, packets could sit in the NIC RX queue for a while before being processed. This could be something deadlocking the CPU core, or it could be something slow preventing the kernel from running softirqs.</p>
<h2 id="narrow-down-processing-to-a-core-or-method"><a class="heading-link" href="#narrow-down-processing-to-a-core-or-method">Narrow down processing to a core or method<span class="heading-hash pl-2 text-italic text-bold" aria-hidden="true"></span></a></h2>
<p>At this point, it makes sense that this could happen, and we know we’re observing something that looks a lot like it. The next step is to confirm this theory, and if we do, understand what’s causing it.</p>
<p>Let’s revisit the slow round trip packets we saw before:</p>
<pre><code class="shell">len=46 ip=172.16.53.32 ttl=61 id=29573 icmp_seq=1953 rtt=99.3 ms
len=46 ip=172.16.53.32 ttl=61 id=29574 icmp_seq=1954 rtt=89.3 ms
len=46 ip=172.16.53.32 ttl=61 id=29575 icmp_seq=1955 rtt=79.2 ms
len=46 ip=172.16.53.32 ttl=61 id=29576 icmp_seq=1956 rtt=69.1 ms
len=46 ip=172.16.53.32 ttl=61 id=29577 icmp_seq=1957 rtt=59.1 ms
len=46 ip=172.16.53.32 ttl=61 id=29790 icmp_seq=2070 rtt=75.7 ms
len=46 ip=172.16.53.32 ttl=61 id=29791 icmp_seq=2071 rtt=65.6 ms
len=46 ip=172.16.53.32 ttl=61 id=29792 icmp_seq=2072 rtt=55.5 ms</code></pre>
<p>As discussed previously, these ICMP packets are hashed to a single NIC RX queue and processed by a single CPU core. If we want to understand what the kernel is doing, it’s helpful to know where (cpu core) and how (softirq, ksoftirqd) it’s processing these packets so we can catch them in action.</p>
<p>Now it’s time to use the tools that allow live tracing of a running Linux kernel—<a href="https://github.com/iovisor/bcc">bcc</a> is what was used here. This allows you to write small C programs that hook arbitrary functions in the kernel, and buffer events back to a userspace Python program which can summarize and return them to you. The “hook arbitrary functions in the kernel” is the difficult part, but it actually goes out of its way to be as safe as possible to use, because it’s designed for tracing exactly this type of production issue that you can’t simply reproduce in a testing or dev environment.</p>
<p>The plan here is simple: we know the kernel is processing those ICMP ping packets, so let’s hook the kernel function <a href="https://github.com/torvalds/linux/blob/v4.19/net/ipv4/icmp.c#L925">icmp_echo</a> which takes an incoming ICMP “echo request” packet and initiates sending the ICMP “echo response” reply. We can identify the packet using the incrementing icmp_seq shown by <code>hping3</code> above.</p>
<p>The code for this <a href="https://gist.github.com/theojulienne/9d78a0cb68dbe56f19a2ae6316bc6846">bcc script</a> looks complex, but breaking it down it’s not as scary as it sounds. The <code>icmp_echo</code> function is passed a <code>struct sk_buff *skb</code>, which is the packet containing the ICMP echo request. We can delve into this live and pull out the <code>echo.sequence</code> (which maps to the <code>icmp_seq</code> shown by <code>hping3</code> above), and send that back to userspace. Conveniently, we can also grab the current process name/id as well. This gives us results like the following, live as the kernel processes these packets:</p>
<pre><code class="shell">TGID    PID     PROCESS NAME    ICMP_SEQ
0       0       swapper/11      770
0       0       swapper/11      771
0       0       swapper/11      772
0       0       swapper/11      773
0       0       swapper/11      774
20041   20086   prometheus      775
0       0       swapper/11      776
0       0       swapper/11      777
0       0       swapper/11      778
4512    4542   spokes-report-s  779</code></pre>
<p>One thing to note about this process name is that in a post-syscall <code>softirq</code> context, you see the process that made the syscall show as the “process”, even though really it’s the kernel processing it safely within the kernel context.</p>
<p>With that running, we can now correlate back from the stalled packets observed with <code>hping3</code> to the process that’s handling it. A simple <code>grep</code> on that capture for the <code>icmp_seq</code> values with some context shows what happened before these packets were processed. The packets that line up with the above <code>hping3</code> icmp_seq values have been marked along with the rtt’s we observed above (and what we’d have expected if &lt;50ms rtt’s weren’t filtered out):</p>
<pre><code class="shell">TGID    PID     PROCESS NAME    ICMP_SEQ ** RTT
--
10137   10436   cadvisor        1951
10137   10436   cadvisor        1952
76      76      ksoftirqd/11    1953 ** 99ms
76      76      ksoftirqd/11    1954 ** 89ms
76      76      ksoftirqd/11    1955 ** 79ms
76      76      ksoftirqd/11    1956 ** 69ms
76      76      ksoftirqd/11    1957 ** 59ms
76      76      ksoftirqd/11    1958 ** (49ms)
76      76      ksoftirqd/11    1959 ** (39ms)
76      76      ksoftirqd/11    1960 ** (29ms)
76      76      ksoftirqd/11    1961 ** (19ms)
76      76      ksoftirqd/11    1962 ** (9ms)
--
10137   10436   cadvisor        2068
10137   10436   cadvisor        2069
76      76      ksoftirqd/11    2070 ** 75ms
76      76      ksoftirqd/11    2071 ** 65ms
76      76      ksoftirqd/11    2072 ** 55ms
76      76      ksoftirqd/11    2073 ** (45ms)
76      76      ksoftirqd/11    2074 ** (35ms)
76      76      ksoftirqd/11    2075 ** (25ms)
76      76      ksoftirqd/11    2076 ** (15ms)
76      76      ksoftirqd/11    2077 ** (5ms)</code></pre>
<p>The results tells us a few things. First, these packets are being processed by <code>ksoftirqd/11</code> which conveniently tells us this particular pair of machines have their ICMP packets hashed to core 11 on the receiving side. We can also see that every time we see a stall, we always see some packets processed in <code>cadvisor</code>’s syscall softirq context, followed by <code>ksoftirqd</code> taking over and processing the backlog, exactly the number we’d expect to work through the backlog.</p>
<p>The fact that <code>cadvisor</code> is always running just prior to this immediately also implicates it in the problem. Ironically, <a href="https://github.com/google/cadvisor">cadvisor</a> “analyzes resource usage and performance characteristics of running containers”, yet it’s triggering this performance problem. As with many things related to containers, it’s all relatively bleeding-edge tooling which can result in some somewhat expected corner cases of bad performance.</p>
<h2 id="what-is-cadvisor-doing-to-stall-things"><a class="heading-link" href="#what-is-cadvisor-doing-to-stall-things">What is cadvisor doing to stall things?<span class="heading-hash pl-2 text-italic text-bold" aria-hidden="true"></span></a></h2>
<p>With the understanding of how the stall can happen, the process causing it, and the CPU core it’s happening on, we now have a pretty good idea of what this looks like. For the kernel to hard block and not schedule <code>ksoftirqd</code> earlier, and given we see packets processed under <code>cadvisor</code>’s softirq context, it’s likely that <code>cadvisor</code> is running a slow syscall which ends with the rest of the packets being processed:</p>
<p><img data-recalc-dims="1" decoding="async" loading="lazy" class="alignnone" src="https://i0.wp.com/user-images.githubusercontent.com/349190/66730943-5a611280-eea0-11e9-902a-454f482d4223.png?resize=841%2C345&#038;ssl=1" alt="slow syscall causing stalled packet processing on NIC RX queue" width="841" height="345" /></p>
<p>That’s a theory but how do we validate this is actually happening? One thing we can do is trace what’s running on the CPU core throughout this process, catch the point where the packets are overflowing budget and processed by ksoftirqd, then look back a bit to see what was running on the CPU core. Think of it like taking an x-ray of the CPU every few milliseconds. It would look something like this:</p>
<p><img data-recalc-dims="1" decoding="async" loading="lazy" class="alignnone" src="https://i0.wp.com/user-images.githubusercontent.com/349190/66730945-5e8d3000-eea0-11e9-89af-daea3a7bd3f6.png?resize=748%2C224&#038;ssl=1" alt="tracing of cpu to catch bad syscall and preceding work" width="748" height="224" /></p>
<p>Conveniently, this is something that’s already mostly supported. The <a href="https://perf.wiki.kernel.org/index.php/Tutorial#Sampling_with_perf_record"><code>perf record</code></a> tool samples a given CPU core at a certain frequency and can generate a call graph of the live system, including both userspace and the kernel. Taking that recording and manipulating it using a quick fork of a tool from <a href="https://github.com/brendangregg/FlameGraph">Brendan Gregg’s FlameGraph</a> that retained stack trace ordering, we can get a one-line stack trace for each 1ms sample, then get a sample of the 100ms before <code>ksoftirqd</code> is in the trace:</p>
<pre><code class="shell"># record 999 times a second, or every 1ms with some offset so not to align exactly with timers
sudo perf record -C 11 -g -F 999
# take that recording and make a simpler stack trace.
sudo perf script 2&gt;/dev/null | ./FlameGraph/stackcollapse-perf-ordered.pl | grep ksoftir -B 100</code></pre>
<p>This results in the following:</p>
<pre><code class="shell">(hundreds of traces that look similar)
cadvisor;[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];entry_SYSCALL_64_after_swapgs;do_syscall_64;sys_read;vfs_read;seq_read;memcg_stat_show;mem_cgroup_nr_lru_pages;mem_cgroup_node_nr_lru_pages
cadvisor;[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];entry_SYSCALL_64_after_swapgs;do_syscall_64;sys_read;vfs_read;seq_read;memcg_stat_show;mem_cgroup_nr_lru_pages;mem_cgroup_node_nr_lru_pages
cadvisor;[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];entry_SYSCALL_64_after_swapgs;do_syscall_64;sys_read;vfs_read;seq_read;memcg_stat_show;mem_cgroup_iter
cadvisor;[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];entry_SYSCALL_64_after_swapgs;do_syscall_64;sys_read;vfs_read;seq_read;memcg_stat_show;mem_cgroup_nr_lru_pages;mem_cgroup_node_nr_lru_pages
cadvisor;[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];[cadvisor];entry_SYSCALL_64_after_swapgs;do_syscall_64;sys_read;vfs_read;seq_read;memcg_stat_show;mem_cgroup_nr_lru_pages;mem_cgroup_node_nr_lru_pages
ksoftirqd/11;ret_from_fork;kthread;kthread;smpboot_thread_fn;smpboot_thread_fn;run_ksoftirqd;__do_softirq;net_rx_action;ixgbe_poll;ixgbe_clean_rx_irq;napi_gro_receive;netif_receive_skb_internal;inet_gro_receive;bond_handle_frame;__netif_receive_skb_core;ip_rcv_finish;ip_rcv;ip_forward_finish;ip_forward;ip_finish_output;nf_iterate;ip_output;ip_finish_output2;__dev_queue_xmit;dev_hard_start_xmit;ipip_tunnel_xmit;ip_tunnel_xmit;iptunnel_xmit;ip_local_out;dst_output;__ip_local_out;nf_hook_slow;nf_iterate;nf_conntrack_in;generic_packet;ipt_do_table;set_match_v4;ip_set_test;hash_net4_kadt;ixgbe_xmit_frame_ring;swiotlb_dma_mapping_error;hash_net4_test
ksoftirqd/11;ret_from_fork;kthread;kthread;smpboot_thread_fn;smpboot_thread_fn;run_ksoftirqd;__do_softirq;net_rx_action;gro_cell_poll;napi_gro_receive;netif_receive_skb_internal;inet_gro_receive;__netif_receive_skb_core;ip_rcv_finish;ip_rcv;ip_forward_finish;ip_forward;ip_finish_output;nf_iterate;ip_output;ip_finish_output2;__dev_queue_xmit;dev_hard_start_xmit;dev_queue_xmit_nit;packet_rcv;tpacket_rcv;sch_direct_xmit;validate_xmit_skb_list;validate_xmit_skb;netif_skb_features;ixgbe_xmit_frame_ring;swiotlb_dma_mapping_error;__dev_queue_xmit;dev_hard_start_xmit;__bpf_prog_run;__bpf_prog_run</code></pre>
<p>There’s a lot there, but looking through it you can see it’s the cadvisor-then-ksoftirqd pattern we saw from the ICMP tracer above. What does it mean?</p>
<p>Each line is a trace of the CPU at a point in time. Each call down the stack is separated by <code>;</code> on that line. Looking at the middle of the lines we can see the syscall being called is <code>read(): .... ;do_syscall_64;sys_read; ...</code> So cadvisor is spending a lot of time in a <code>read()</code> syscall relating to <code>mem_cgroup_*</code> functions (the top of the call stack / end of line).</p>
<p>The call stack trace isn’t convenient to see what’s being read, so let’s use <code>strace</code> to see what cadvisor is doing and find 100ms-or-slower syscalls:</p>
<pre><code class="shell">theojulienne@kube-node-bad ~ $ sudo strace -p 10137 -T -ff 2&gt;&amp;1 | egrep '&lt;0\.[1-9]'
[pid 10436] &lt;... futex resumed&gt; )       = 0 &lt;0.156784&gt;
[pid 10432] &lt;... futex resumed&gt; )       = 0 &lt;0.258285&gt;
[pid 10137] &lt;... futex resumed&gt; )       = 0 &lt;0.678382&gt;
[pid 10384] &lt;... futex resumed&gt; )       = 0 &lt;0.762328&gt;
[pid 10436] &lt;... read resumed&gt; "cache 154234880\nrss 507904\nrss_h"..., 4096) = 658 &lt;0.179438&gt;
[pid 10384] &lt;... futex resumed&gt; )       = 0 &lt;0.104614&gt;
[pid 10436] &lt;... futex resumed&gt; )       = 0 &lt;0.175936&gt;
[pid 10436] &lt;... read resumed&gt; "cache 0\nrss 0\nrss_huge 0\nmapped_"..., 4096) = 577 &lt;0.228091&gt;
[pid 10427] &lt;... read resumed&gt; "cache 0\nrss 0\nrss_huge 0\nmapped_"..., 4096) = 577 &lt;0.207334&gt;
[pid 10411] &lt;... epoll_ctl resumed&gt; )   = 0 &lt;0.118113&gt;
[pid 10382] &lt;... pselect6 resumed&gt; )    = 0 (Timeout) &lt;0.117717&gt;
[pid 10436] &lt;... read resumed&gt; "cache 154234880\nrss 507904\nrss_h"..., 4096) = 660 &lt;0.159891&gt;
[pid 10417] &lt;... futex resumed&gt; )       = 0 &lt;0.917495&gt;
[pid 10436] &lt;... futex resumed&gt; )       = 0 &lt;0.208172&gt;
[pid 10417] &lt;... futex resumed&gt; )       = 0 &lt;0.190763&gt;
[pid 10417] &lt;... read resumed&gt; "cache 0\nrss 0\nrss_huge 0\nmapped_"..., 4096) = 576 &lt;0.154442&gt;</code></pre>
<p>Sure enough, we see the slow <code>read()</code> calls. From the content being read and <code>mem_cgroup</code> context above, these <code>read()</code> calls are to a <code>memory.stat</code> file which shows the memory usage and limits of a cgroup (the resource isolation technology used by Docker). cadvisor is polling this file to get resource utilization details for the containers. Let’s see if it’s the kernel or cadvisor that’s doing something unexpected by attempting the read ourselves:</p>
<pre><code class="shell">theojulienne@kube-node-bad ~ $ time cat /sys/fs/cgroup/memory/memory.stat &gt;/dev/null
real    0m0.153s
user    0m0.000s
sys    0m0.152s
theojulienne@kube-node-bad ~ $</code></pre>
<p>Since we can reproduce it, this indicates that it’s the kernel hitting a pathologically bad case.</p>
<h2 id="what-causes-this-read-to-be-so-slow"><a class="heading-link" href="#what-causes-this-read-to-be-so-slow">What causes this read to be so slow<span class="heading-hash pl-2 text-italic text-bold" aria-hidden="true"></span></a></h2>
<p>At this point it’s much more simple to find similar issues reported by others. As it turns out, this has been reported to cadvisor as an <a href="https://github.com/google/cadvisor/issues/1774">excessive CPU usage problem</a>, it just hadn’t been observed that latency was also being introduced to the network stack randomly as well. In fact, some folks internally had noticed cadvisor was consuming more CPU than expected, but it didn’t seem to be causing an issue since our servers had plenty of CPU capacity, and so the CPU usage hadn’t yet been investigated.</p>
<p>The overview of the issue is that the memory cgroup is accounting for memory usage inside a namespace (container). When all processes in that cgroup exit, the memory cgroup is released by Docker. However, “memory” isn’t just process memory, and although processes memory usage itself is gone, it turns out the kernel also assigns cached content like dentries and inodes (directory and file metadata) that are cached to the memory cgroup. From that issue:</p>
<p style="padding-left: 40px;">“zombie” cgroups: cgroups that have no processes and have been deleted but still have memory charged to them (in my case, from the dentry cache, but it could also be from page cache or tmpfs).</p>
<p>Rather than the kernel iterating over every page in the cache at cgroup release time, which could be very slow, they choose to wait for those pages to be reclaimed and then finally clean up the cgroup once all are reclaimed when memory is needed, lazily. In the meantime, the cgroup still needs to be counted during stats collection.</p>
<p>From a performance perspective, they are trading off time on a slow process by amortizing it over the reclamation of each page, opting to make the initial cleanup fast in return for leaving some cached memory around. That’s fine, when the kernel reclaims the last of the cached memory, the cgroup eventually gets cleaned up, so it’s not really a “leak”. Unfortunately the search that <code>memory.stat</code> performs, the way it’s implemented on the kernel version (4.9) we’re running on some servers, combined with the huge amount of memory on our servers, means it can take a significantly long time for the last of the cached data to be reclaimed and for the zombie cgroup to be cleaned up.</p>
<p>It turns out we had nodes that had such a large number of zombie cgroups that some had reads/stalls of over a second.</p>
<p>The workaround on that cadvisor issue, to immediately free the dentries/inodes cache systemwide, immediately stopped the read latency, and also the network latency stalls on the host, since the dropping of the cache included the cached pages in the “zombie” cgroups and so they were also freed. This isn’t a solution, but it does validate the cause of the issue.</p>
<p>As it turns out newer kernel releases (4.19+) have improved the performance of the <code>memory.stat</code> call and so this is no longer a problem after moving to that kernel. In the interim, we had existing tooling that was able to detect problems with nodes in our Kubernetes clusters and gracefully drain and reboot them, which we used to detect the cases of high enough latency that would cause issues, and treat them with a graceful reboot. This gave us breathing room while OS and kernel upgrades were rolled out to the remainder of the fleet.</p>
<h2 id="wrapping-up"><a class="heading-link" href="#wrapping-up">Wrapping up<span class="heading-hash pl-2 text-italic text-bold" aria-hidden="true"></span></a></h2>
<p>Since this problem manifested as NIC RX queues not being processed for hundreds of milliseconds, it was responsible for both high latency on short connections and latency observed mid-connection such as between MySQL query and response packets. Understanding and maintaining performance of our most foundational systems like Kubernetes is critical to the reliability and speed of all services that build on top of them. As we invest in and improve on this performance, every system we run benefits from those improvements.</p>
<div class="mt-8 mb-8 mb-md-0">
<h2 class="h5-mktg">
Written by </h2>
<div class="gradient-separator mt-3 mb-4"></div>
<article class="author-bio mb-5 mb-md-4">
<div class="author-bio__content">
<div class="author-bio__avatar">
<picture>
<source srcset="https://avatars3.githubusercontent.com/u/349190?v=4&#038;s=200" width="120" height="120" media="(min-width: 768px)" />
<img class="d-block circle" src="https://avatars3.githubusercontent.com/u/349190?v=4&#038;s=200" alt="Theo Julienne" width="80" height="80" loading="lazy" decoding="async" >
</picture>
</div>
<header class="author-bio__header">
<h3 class="f3-mktg text-bold mb-1">
<a href="https://github.blog/author/theojulienne/" class="Link--primary color-fg-default">Theo Julienne</a>
</h3>
<p class="mb-0 f4 lh-condensed">
<a href="https://github.com/theojulienne" target="_blank">@theojulienne</a>
</p>
</header>
</div>
</article>
</div>
</section>
<aside id="sidebar" class="col-12 col-md-4 col-lg-3 post__sidebar" aria-label="Sidebar">
<div class="position-sticky top-md-scroll">
<div class="d-none d-md-block">
</div>
</div>
</aside>
</div>
</div>
<section class="container-xl p-responsive-blog mx-auto mt-12 mb-8">
<h2 class="h5-mktg mb-4 mb-lg-7 pb-3 border-bottom color-border-subtle">
Related posts </h2>
<div class="post-columns post-columns--3-3">
<article class="color-border-muted post-card">
<div class="d-flex flex-column">
<div class="d-block col-12 position-relative rounded-2 mb-3 overflow-hidden tease-thumbnail" target="_self">
<svg aria-hidden="true" width="1032" height="548" class="width-full height-auto d-block" role="presentation"></svg>
<img width="400" height="212" src="https://github.blog/wp-content/uploads/2026/01/generic-github-copilot-commit-logo.png?resize=400%2C212" class="d-block cover-image tease-thumbnail__img wp-post-image" alt="Copilot hovering above a mosaic of green squares in a decorative scene." decoding="async" loading="lazy" srcset="https://github.blog/wp-content/uploads/2026/01/generic-github-copilot-commit-logo.png?w=1600 1600w, https://github.blog/wp-content/uploads/2026/01/generic-github-copilot-commit-logo.png?w=800 800w, https://github.blog/wp-content/uploads/2026/01/generic-github-copilot-commit-logo.png?w=400 400w, https://github.blog/wp-content/uploads/2026/01/generic-github-copilot-commit-logo.png?w=1032 1032w, https://github.blog/wp-content/uploads/2026/01/generic-github-copilot-commit-logo.png?w=516 516w, https://github.blog/wp-content/uploads/2026/01/generic-github-copilot-commit-logo.png?zoom=3&amp;resize=400%2C212 1200w" sizes="auto, (max-width: 400px) 100vw, 400px" /> </div>
<div class="mb-1"><a href="https://github.blog/ai-and-ml/" class="first-post-category f5-mktg text-gradient-purple-coral text-bold pb-1 position-relative z-2">AI &amp; ML</a></div> <div class="">
<h3 class="h6-mktg mb-2">
<a class="Link--primary post-card__link" href="https://github.blog/ai-and-ml/github-copilot/how-we-make-ai-coding-more-cost-efficient-without-sacrificing-task-quality/" target="_self">
How we make AI coding more cost efficient without sacrificing task quality </a>
</h3>
<div class="f4-mktg color-fg-muted">
<p>Why shorter outputs can cost more, and how GitHub Copilot reduces wasted work across the complete coding task.</p>
</div>
</div>
<footer>
<div class="d-flex flex-items-center mb-6px">
<span>
<span class="text-bold">
<a href="https://github.blog/author/erikkrogh/" title="Posts by Erik Kristensen" class="author url fn Link--primary no-wrap position-relative z-2" rel="author">Erik Kristensen</a> & <a href="https://github.blog/author/napalys/" title="Posts by Napalys Klicius" class="author url fn Link--primary no-wrap position-relative z-2" rel="author">Napalys Klicius</a> </span>
</span>
</div>
</footer>
</div>
</article>
<article class="color-border-muted post-card">
<div class="d-flex flex-column">
<div class="d-block col-12 position-relative rounded-2 mb-3 overflow-hidden tease-thumbnail" target="_self">
<svg aria-hidden="true" width="1032" height="548" class="width-full height-auto d-block" role="presentation"></svg>
<img width="400" height="212" src="https://github.blog/wp-content/uploads/2026/08/header-1.png?resize=400%2C212" class="d-block cover-image tease-thumbnail__img wp-post-image" alt="Humorous header image with &#039;alt=IMG_2847.png&#039; written across the top, and Mona&#039;s head looking up at a cube next to a GitHub logo on the bottom." decoding="async" loading="lazy" srcset="https://github.blog/wp-content/uploads/2026/08/header-1.png?w=300 300w, https://github.blog/wp-content/uploads/2026/08/header-1.png?w=1600 1600w, https://github.blog/wp-content/uploads/2026/08/header-1.png?w=800 800w, https://github.blog/wp-content/uploads/2026/08/header-1.png?w=400 400w, https://github.blog/wp-content/uploads/2026/08/header-1.png?w=1032 1032w, https://github.blog/wp-content/uploads/2026/08/header-1.png?w=516 516w, https://github.blog/wp-content/uploads/2026/08/header-1.png?zoom=3&amp;resize=400%2C212 1200w" sizes="auto, (max-width: 400px) 100vw, 400px" /> </div>
<div class="mb-1"><a href="https://github.blog/engineering/" class="first-post-category f5-mktg text-gradient-purple-coral text-bold pb-1 position-relative z-2">Engineering</a></div> <div class="">
<h3 class="h6-mktg mb-2">
<a class="Link--primary post-card__link" href="https://github.blog/engineering/user-experience/your-alt-text-passes-automated-checks-that-doesnt-mean-its-any-good/" target="_self">
Your alt text passes automated checks. That doesn&#8217;t mean it&#8217;s any good. </a>
</h3>
<div class="f4-mktg color-fg-muted">
<p>We built a plugin for the GitHub Accessibility Scanner to make sure your alt text is actually accessible. Here&#8217;s how it works.</p>
</div>
</div>
<footer>
<div class="d-flex flex-items-center mb-6px">
<span>
<span class="text-bold">
<a href="https://github.blog/author/taarikashenafi/" title="Posts by Taarik Ashenafi" class="author url fn Link--primary no-wrap position-relative z-2" rel="author">Taarik Ashenafi</a> & <a href="https://github.blog/author/kzhou314/" title="Posts by Keenan Zhou" class="author url fn Link--primary no-wrap position-relative z-2" rel="author">Keenan Zhou</a> </span>
</span>
</div>
</footer>
</div>
</article>
<article class="color-border-muted post-card">
<div class="d-flex flex-column">
<div class="d-block col-12 position-relative rounded-2 mb-3 overflow-hidden tease-thumbnail" target="_self">
<svg aria-hidden="true" width="1032" height="548" class="width-full height-auto d-block" role="presentation"></svg>
<img width="400" height="212" src="https://github.blog/wp-content/uploads/2026/01/generic-copilot-flying-invertocat-logo-github.png?resize=400%2C212" class="d-block cover-image tease-thumbnail__img wp-post-image" alt="Copilot hovering above geometric blocks featuring the GitHub invertocat logo in a decorative scene." decoding="async" loading="lazy" srcset="https://github.blog/wp-content/uploads/2026/01/generic-copilot-flying-invertocat-logo-github.png?w=1600 1600w, https://github.blog/wp-content/uploads/2026/01/generic-copilot-flying-invertocat-logo-github.png?w=800 800w, https://github.blog/wp-content/uploads/2026/01/generic-copilot-flying-invertocat-logo-github.png?w=400 400w, https://github.blog/wp-content/uploads/2026/01/generic-copilot-flying-invertocat-logo-github.png?w=1032 1032w, https://github.blog/wp-content/uploads/2026/01/generic-copilot-flying-invertocat-logo-github.png?w=516 516w, https://github.blog/wp-content/uploads/2026/01/generic-copilot-flying-invertocat-logo-github.png?zoom=3&amp;resize=400%2C212 1200w" sizes="auto, (max-width: 400px) 100vw, 400px" /> </div>
<div class="mb-1"><a href="https://github.blog/ai-and-ml/" class="first-post-category f5-mktg text-gradient-purple-coral text-bold pb-1 position-relative z-2">AI &amp; ML</a></div> <div class="">
<h3 class="h6-mktg mb-2">
<a class="Link--primary post-card__link" href="https://github.blog/engineering/using-the-github-copilot-sdk-for-java/" target="_self">
Using the GitHub Copilot SDK for Java </a>
</h3>
<div class="f4-mktg color-fg-muted">
<p>Enterprise Java developers have a new superpower—drive GitHub Copilot from idiomatic Java code with annotations, virtual threads, and more.</p>
</div>
</div>
<footer>
<div class="d-flex flex-items-center mb-6px">
<span>
<span class="text-bold">
<a href="https://github.blog/author/edburns/" title="Posts by Edward Burns" class="author url fn Link--primary no-wrap position-relative z-2" rel="author">Edward Burns</a> </span>
</span>
</div>
</footer>
</div>
</article>
</div>
</section>
<aside class="container-xl p-responsive-blog mx-auto mb-12">
<h2 class="h5-mktg border-bottom pb-3">
Explore more from GitHub </h2>
<div class="d-flex flex-wrap ml-md-n4">
<div class="col-12 col-md-6 col-lg-3 d-flex mt-4 mt-lg-6 pl-md-4">
<div class="rounded-3 color-bg-subtle d-flex flex-column flex-items-start width-full p-5 f4-mktg color-fg-muted">
<img src="https://github.blog/wp-content/uploads/2024/07/Icon-Circle.svg" width="44" height="44" class="width-auto d-block mb-3" alt="Docs"/> <h3 class="f3-mktg text-bold color-fg-default mb-3">
Docs </h3>
<div class="mb-auto">
<p>Everything you need to master GitHub, all in one place.</p>
</div>
<div class="mt-7">
<a data-analytics-click="Blog, click on module, text: Go to Docs; ref_location:bottom recirculation;" href="https://docs.github.com/" class="text-semibold arrow-target-mktg Link--primary" target="_blank" aria-label="Go to Docs">
Go to Docs <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-link-external ml-1"><path fill-rule="evenodd" d="M10.604 1h4.146a.25.25 0 01.25.25v4.146a.25.25 0 01-.427.177L13.03 4.03 9.28 7.78a.75.75 0 01-1.06-1.06l3.75-3.75-1.543-1.543A.25.25 0 0110.604 1zM3.75 2A1.75 1.75 0 002 3.75v8.5c0 .966.784 1.75 1.75 1.75h8.5A1.75 1.75 0 0014 12.25v-3.5a.75.75 0 00-1.5 0v3.5a.25.25 0 01-.25.25h-8.5a.25.25 0 01-.25-.25v-8.5a.25.25 0 01.25-.25h3.5a.75.75 0 000-1.5h-3.5z"></path></svg>
</a>
</div>
</div>
</div>
<div class="col-12 col-md-6 col-lg-3 d-flex mt-4 mt-lg-6 pl-md-4">
<div class="rounded-3 color-bg-subtle d-flex flex-column flex-items-start width-full p-5 f4-mktg color-fg-muted">
<img src="https://github.blog/wp-content/uploads/2024/07/recirculation-github-icon.svg" width="44" height="44" class="width-auto d-block mb-3" alt="GitHub"/> <h3 class="f3-mktg text-bold color-fg-default mb-3">
GitHub </h3>
<div class="mb-auto">
<p>Build what&#8217;s next on GitHub, the place for anyone from anywhere to build anything.</p>
</div>
<div class="mt-7">
<a data-analytics-click="Blog, click on module, text: Start building; ref_location:bottom recirculation;" href="https://github.com/" class="text-semibold arrow-target-mktg Link--primary" target="_blank" aria-label="Start building">
Start building <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-link-external ml-1"><path fill-rule="evenodd" d="M10.604 1h4.146a.25.25 0 01.25.25v4.146a.25.25 0 01-.427.177L13.03 4.03 9.28 7.78a.75.75 0 01-1.06-1.06l3.75-3.75-1.543-1.543A.25.25 0 0110.604 1zM3.75 2A1.75 1.75 0 002 3.75v8.5c0 .966.784 1.75 1.75 1.75h8.5A1.75 1.75 0 0014 12.25v-3.5a.75.75 0 00-1.5 0v3.5a.25.25 0 01-.25.25h-8.5a.25.25 0 01-.25-.25v-8.5a.25.25 0 01.25-.25h3.5a.75.75 0 000-1.5h-3.5z"></path></svg>
</a>
</div>
</div>
</div>
<div class="col-12 col-md-6 col-lg-3 d-flex mt-4 mt-lg-6 pl-md-4">
<div class="rounded-3 color-bg-subtle d-flex flex-column flex-items-start width-full p-5 f4-mktg color-fg-muted">
<img src="https://github.blog/wp-content/uploads/2024/07/Icon_da43dc.svg" width="44" height="44" class="width-auto d-block mb-3" alt="Customer stories"/> <h3 class="f3-mktg text-bold color-fg-default mb-3">
Customer stories </h3>
<div class="mb-auto">
<p>Meet the companies and engineering teams that build with GitHub.</p>
</div>
<div class="mt-7">
<a data-analytics-click="Blog, click on module, text: Learn more; ref_location:bottom recirculation;" href="https://github.com/customer-stories" class="text-semibold arrow-target-mktg Link--primary" target="_blank" aria-label="Learn more">
Learn more <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-link-external ml-1"><path fill-rule="evenodd" d="M10.604 1h4.146a.25.25 0 01.25.25v4.146a.25.25 0 01-.427.177L13.03 4.03 9.28 7.78a.75.75 0 01-1.06-1.06l3.75-3.75-1.543-1.543A.25.25 0 0110.604 1zM3.75 2A1.75 1.75 0 002 3.75v8.5c0 .966.784 1.75 1.75 1.75h8.5A1.75 1.75 0 0014 12.25v-3.5a.75.75 0 00-1.5 0v3.5a.25.25 0 01-.25.25h-8.5a.25.25 0 01-.25-.25v-8.5a.25.25 0 01.25-.25h3.5a.75.75 0 000-1.5h-3.5z"></path></svg>
</a>
</div>
</div>
</div>
<div class="col-12 col-md-6 col-lg-3 d-flex mt-4 mt-lg-6 pl-md-4">
<div class="rounded-3 color-bg-subtle d-flex flex-column flex-items-start width-full p-5 f4-mktg color-fg-muted">
<img src="https://github.blog/wp-content/uploads/2025/06/Universe26-Icon.svg" width="44" height="44" class="width-auto d-block mb-3" alt="GitHub Universe 2026"/> <h3 class="f3-mktg text-bold color-fg-default mb-3">
GitHub Universe 2026 </h3>
<div class="mb-auto">
<p>Join us October 28-29 in San Francisco or online for GitHub Universe, our flagship developer event uniting people, agents, and the world’s code.</p>
</div>
<div class="mt-7">
<a data-analytics-click="Blog, click on module, text: Register now; ref_location:bottom recirculation;" href="https://githubuniverse.com/?utm_source=Blog&#038;utm_medium=GitHub&#038;utm_campaign=module_uni_26" class="text-semibold arrow-target-mktg Link--primary" target="_blank" aria-label="Register now">
Register now <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16" class="octicon octicon-link-external ml-1"><path fill-rule="evenodd" d="M10.604 1h4.146a.25.25 0 01.25.25v4.146a.25.25 0 01-.427.177L13.03 4.03 9.28 7.78a.75.75 0 01-1.06-1.06l3.75-3.75-1.543-1.543A.25.25 0 0110.604 1zM3.75 2A1.75 1.75 0 002 3.75v8.5c0 .966.784 1.75 1.75 1.75h8.5A1.75 1.75 0 0014 12.25v-3.5a.75.75 0 00-1.5 0v3.5a.25.25 0 01-.25.25h-8.5a.25.25 0 01-.25-.25v-8.5a.25.25 0 01.25-.25h3.5a.75.75 0 000-1.5h-3.5z"></path></svg>
</a>
</div>
</div>
</div>
</div>
</aside>
<div class="py-6 py-lg-8 color-bg-subtle" data-color-mode="dark" data-light-theme="light" data-dark-theme="dark">
<div class="container-xl p-responsive-blog">
<section id="newsletter" class="newsletter rounded-2">
<div class="color-bg-subtle rounded-2">
<div class="d-flex flex-row flex-wrap gutter-spacious py-5 px-4 py-md-6 px-md-5">
<div class="newsletter-content col-12 col-lg-6 col-xl-7">
<h2 class="h4-mktg color-fg-default">We do newsletters, too</h2><p class="f3-mktg color-fg-muted mt-2 mb-3">Discover tips, technical guides, and best practices in our biweekly newsletter just for devs.</p> </div>
<div class="col-12 col-lg-6 col-xl-5">
<label for="newsletter_emailAddress" class="">Your email address</label>
<form method="post" action="https://s88570519.t.eloqua.com/e/f2?elqFormName=copynewsletter-signup-form-637872624660309567&elqSiteID=88570519" class="js-form-validator newsletter-form">
<div class="newsletter-form d-md-flex flex-row mt-0 mb-3 rounded-2 required form-group">
<div class="m-0 width-full newsletter-input-container">
<span aria-hidden="true" class="required-indicator">*</span>
<label for="newsletter_emailAddress" class="screen-reader-text">Your email address</label>
<input type="email" required id="newsletter_emailAddress" name="emailAddress" placeholder="Your email address" class="required js-validity-check d-block width-full height-md-full f4-mktg newsletter-field rounded-2" />
</div>
<input type="hidden" name="classification" value="Practitioner">
<button type="submit" class="js-form-validator-submit newsletter-submit flex-shrink-0 arrow-target-mktg d-none d-md-flex flex-row f3-mktg flex-items-center text-semibold rounded-2">
<span>Subscribe</span>
<svg xmlns="http://www.w3.org/2000/svg" class="octicon arrow-symbol-mktg" width="24" height="24" viewBox="0 0 16 16" fill="none"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="octicon-chevrow-stem" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg>
</button>
</div>
<div class="form-checkbox color-fg-muted f6-mktg m-0 mt-2 form-group">
<label class="text-normal">
<input id="optincheckbox" name="marketingEmailOptIn1" type="checkbox" value="True">
Yes please, I’d like GitHub and affiliates to use my information for personalized communications, targeted advertising and campaign effectiveness. See the <a href="https://github.com/site/privacy" target="blank">GitHub Privacy Statement</a> for more details. </label>
</div>
<button type="submit" class="js-form-validator-submit newsletter-submit flex-shrink-0 arrow-target-mktg mt-3 d-flex d-md-none flex-row f3-mktg flex-items-center text-semibold rounded-2">
<span>Subscribe</span>
<svg xmlns="http://www.w3.org/2000/svg" class="octicon arrow-symbol-mktg" width="24" height="24" viewBox="0 0 16 16" fill="none"><path fill="currentColor" d="M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z"></path><path class="octicon-chevrow-stem" stroke="currentColor" d="M1.75 8H11" stroke-width="1.5" stroke-linecap="round"></path></svg>
</button>
</form>
</div>
</div>
</div>
</section>
</div>
</div>
</main>
<div data-color-mode="dark" data-light-theme="light" data-dark-theme="dark">
<footer class="footer pt-6">
<h2 class="sr-only">Site-wide Links</h2>
<div class="container-xl p-responsive-blog">
<div class="d-flex flex-wrap py-5 mb-5">
<div class="col-12 col-lg-4 mb-5">
<a href="https://github.com/" data-ga-click="Resources, go to home, resources footer" class="color-fg-default" aria-label="Go to GitHub homepage">
<svg height="30" class="octicon octicon-logo-github" viewBox="0 0 45 16" version="1.1" width="84" aria-hidden="true">
<path fill-rule="evenodd" d="M18.53 12.03h-.02c.009 0 .015.01.024.011h.006l-.01-.01zm.004.011c-.093.001-.327.05-.574.05-.78 0-1.05-.36-1.05-.83V8.13h1.59c.09 0 .16-.08.16-.19v-1.7c0-.09-.08-.17-.16-.17h-1.59V3.96c0-.08-.05-.13-.14-.13h-2.16c-.09 0-.14.05-.14.13v2.17s-1.09.27-1.16.28c-.08.02-.13.09-.13.17v1.36c0 .11.08.19.17.19h1.11v3.28c0 2.44 1.7 2.69 2.86 2.69.53 0 1.17-.17 1.27-.22.06-.02.09-.09.09-.16v-1.5a.177.177 0 00-.146-.18zM42.23 9.84c0-1.81-.73-2.05-1.5-1.97-.6.04-1.08.34-1.08.34v3.52s.49.34 1.22.36c1.03.03 1.36-.34 1.36-2.25zm2.43-.16c0 3.43-1.11 4.41-3.05 4.41-1.64 0-2.52-.83-2.52-.83s-.04.46-.09.52c-.03.06-.08.08-.14.08h-1.48c-.1 0-.19-.08-.19-.17l.02-11.11c0-.09.08-.17.17-.17h2.13c.09 0 .17.08.17.17v3.77s.82-.53 2.02-.53l-.01-.02c1.2 0 2.97.45 2.97 3.88zm-8.72-3.61h-2.1c-.11 0-.17.08-.17.19v5.44s-.55.39-1.3.39-.97-.34-.97-1.09V6.25c0-.09-.08-.17-.17-.17h-2.14c-.09 0-.17.08-.17.17v5.11c0 2.2 1.23 2.75 2.92 2.75 1.39 0 2.52-.77 2.52-.77s.05.39.08.45c.02.05.09.09.16.09h1.34c.11 0 .17-.08.17-.17l.02-7.47c0-.09-.08-.17-.19-.17zm-23.7-.01h-2.13c-.09 0-.17.09-.17.2v7.34c0 .2.13.27.3.27h1.92c.2 0 .25-.09.25-.27V6.23c0-.09-.08-.17-.17-.17zm-1.05-3.38c-.77 0-1.38.61-1.38 1.38 0 .77.61 1.38 1.38 1.38.75 0 1.36-.61 1.36-1.38 0-.77-.61-1.38-1.36-1.38zm16.49-.25h-2.11c-.09 0-.17.08-.17.17v4.09h-3.31V2.6c0-.09-.08-.17-.17-.17h-2.13c-.09 0-.17.08-.17.17v11.11c0 .09.09.17.17.17h2.13c.09 0 .17-.08.17-.17V8.96h3.31l-.02 4.75c0 .09.08.17.17.17h2.13c.09 0 .17-.08.17-.17V2.6c0-.09-.08-.17-.17-.17zM8.81 7.35v5.74c0 .04-.01.11-.06.13 0 0-1.25.89-3.31.89-2.49 0-5.44-.78-5.44-5.92S2.58 1.99 5.1 2c2.18 0 3.06.49 3.2.58.04.05.06.09.06.14L7.94 4.5c0 .09-.09.2-.2.17-.36-.11-.9-.33-2.17-.33-1.47 0-3.05.42-3.05 3.73s1.5 3.7 2.58 3.7c.92 0 1.25-.11 1.25-.11v-2.3H4.88c-.11 0-.19-.08-.19-.17V7.35c0-.09.08-.17.19-.17h3.74c.11 0 .19.08.19.17z"></path>
</svg>
</a>
</div>
<nav aria-labelledby="product-menu" class="col-6 col-sm-3 col-lg-2 mb-6 mb-md-2 pr-3 pr-lg-0 pl-lg-4"><h3 id="product-menu" class="h5 mb-3 text-mono color-text-tertiary text-normal">Product</h3><ul class="list-style-none text-gray f5"><li class="lh-condensed mb-3"><a href="https://github.com/features" data-ga-click="Site Foundation Components, go to Features, site foundation components footer" class="Link--secondary">Features</a></li><li class="lh-condensed mb-3"><a href="https://github.com/security" data-ga-click="Site Foundation Components, go to Security, site foundation components footer" class="Link--secondary">Security</a></li><li class="lh-condensed mb-3"><a href="https://github.com/enterprise" data-ga-click="Site Foundation Components, go to Enterprise, site foundation components footer" class="Link--secondary">Enterprise</a></li><li class="lh-condensed mb-3"><a href="https://github.com/customer-stories?type=enterprise" data-ga-click="Site Foundation Components, go to Customer Stories, site foundation components footer" class="Link--secondary">Customer Stories</a></li><li class="lh-condensed mb-3"><a href="https://github.com/pricing" data-ga-click="Site Foundation Components, go to Pricing, site foundation components footer" class="Link--secondary">Pricing</a></li><li class="lh-condensed mb-3"><a href="https://resources.github.com/" data-ga-click="Site Foundation Components, go to Resources, site foundation components footer" class="Link--secondary">Resources</a></li></ul></nav><nav aria-labelledby="platform-menu" class="col-6 col-sm-3 col-lg-2 mb-6 mb-md-2 pr-3 pr-lg-0 pl-lg-4"><h3 id="platform-menu" class="h5 mb-3 text-mono color-text-tertiary text-normal">Platform</h3><ul class="list-style-none text-gray f5"><li class="lh-condensed mb-3"><a href="https://developer.github.com/" data-ga-click="Site Foundation Components, go to Developer API, site foundation components footer" class="Link--secondary">Developer API</a></li><li class="lh-condensed mb-3"><a href="https://partner.github.com/" data-ga-click="Site Foundation Components, go to Partners, site foundation components footer" class="Link--secondary">Partners</a></li><li class="lh-condensed mb-3"><a href="https://atom.io/" data-ga-click="Site Foundation Components, go to Atom, site foundation components footer" class="Link--secondary">Atom</a></li><li class="lh-condensed mb-3"><a href="https://www.electronjs.org/" data-ga-click="Site Foundation Components, go to Electron, site foundation components footer" class="Link--secondary">Electron</a></li><li class="lh-condensed mb-3"><a href="https://desktop.github.com/" data-ga-click="Site Foundation Components, go to GitHub Desktop, site foundation components footer" class="Link--secondary">GitHub Desktop</a></li></ul></nav><nav aria-labelledby="support-menu" class="col-6 col-sm-3 col-lg-2 mb-6 mb-md-2 pr-3 pr-lg-0 pl-lg-4"><h3 id="support-menu" class="h5 mb-3 text-mono color-text-tertiary text-normal">Support</h3><ul class="list-style-none text-gray f5"><li class="lh-condensed mb-3"><a href="https://docs.github.com/" data-ga-click="Site Foundation Components, go to Docs, site foundation components footer" class="Link--secondary">Docs</a></li><li class="lh-condensed mb-3"><a href="https://github.community/" data-ga-click="Site Foundation Components, go to Community Forum, site foundation components footer" class="Link--secondary">Community Forum</a></li><li class="lh-condensed mb-3"><a href="https://services.github.com/" data-ga-click="Site Foundation Components, go to Training, site foundation components footer" class="Link--secondary">Training</a></li><li class="lh-condensed mb-3"><a href="https://www.githubstatus.com/" data-ga-click="Site Foundation Components, go to Status, site foundation components footer" class="Link--secondary">Status</a></li><li class="lh-condensed mb-3"><a href="https://support.github.com/" data-ga-click="Site Foundation Components, go to Contact, site foundation components footer" class="Link--secondary">Contact</a></li></ul></nav><nav aria-labelledby="company-menu" class="col-6 col-sm-3 col-lg-2 mb-6 mb-md-2 pr-3 pr-lg-0 pl-lg-4"><h3 id="company-menu" class="h5 mb-3 text-mono color-text-tertiary text-normal">Company</h3><ul class="list-style-none text-gray f5"><li class="lh-condensed mb-3"><a href="https://github.com/about" data-ga-click="Site Foundation Components, go to About, site foundation components footer" class="Link--secondary">About</a></li><li class="lh-condensed mb-3"><a href="https://github.blog/" data-ga-click="Site Foundation Components, go to Blog, site foundation components footer" class="Link--secondary">Blog</a></li><li class="lh-condensed mb-3"><a href="https://github.com/about/careers" data-ga-click="Site Foundation Components, go to Careers, site foundation components footer" class="Link--secondary">Careers</a></li><li class="lh-condensed mb-3"><a href="https://github.com/about/press" data-ga-click="Site Foundation Components, go to Press, site foundation components footer" class="Link--secondary">Press</a></li><li class="lh-condensed mb-3"><a href="https://shop.github.com/" data-ga-click="Site Foundation Components, go to Shop, site foundation components footer" class="Link--secondary">Shop</a></li></ul></nav>
</div>
</div>
<div class="color-bg-subtle">
<div class="container-xl p-responsive-blog f6 py-4 d-sm-flex flex-justify-between flex-items-center">
<ul class="list-style-none d-flex flex-wrap text-gray mb-3">
<li class="mr-3">&copy; 2026 GitHub, Inc.</li>
<li class="mr-3">
<a href="https://docs.github.com/en/github/site-policy/github-terms-of-service" data-ga-click="Site Foundation Components, go to terms, site foundation components footer" class="Link--secondary">Terms</a>
</li>
<li class="mr-3">
<a href="https://docs.github.com/en/github/site-policy/github-privacy-statement" data-ga-click="Site Foundation Components, go to privacy, site foundation components footer" class="Link--secondary">Privacy</a>
</li>
<li class="mr-3">
<button type="button" class="btn-link Link--secondary" onClick="_ghcc.showPreferences()">Manage Cookies</button>
</li>
<li class="mr-3">
<button type="button" class="btn-link Link--secondary" onClick="_ghcc.showPreferences()">Do not share my personal information</button>
</li>
</ul>
<ul class="list-style-none d-flex flex-items-center mb-sm-0 lh-condensed-ultra">
<li class="mr-3 flex-self-start focusable-svg">
<a href="https://www.linkedin.com/company/github" data-ga-click="Blog, go to Linkedin, resources footer" style="color: #959da5;">
<svg focusable="false" tabindex="-1" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 19 18" class="d-block" height="18">
<title>LinkedIn icon</title>
<path d="M3.94 2A2 2 0 1 1 2 0a2 2 0 0 1 1.94 2zM4 5.48H0V18h4zm6.32 0H6.34V18h3.94v-6.57c0-3.66 4.77-4 4.77 0V18H19v-7.93c0-6.17-7.06-5.94-8.72-2.91z" fill="currentColor"></path>
</svg>
<span class="sr-only">GitHub on LinkedIn</span>
</a>
</li>
<li class="mr-3 focusable-svg">
<a href="https://www.instagram.com/github/" data-ga-click="Blog, go to Instagram, resources footer" style="color: #959da5;">
<svg focusable="false" tabindex="-1" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 169.063 169.063" class="d-block" height="18">
<title>Instagram icon</title>
<g>
<path d="M122.406,0H46.654C20.929,0,0,20.93,0,46.655v75.752c0,25.726,20.929,46.655,46.654,46.655h75.752
c25.727,0,46.656-20.93,46.656-46.655V46.655C169.063,20.93,148.133,0,122.406,0z M154.063,122.407
c0,17.455-14.201,31.655-31.656,31.655H46.654C29.2,154.063,15,139.862,15,122.407V46.655C15,29.201,29.2,15,46.654,15h75.752
c17.455,0,31.656,14.201,31.656,31.655V122.407z" fill="currentColor"/>
<path d="M84.531,40.97c-24.021,0-43.563,19.542-43.563,43.563c0,24.02,19.542,43.561,43.563,43.561s43.563-19.541,43.563-43.561
C128.094,60.512,108.552,40.97,84.531,40.97z M84.531,113.093c-15.749,0-28.563-12.812-28.563-28.561
c0-15.75,12.813-28.563,28.563-28.563s28.563,12.813,28.563,28.563C113.094,100.281,100.28,113.093,84.531,113.093z" fill="currentColor"/>
<path d="M129.921,28.251c-2.89,0-5.729,1.17-7.77,3.22c-2.051,2.04-3.23,4.88-3.23,7.78c0,2.891,1.18,5.73,3.23,7.78
c2.04,2.04,4.88,3.22,7.77,3.22c2.9,0,5.73-1.18,7.78-3.22c2.05-2.05,3.22-4.89,3.22-7.78c0-2.9-1.17-5.74-3.22-7.78
C135.661,29.421,132.821,28.251,129.921,28.251z" fill="currentColor"/>
</g>
</svg>
<span class="sr-only">GitHub on Instagram</span>
</a>
</li>
<li class="mr-3 focusable-svg">
<a href="https://www.youtube.com/github" data-ga-click="Blog, go to YouTube, resources footer" style="color: #959da5;">
<svg focusable="false" tabindex="-1" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 19.17 13.6" class="d-block" height="16">
<title>YouTube icon</title>
<path d="M18.77 2.13A2.4 2.4 0 0 0 17.09.42C15.59 0 9.58 0 9.58 0a57.55 57.55 0 0 0-7.5.4A2.49 2.49 0 0 0 .39 2.13 26.27 26.27 0 0 0 0 6.8a26.15 26.15 0 0 0 .39 4.67 2.43 2.43 0 0 0 1.69 1.71c1.52.42 7.5.42 7.5.42a57.69 57.69 0 0 0 7.51-.4 2.4 2.4 0 0 0 1.68-1.71 25.63 25.63 0 0 0 .4-4.67 24 24 0 0 0-.4-4.69zM7.67 9.71V3.89l5 2.91z" fill="currentColor"></path>
</svg>
<span class="sr-only">GitHub on YouTube</span>
</a>
</li>
<li class="mr-3 focusable-svg">
<a href="https://twitter.com/github" data-ga-click="Blog, go to X, resources footer" style="color: #959da5;">
<svg focusable="false" tabindex="-1" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1200 1227" class="d-block" height="18">
<title>X icon</title>
<path xmlns="http://www.w3.org/2000/svg" d="M714.163 519.284 1160.89 0h-105.86L667.137 450.887 357.328 0H0l468.492 681.821L0 1226.37h105.866l409.625-476.152 327.181 476.152H1200L714.137 519.284h.026ZM569.165 687.828l-47.468-67.894-377.686-540.24h162.604l304.797 435.991 47.468 67.894 396.2 566.721H892.476L569.165 687.854v-.026Z" fill="currentColor"></path>
</svg>
<span class="sr-only">GitHub on X</span>
</a>
</li>
<li class="mr-3 flex-self-start focusable-svg">
<a href="https://www.tiktok.com/@github" data-ga-click="Blog, go to TikTok, resources footer" style="color: #959da5;">
<svg focusable="false" tabindex="-1" aria-hidden="true" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" class="d-block" height="18">
<title>TikTok icon</title>
<path d="M12.525.02c1.31-.02 2.61-.01 3.91-.02.08 1.53.63 3.09 1.75 4.17 1.12 1.11 2.7 1.62 4.24 1.79v4.03c-1.44-.05-2.89-.35-4.2-.97-.57-.26-1.1-.59-1.62-.93-.01 2.92.01 5.84-.02 8.75-.08 1.4-.54 2.79-1.35 3.94-1.31 1.92-3.58 3.17-5.91 3.21-1.43.08-2.86-.31-4.08-1.03-2.02-1.19-3.44-3.37-3.65-5.71-.02-.5-.03-1-.01-1.49.18-1.9 1.12-3.72 2.58-4.96 1.66-1.44 3.98-2.13 6.15-1.72.02 1.48-.04 2.96-.04 4.44-.99-.32-2.15-.23-3.02.37-.63.41-1.11 1.04-1.36 1.75-.21.51-.15 1.07-.14 1.61.24 1.64 1.82 3.02 3.5 2.87 1.12-.01 2.19-.66 2.77-1.61.19-.33.4-.67.41-1.06.1-1.79.06-3.57.07-5.36.01-4.03-.01-8.05.02-12.07z" fill="currentColor"/>
</svg>
<span class="sr-only">GitHub on TikTok</span>
</a>
</li>
<li class="mr-3 flex-self-start focusable-svg">
<a href="https://www.twitch.tv/github" data-ga-click="Blog, go to Twitch, resources footer" style="color: #959da5;">
<svg focusable="false" tabindex="-1" aria-hidden="true" role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" class="d-block" height="18">
<title>Twitch icon</title>
<path d="M11.571 4.714h1.715v5.143H11.57zm4.715 0H18v5.143h-1.714zM6 0L1.714 4.286v15.428h5.143V24l4.286-4.286h3.428L22.286 12V0zm14.571 11.143l-3.428 3.428h-3.429l-3 3v-3H6.857V1.714h13.714Z" fill="currentColor"/>
</svg>
<span class="sr-only">GitHub on Twitch</span>
</a>
</li>
<li class="focusable-svg">
<a href="https://github.com/github" data-ga-click="Blog, go to github's org, resources footer" style="color: #959da5;">
<svg focusable="false" tabindex="-1" aria-hidden="true" height="20" class="octicon octicon-mark-github d-block" alt="" viewBox="0 0 98 96" fill="none" version="1.1" width="20" aria-hidden="true"><g clip-path="url(#a)">
<title>GitHub icon</title>
<path fill-rule="evenodd" d="M41.44 69.385C28.807 67.853 19.906 58.762 19.906 46.99c0-4.785 1.723-9.953 4.594-13.398-1.244-3.158-1.053-9.858.383-12.633 3.828-.479 8.996 1.531 12.058 4.307 3.637-1.149 7.465-1.723 12.155-1.723 4.69 0 8.517.574 11.963 1.627 2.966-2.68 8.23-4.69 12.058-4.211 1.34 2.584 1.531 9.283.287 12.537 3.063 3.637 4.69 8.518 4.69 13.494 0 11.772-8.9 20.672-21.725 22.3 3.254 2.104 5.455 6.698 5.455 11.962v9.953c0 2.871 2.393 4.498 5.264 3.35C84.41 87.95 98 70.629 98 49.19 98 22.107 75.988 0 48.904 0 21.82 0 0 22.107 0 49.191c0 21.246 13.494 38.856 31.678 45.46 2.584.956 5.072-.766 5.072-3.35v-7.657c-1.34.575-3.063.958-4.594.958-6.316 0-10.049-3.446-12.728-9.858-1.053-2.584-2.201-4.115-4.403-4.402-1.148-.096-1.53-.574-1.53-1.149 0-1.148 1.913-2.01 3.827-2.01 2.776 0 5.168 1.723 7.657 5.264 1.914 2.776 3.923 4.02 6.316 4.02 2.392 0 3.924-.861 6.125-3.063 1.627-1.627 2.871-3.062 4.02-4.02z"/></g><defs><clipPath id="a"><path d="M0 0h98v96H0z"/></clipPath></defs>
</svg>
<span class="sr-only">GitHub’s organization on GitHub</span>
</a>
</li>
</ul>
</div>
</div>
</footer>
</div>
<script type="speculationrules">
{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/github-2021-child/*","/wp-content/themes/github-2021/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}
</script>
<div id="ghcc" style="position: sticky; bottom: 0; z-index: 99999;"></div><script id="github_cookie_consent-js" src="https://ghcc.githubassets.com/ghcc.min.js"></script>
<script id="github_microsoft_analytics-js" src="https://js.monitor.azure.com/scripts/c/ms.analytics-web-4.js"></script>
<script type="text/javascript" src="https://github.blog/_static/??-eJx9jdEOwiAMRX9IVsZi3IvxWzYgUAZlkZL5+VYTjU++9px7CseubCX2xLDnHpAahCinuqF/kSZELeQU3xe7IQVw2BhSgz/OkNoJfsocffESRo59VUabUdmI2X1bGUPktT4+QySbu5OJsLcT5VkbCpIYt3IdL/M8aT2dTXoCiqVGIQ==" ></script><script id="wp-i18n-js" src="https://github.blog/wp-includes/js/dist/i18n.min.js?ver=1dfe7db3940c23ea9216"></script>
<script id="wp-i18n-js-after">
wp.i18n.setLocaleData( { 'text direction\u0004ltr': [ 'ltr' ] } );
//# sourceURL=wp-i18n-js-after
</script>
<script type="text/javascript" src="https://github.blog/_static/??/wp-includes/js/dist/private-apis.min.js,/wp-includes/js/dist/url.min.js?m=1787604914j" ></script><script id="wp-api-fetch-js" src="https://github.blog/wp-includes/js/dist/api-fetch.min.js?ver=6f2a4faeee3c722b1e57"></script>
<script id="wp-api-fetch-js-after">
wp.apiFetch.use( wp.apiFetch.createRootURLMiddleware( "https://github.blog/wp-json/" ) );
wp.apiFetch.nonceMiddleware = wp.apiFetch.createNonceMiddleware( "46518e8385" );
wp.apiFetch.use( wp.apiFetch.nonceMiddleware );
wp.apiFetch.use( wp.apiFetch.mediaUploadMiddleware );
wp.apiFetch.nonceEndpoint = "https://github.blog/wp-admin/admin-ajax.php?action=rest-nonce";
//# sourceURL=wp-api-fetch-js-after
</script>
<script type="text/javascript" src="https://github.blog/wp-content/themes/github-2021/dist/js/site-script.js?m=1788300352g" ></script><script id="highlightjs-js" src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.11.1/highlight.min.js?ver=11.4.0"></script>
<script id="highlightjs-js-after">
hljs.highlightAll()
//# sourceURL=highlightjs-js-after
</script>
<script type="text/javascript" src="https://github.blog/wp-content/plugins/page-links-to/dist/new-tab.js?m=1788300351g" ></script><script integrity="sha256-kAnFXX7lCXF9K2o4g5q1lKyk167yRq6C4TiXWtbgvQw=" crossorigin="anonymous" id="hydro-analytics-js" src="https://analytics.githubassets.com/v1.1.0/hydro-marketing.min.js?ver=7.1"></script>
<script id="jetpack-stats-js-before">
_stq = window._stq || [];
_stq.push([ "view", {"v":"ext","blog":"153214340","post":"51321","tz":"-7","srv":"github.blog","hp":"vip","j":"1:16.1.3"} ]);
_stq.push([ "clickTrackerInit", "153214340", "51321" ]);
//# sourceURL=jetpack-stats-js-before
</script>
<script data-wp-strategy="defer" defer fetchpriority="low" id="jetpack-stats-js" src="https://stats.wp.com/e-202637.js"></script>
<script id="wp-emoji-settings" type="application/json">
{"baseUrl":"https://s.w.org/images/core/emoji/17.0.2/72x72/","ext":".png","svgUrl":"https://s.w.org/images/core/emoji/17.0.2/svg/","svgExt":".svg","source":{"concatemoji":"https://github.blog/wp-includes/js/wp-emoji-release.min.js?ver=7.1"}}
</script>
<script type="module">
/*! This file is auto-generated */
var e="script#wp-emoji-settings",t=document.querySelector(e);if(!(t instanceof HTMLScriptElement))throw new Error("Element missing: "+e);const r=JSON.parse(t.text),s=(window._wpemojiSettings=r,"wpEmojiSettingsSupports"),o=["flag","emoji"];function i(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(s,JSON.stringify(t))}catch(e){}}function c(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0);const r=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data);return t.every((e,t)=>e===r[t])}function p(e,t){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var n=e.getImageData(16,16,1,1);for(let e=0;e<n.data.length;e++)if(0!==n.data[e])return!1;return!0}function u(e,t,n,r){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\udde8\ud83c\uddf6","\ud83c\udde8\u200b\ud83c\uddf6")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!r(e,"\ud83e\u1fac8")}return!1}function f(e,t,n,r){let a;const s=(a="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):document.createElement("canvas")).getContext("2d",{willReadFrequently:!0}),o=(s.textBaseline="top",s.font="600 32px Arial",{});return e.forEach(e=>{o[e]=t(s,e,n,r)}),o}function a(e){var t=document.createElement("script");t.src=e,t.defer=!0,document.head.appendChild(t)}r.supports={everything:!0,everythingExceptFlag:!0},new Promise(t=>{let n=function(){try{var e=JSON.parse(sessionStorage.getItem(s));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(o),u.toString(),c.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"});const a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=e=>{i(n=e.data),a.terminate(),t(n)})}catch(e){}i(n=f(o,u,c,p))}t(n)}).then(e=>{for(const n in e)r.supports[n]=e[n],r.supports.everything=r.supports.everything&&r.supports[n],"flag"!==n&&(r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&r.supports[n]);var t;r.supports.everythingExceptFlag=r.supports.everythingExceptFlag&&!r.supports.flag,r.supports.everything||((t=r.source||{}).concatemoji?a(t.concatemoji):t.wpemoji&&t.twemoji&&(a(t.twemoji),a(t.wpemoji)))});
//# sourceURL=https://github.blog/wp-includes/js/wp-emoji-loader.min.js
</script>
</body>
</html>