415 lines
252 KiB
HTML
415 lines
252 KiB
HTML
<!DOCTYPE html><html lang="en-US" data-country-code="us"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>How we diagnosed and resolved Redis latency spikes with BPF and other tools</title><link rel="preconnect" href="https://cdn.cookielaw.org"><link rel="preconnect" href="https://geolocation.onetrust.com"><link rel="preconnect" href="https://page.gitlab.com" data-hid="mkto-preconnect"><script src="https://client-registry.mutinycdn.com/personalize/client/c18972324098ea25.js" async type="text/plain" class="optanon-category-C0004"></script><script hid="gtagConsent" defer>
|
||
window.dataLayer = window.dataLayer || [];
|
||
function gtag(){dataLayer.push(arguments);}
|
||
|
||
const defaultConsents = window.dataLayer.filter(item => item.length && Array.from(item).indexOf('default') >= 0);
|
||
|
||
if(defaultConsents.length <= 2) {
|
||
gtag('consent', 'default', {
|
||
'analytics_storage': 'granted',
|
||
'ad_storage': 'granted',
|
||
'ad_user_data': 'granted',
|
||
'functionality_storage': 'granted',
|
||
'ad_personalization': 'granted',
|
||
'wait_for_update': 500
|
||
});
|
||
gtag('consent', 'default', {
|
||
'analytics_storage': 'denied',
|
||
'ad_storage': 'denied',
|
||
'ad_user_data': 'denied',
|
||
'functionality_storage': 'denied',
|
||
'ad_personalization': 'denied',
|
||
'region': [
|
||
'AT','BE','BG','HR','CY','CZ','DK','EE','FI','FR','DE','GR','HU','IE','IT','LV','LT','LU','MT','NL','PL','PT','RO','SK','SI','ES','SE','IS','LI','NO','RU','GB','PE','CO','KR','CA-QC'
|
||
],
|
||
'wait_for_update': 500
|
||
});
|
||
}
|
||
|
||
window.geofeed = (options) => {
|
||
dataLayer.push({
|
||
'event': 'OneTrustCountryLoad',
|
||
'oneTrustCountryId': options.country.toString(),
|
||
'oneTrustStateId': options.state?.toString()
|
||
});
|
||
}
|
||
</script><script id="oneTrustAutoBlocking" src="https://cdn.cookielaw.org/consent/7f944245-c5cd-4eed-a90e-dd955adfdd08/OtAutoBlock.js" type="text/javascript"></script><script id="oneTrustSDK" src="https://cdn.cookielaw.org/scripttemplates/otSDKStub.js" type="text/javascript" data-domain-script="7f944245-c5cd-4eed-a90e-dd955adfdd08"></script><script id="oneTrustGeolocation" src="https://geolocation.onetrust.com/cookieconsentpub/v1/geo/location/geofeed" type="text/javascript"></script><script src="https://cdn.optimizely.com/js/5113954737848320.js" type="text/plain" class="optanon-category-C0002"></script><script id="mutiny" defer>
|
||
(function () {
|
||
var a = (window.mutiny = window.mutiny || {});
|
||
if (!window.mutiny.client) {
|
||
a.client = { _queue: {} };
|
||
var b = ['identify', 'trackConversion'];
|
||
var c = [].concat(b, ['defaultOptOut', 'optOut', 'optIn']);
|
||
var d = function factory(c) {
|
||
return function () {
|
||
for (var d = arguments.length, e = new Array(d), f = 0; f < d; f++) {
|
||
e[f] = arguments[f];
|
||
}
|
||
a.client._queue[c] = a.client._queue[c] || [];
|
||
if (b.includes(c)) {
|
||
return new Promise(function (b, d) {
|
||
a.client._queue[c].push({ args: e, resolve: b, reject: d });
|
||
});
|
||
} else {
|
||
a.client._queue[c].push({ args: e });
|
||
}
|
||
};
|
||
};
|
||
c.forEach(function (b) {
|
||
a.client[b] = d(b);
|
||
});
|
||
}
|
||
})();
|
||
</script><script id="onetrust-initialization">
|
||
function OptanonWrapper() {
|
||
try {
|
||
setupConsentObservers();
|
||
} catch (e) {
|
||
console.error("OneTrust consent observer failed to setup", e);
|
||
}
|
||
}
|
||
function removeNonConsentedCookies(activeGroups) {
|
||
if (!window.OneTrust || !OneTrust.GetDomainData) return;
|
||
var activeArr = Array.isArray(activeGroups) ? activeGroups : String(activeGroups || '').split(',');
|
||
var groups = OneTrust.GetDomainData().Groups || [];
|
||
function rulesFor(wantConsented) {
|
||
var exact = {}, prefixes = [];
|
||
groups.forEach(function (g) {
|
||
if ((activeArr.indexOf(g.OptanonGroupId) !== -1) !== wantConsented) return;
|
||
(g.FirstPartyCookies || []).forEach(function (c) {
|
||
var x = c.Name.search(/x{2,}$/);
|
||
if (x === -1) exact[c.Name] = true; else if (x >= 2) prefixes.push(c.Name.slice(0, x));
|
||
});
|
||
});
|
||
return { exact: exact, prefixes: prefixes };
|
||
}
|
||
var keep = rulesFor(true), remove = rulesFor(false);
|
||
function hit(r, name) { return r.exact[name] || r.prefixes.some(function (p) { return name.indexOf(p) === 0; }); }
|
||
document.cookie.split('; ').forEach(function (pair) {
|
||
var name = pair.split('=')[0];
|
||
if (!name || !hit(remove, name) || hit(keep, name)) return;
|
||
['', location.hostname, '.' + location.hostname, '.gitlab.com'].forEach(function (d) {
|
||
document.cookie = name + '=; expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/' + (d ? '; domain=' + d : '');
|
||
});
|
||
});
|
||
}
|
||
function setupConsentObservers() {
|
||
// Check if onetrust and mutiny is active and opt in if right group is active
|
||
if (OnetrustActiveGroups && window.mutiny) {
|
||
if (OnetrustActiveGroups.includes('C0002') || OnetrustActiveGroups.includes('C0004')) {
|
||
window.mutiny.client.optIn();
|
||
}
|
||
}
|
||
|
||
removeNonConsentedCookies(OnetrustActiveGroups);
|
||
|
||
if (OneTrust) {
|
||
OneTrust.OnConsentChanged((consentEvent) => {
|
||
const mutinyEnabled = consentEvent.detail.includes('C0002') || consentEvent.detail.includes('C0004');
|
||
if (mutinyEnabled && window.mutiny) {
|
||
window.mutiny.client.optIn();
|
||
} else if (!mutinyEnabled) {
|
||
window.mutiny.client.optOut();
|
||
}
|
||
removeNonConsentedCookies(consentEvent.detail);
|
||
});
|
||
}
|
||
}
|
||
</script><style>.grid-wrapper[data-v-89b552f1]{--search-top-offset:68px;display:flex;flex-direction:column;overflow-x:clip}@media(min-width:1025px){.grid-wrapper--has-release-banner[data-v-89b552f1]{--search-top-offset:120px}}.grid-wrapper[data-v-89b552f1]>*{max-width:100%;min-width:0}.banner[data-v-89b552f1]{order:0}@media screen and (min-width:768px){.banner[data-v-89b552f1]{order:-1}}</style><style>.navigation[data-v-07f51b8b]{background:#fff;left:0;position:sticky;right:0;top:0;z-index:1025}.navigation__desktop[data-v-07f51b8b]{align-items:center;display:flex;height:68px;justify-content:space-between;padding:0 24px}.navigation__logo[data-v-07f51b8b]{margin-right:8px}@media not (max-width:1200px){.navigation__logo[data-v-07f51b8b]{margin-right:24px}}.navigation__menu[data-v-07f51b8b]{align-items:center;display:flex;list-style:none;margin:0;padding:0}.navigation__menu-item[data-v-07f51b8b]{padding:12px;position:relative}@media not (min-width:1200px){.navigation__menu-item[data-v-07f51b8b]{padding:12px 8px}}.navigation__menu-item[data-v-07f51b8b]:hover{cursor:pointer}.navigation__menu-item:hover .navigation__menu-link[data-v-07f51b8b],.navigation__menu-item:hover .navigation__menu-trigger[data-v-07f51b8b]{color:#171321}.navigation__menu-item:hover .navigation__chevron[data-v-07f51b8b]{transform:rotate(180deg)}.navigation__menu-item[data-v-07f51b8b]:focus-visible{background:#f2f1f5;outline:none}@media not (min-width:1200px){.navigation__menu-item--localized[data-v-07f51b8b]{padding:12px 4px}}@media(min-width:1200px)and (not (min-width:1440px)){.navigation__menu-item--localized[data-v-07f51b8b]{padding:12px 8px}}.navigation__menu-link[data-v-07f51b8b],.navigation__menu-trigger[data-v-07f51b8b]{color:#74717a;padding:0!important;transition:color .2s ease}.navigation__menu-link[data-v-07f51b8b]:focus-visible,.navigation__menu-link[data-v-07f51b8b]:hover,.navigation__menu-trigger[data-v-07f51b8b]:focus-visible,.navigation__menu-trigger[data-v-07f51b8b]:hover{color:#171321}.navigation__menu-trigger[data-v-07f51b8b]{align-items:center;background:transparent;border:0;cursor:pointer;display:inline-flex;font:inherit;gap:8px}.navigation__menu-trigger[aria-expanded=true][data-v-07f51b8b]{color:#171321}.navigation__menu-link[data-v-07f51b8b]{min-width:unset!important}.navigation__chevron[data-v-07f51b8b]{color:currentColor;transform:rotate(0);transform-origin:center;transition:transform .3s ease}.navigation__chevron--rotated[data-v-07f51b8b]{transform:rotate(180deg)}.navigation__dropdown[data-v-07f51b8b]{left:0;max-width:calc(100vw - 32px);min-width:260px;opacity:0;padding-top:12px;pointer-events:none;position:absolute;top:100%;visibility:hidden;will-change:transform;z-index:1026}.navigation__dropdown--open[data-v-07f51b8b]{opacity:1;pointer-events:auto;visibility:visible}.navigation__dropdown-content[data-v-07f51b8b]{align-items:stretch;background:#fff;border:1px solid #e8e7eb;border-radius:16px;box-shadow:0 81px 23px 0 transparent,0 52px 21px #00000003,0 29px 17px #00000008,0 13px 13px #0000000a,0 3px 7px #0000000d;display:flex;flex:1;height:100%;overflow:hidden}.navigation__actions[data-v-07f51b8b]{align-items:center;display:flex;gap:24px}@media not (min-width:1300px){.navigation__actions[data-v-07f51b8b]{gap:8px}.navigation__actions .slp-btn[data-v-07f51b8b]{line-height:1.15;padding:10px 8px;white-space:normal}.navigation__actions--localized[data-v-07f51b8b]{gap:4px}.navigation__actions--localized .slp-btn[data-v-07f51b8b]{line-height:1.15;padding:10px 8px;white-space:normal}}@media(min-width:1200px)and (not (min-width:1550px)){.navigation__actions--localized[data-v-07f51b8b]{gap:8px}.navigation__actions--localized .slp-btn[data-v-07f51b8b]{line-height:1.15;padding:10px 8px;white-space:normal}}.navigation__search[data-v-07f51b8b]{color:#74717a;padding-right:0!important;transition:color .2s ease}.navigation__search[data-v-07f51b8b]:focus-visible,.navigation__search[data-v-07f51b8b]:hover{color:#171321}.navigation__login[data-v-07f51b8b]{color:#74717a;transition:color .2s ease}.navigation__login[data-v-07f51b8b]:focus-visible,.navigation__login[data-v-07f51b8b]:hover{color:#171321}.navigation__overlay[data-v-07f51b8b]{background:#0000001a;inset:0;opacity:0;padding:0;pointer-events:none;position:fixed;transition:opacity .2s ease;will-change:opacity;z-index:1024}.navigation__overlay--visible[data-v-07f51b8b]{opacity:1;pointer-events:none}.navigation__mobile[data-v-07f51b8b]{align-items:center;display:none;grid-template-columns:1fr auto 1fr;height:68px;padding:0 16px;position:relative}.navigation__mobile-logo[data-v-07f51b8b]{left:50%;position:absolute;transform:translate(-50%)}.navigation__mobile-toggle[data-v-07f51b8b]{justify-self:end}.navigation__mobile-trial-button[data-v-07f51b8b]{font-size:.875rem!important;justify-self:start;margin:0!important;padding:8px 12px!important}@media(not (max-width:375px))and (not (min-width:500px)){.navigation__mobile-trial-button[data-v-07f51b8b]{max-width:160px}}@media(max-width:375px){.navigation__mobile-trial-button[data-v-07f51b8b]{max-width:150px}}.navigation__mobile-menu-toggle[data-v-07f51b8b]{padding:0}@media(max-width:1025px){.navigation__desktop[data-v-07f51b8b]{display:none}.navigation__mobile[data-v-07f51b8b]{display:flex;justify-content:space-between}}@media(prefers-reduced-motion:reduce){.navigation__chevron[data-v-07f51b8b]{transition:none}}</style><style>.be-nav-search[data-v-25bdfcad]{background-color:#fff;bottom:0;left:0;overflow-y:scroll;position:fixed;right:0;top:var(--search-top-offset,68px);z-index:1060}@media(max-width:1025px){.be-nav-search[data-v-25bdfcad]{overflow-x:hidden;top:0}}.be-nav-search__container[data-v-25bdfcad]{height:100%}@media(max-width:1025px){.be-nav-search__container[data-v-25bdfcad]{padding:0!important}}.be-nav-search__row[data-v-25bdfcad]{flex-wrap:nowrap!important;height:100%;justify-content:center}.be-nav-search__row[data-v-25bdfcad]:focus-visible{outline:none}.be-nav-search__close[data-v-25bdfcad]{display:flex;justify-content:flex-end;margin-top:16px}.be-nav-search__close[data-v-25bdfcad],.be-nav-search__close .slp-icon[data-v-25bdfcad],.be-nav-search__close--text[data-v-25bdfcad]{color:#171321!important}</style><style>.be-nav-search-content{padding:24px}@media(max-width:1025px){.be-nav-search-content{padding:8px}}.be-nav-search-content__input{position:relative;width:100%}.be-nav-search-content__input input{-webkit-appearance:none;border:0;border-bottom:1px solid #d1d0d3;border-radius:0;color:#74717a;font-size:2.5rem;width:calc(100% - 8px)}.be-nav-search-content__input input::-moz-placeholder{color:#d1d0d3}.be-nav-search-content__input input::placeholder{color:#d1d0d3}.be-nav-search-content__input input:focus{outline:none}@media(max-width:1025px){.be-nav-search-content__input input{caret-color:#171321!important;font-size:1.375rem}}.be-nav-search-content__input--close{cursor:pointer;position:absolute;right:16px;top:20px}@media(max-width:1025px){.be-nav-search-content__input--close{top:4px}}.be-nav-search-content__link{color:#171321!important;font-weight:700!important}</style><style>.be-nav-suggestions__default[data-v-0823afcc]{display:flex;flex-wrap:wrap;justify-content:space-between;margin-top:4px}@media(min-width:1025px){.be-nav-suggestions__default[data-v-0823afcc]{-moz-column-gap:32px;column-gap:32px;display:grid;grid-template-columns:repeat(2,1fr)}}@media(min-width:1200px){.be-nav-suggestions__default[data-v-0823afcc]{grid-template-columns:repeat(3,1fr)}}@media(max-width:768px){.be-nav-suggestions__default[data-v-0823afcc]{width:100%}}.be-nav-suggestions__default-btn[data-v-0823afcc]{background-color:#f6f3fe!important;border:none!important;color:#171321!important;justify-content:center;margin-bottom:16px;transition:background-color .3s ease-in-out;width:100%}@media(max-width:1025px){.be-nav-suggestions__default-btn[data-v-0823afcc]{max-width:100%}}.be-nav-suggestions__default-btn[data-v-0823afcc]:hover{background-color:#e3d6ff!important}.be-nav-suggestions__result[data-v-0823afcc]{align-items:center;display:flex}.be-nav-suggestions__result--icon[data-v-0823afcc]{min-width:16px}.be-nav-suggestions__result--icon[data-v-0823afcc] svg{min-height:16px;min-width:16px}.be-nav-suggestions__result .slp-btn[data-v-0823afcc]{color:#7759c2;font-weight:400}.be-nav-suggestions__result .slp-btn[data-v-0823afcc]:hover{text-decoration-color:#7759c2}@media(max-width:768px){.be-nav-suggestions__result .slp-btn[data-v-0823afcc]{padding-left:8px!important}}.be-nav-suggestions__result .slp-btn span[data-v-0823afcc]{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}</style><style>.navigation-dropdown-lists[data-v-70f8ea54]{min-width:190px}@media not (max-width:1025px){.navigation-dropdown-lists[data-v-70f8ea54]{padding:16px}}.navigation-dropdown-lists__columns[data-v-70f8ea54]{display:flex;flex-direction:row;gap:8px;width:100%}@media not (min-width:1025px){.navigation-dropdown-lists__columns[data-v-70f8ea54]{flex-direction:column;gap:16px}}.navigation-dropdown-lists__column-title[data-v-70f8ea54]{color:#74717a;margin:8px}@media not (max-width:768px){.navigation-dropdown-lists__column-title[data-v-70f8ea54]{margin:8px 16px}}.navigation-dropdown-lists__list[data-v-70f8ea54]{flex:1 0 auto;list-style:none;margin:0;min-width:220px;padding:0;width:100%}@media not (max-width:1025px){.navigation-dropdown-lists__list[data-v-70f8ea54]{width:-moz-max-content;width:max-content}}.navigation-dropdown-lists__link[data-v-70f8ea54]{background-color:transparent;border-radius:8px;color:#171321;display:flex;justify-content:flex-start;padding:8px;text-align:left;transition:background-color .2s ease;white-space:nowrap;width:auto}@media not (max-width:768px){.navigation-dropdown-lists__link[data-v-70f8ea54]{padding:8px 16px}}.navigation-dropdown-lists__link[data-v-70f8ea54]:focus-visible,.navigation-dropdown-lists__link[data-v-70f8ea54]:hover{background-color:#f2f1f5;color:#171321}</style><style>.navigation__dropdown-feature[data-v-f3a57a08]{display:flex;flex:1;flex-direction:column;padding-bottom:16px}@media not (max-width:1025px){.navigation__dropdown-feature[data-v-f3a57a08]{border-left:1px solid #e8e7eb;padding-bottom:0}}.navigation__feature[data-v-f3a57a08]{align-items:center;background:var(--background,#fff);background-position:bottom;background-repeat:no-repeat;background-size:cover;border-radius:16px;color:var(--textColor,#171321);display:flex;flex:1;flex-direction:column;justify-content:space-between;min-width:290px;overflow:hidden;padding:24px;position:relative;white-space:normal}@media not (max-width:1025px){.navigation__feature[data-v-f3a57a08]{border:unset;border-radius:unset}}.navigation__feature[data-v-f3a57a08]>*{position:relative}.navigation__feature-content[data-v-f3a57a08]{flex:1}.navigation__feature-article[data-v-f3a57a08],.navigation__feature-content[data-v-f3a57a08],.navigation__feature-link[data-v-f3a57a08]{color:var(--textColor,#171321);display:flex;flex-direction:column;height:100%;justify-content:space-between;white-space:normal}.navigation__feature-link[data-v-f3a57a08]{flex:1}.navigation__feature-link[data-v-f3a57a08] svg{transition:transform .2s ease-in-out}.navigation__feature-link:focus-visible .navigation__feature-cta[data-v-f3a57a08],.navigation__feature-link:hover .navigation__feature-cta[data-v-f3a57a08]{text-decoration:none}.navigation__feature-link:focus-visible .navigation__feature-cta[data-v-f3a57a08] svg,.navigation__feature-link:hover .navigation__feature-cta[data-v-f3a57a08] svg{transform:translate(8px)}.navigation__feature-title[data-v-f3a57a08],.navigation__feature-title span[data-v-f3a57a08]{color:var(--textColor,#171321)}.navigation__feature-image[data-v-f3a57a08]{height:100%;margin-bottom:16px;max-width:375px;width:100%}@media(min-width:1025px){.navigation__feature-image[data-v-f3a57a08]{height:160px;max-width:200px;width:200px}}@media(min-width:1200px){.navigation__feature-image[data-v-f3a57a08]{max-width:239px;width:239px}}.navigation__feature-image img[data-v-f3a57a08]{height:100%;width:100%}.navigation__feature-description[data-v-f3a57a08]{margin:0 0 48px}@media(min-width:1025px){.navigation__feature-description[data-v-f3a57a08]{margin:0 0 8px}}.navigation__feature-cta[data-v-f3a57a08]{color:var(--textColor,#171321);text-align:left;text-decoration:none}.navigation__feature-cta[data-v-f3a57a08] svg *{fill:currentColor}</style><style>.banner[data-v-9f195807]{display:block}.banner[data-v-9f195807]:hover .icon{transform:translate(8px)}.banner__desktop[data-v-9f195807]{align-items:center;background:#171321;box-sizing:border-box;display:flex;justify-content:center;overflow:hidden;width:100%}@media(max-width:1025px){.banner__desktop[data-v-9f195807]{display:none}}.banner__desktop .content[data-v-9f195807]{align-items:center;display:flex;justify-content:center;z-index:1}.banner__desktop .icon[data-v-9f195807]{color:#000;display:flex;transition:color .3s ease,transform .3s ease}.banner__desktop .icon[data-v-9f195807] svg *{fill:currentColor}.banner__desktop .image[data-v-9f195807]{margin-bottom:3px;margin-right:6px;max-width:200px}@media(max-width:1025px){.banner__desktop .image[data-v-9f195807]{margin-bottom:0;margin-left:0;margin-top:16px}}.banner__desktop .text[data-v-9f195807]{color:#171321;padding:16px 0;text-decoration:underline}.banner__cta[data-v-9f195807]{align-items:center;display:inline-flex;gap:4px;vertical-align:middle}.banner__mobile[data-v-9f195807]{background:#171321;color:#171321;display:block;font-size:14px;margin:9px;padding:8px 12px;position:relative;text-decoration:none}.banner__mobile .pill-content[data-v-9f195807]{align-items:center;display:flex;justify-content:center;position:relative;z-index:1}.banner__mobile .pill-content .slp-icon[data-v-9f195807]{margin-left:16px;transition:transform .2s ease-in-out}@media(min-width:1025px){.banner__mobile[data-v-9f195807]{display:none}}.banner--release[data-v-9f195807]{overflow:hidden;position:relative}.banner--release .banner__desktop[data-v-9f195807]{background-color:#35333b;background-image:url(https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto/v1787067438/sdybvdl8fcg12lltqlqh.jpg);background-position:50%;background-repeat:no-repeat;background-size:100% 100%;display:flex;justify-content:center;position:relative}@media(max-width:768px){.banner--release .banner__desktop[data-v-9f195807]{background-image:url(https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto/v1787071042/ejayd8teid41oggjvxyi.jpg);background-position:50%;background-size:100% 100%;flex-direction:column}}.banner--release .content[data-v-9f195807]{gap:16px;line-height:20px;padding:16px 0;position:relative;text-align:center;z-index:1}@media(max-width:768px){.banner--release .content[data-v-9f195807]{display:block;padding:16px}}.banner--release .text[data-v-9f195807]{color:#fff;line-height:20px;padding:0;text-decoration:none}@media(max-width:768px){.banner--release .banner__cta[data-v-9f195807]{margin-left:8px}}.banner--release .banner__cta-text[data-v-9f195807]{color:#fff;line-height:20px;text-decoration:underline}@media(max-width:768px){.banner--release .banner__cta-text[data-v-9f195807]{display:none}}.banner--release .icon[data-v-9f195807]{color:#fff}</style><style>@font-face{font-family:GitLab Sans;src:local("GitLab Sans Variable"),url(/fonts/gitlab-sans/GitLabSans.woff2) format(woff2);font-display:swap;font-weight:100 900;font-style:normal}.be-nav-footer[data-v-d02fbda3]{background-color:#171321;color:#fff;overflow:hidden;padding:40px 16px;z-index:1}.be-nav-footer[data-v-d02fbda3] *,.be-nav-footer[data-v-d02fbda3] :after,.be-nav-footer[data-v-d02fbda3] :before{box-sizing:border-box}.be-nav-footer .disclaimer[data-v-d02fbda3]{background:#f2f1f5;padding:32px 0}@media(max-width:768px){.be-nav-footer .container[data-v-d02fbda3]{padding:0!important}}.be-nav-footer .footer__navigation[data-v-d02fbda3]{align-items:flex-start;display:flex;flex-direction:column;flex-wrap:wrap;justify-content:space-between}@media(min-width:1220px){.be-nav-footer .footer__navigation[data-v-d02fbda3]{flex-direction:row;flex-wrap:nowrap}}.be-nav-footer .logo[data-v-d02fbda3]{display:flex;margin:0 auto 0 0}.be-nav-footer .logo span[data-v-d02fbda3]{font-size:1.125rem;margin-left:4px}.be-nav-footer .content[data-v-d02fbda3]{display:flex;flex-wrap:wrap;gap:30px;justify-content:space-between;margin-top:48px}@media(min-width:1220px){.be-nav-footer .content[data-v-d02fbda3]{margin-top:0}}@media(min-width:769px){.be-nav-footer .column-wrapper[data-v-d02fbda3]{margin-top:24px}}.be-nav-footer .column[data-v-d02fbda3]{flex:1 0 auto;max-width:150px;min-width:2rem;width:100%}.be-nav-footer .column__heading[data-v-d02fbda3]{margin-bottom:16px}@media(max-width:768px){.be-nav-footer .column__nested[data-v-d02fbda3]{align-items:flex-start;display:flex;gap:30px;justify-content:space-between;max-width:100%}.be-nav-footer .column__nested[data-v-d02fbda3]>*{width:145px}}@media(min-width:768px){.be-nav-footer .column[data-v-d02fbda3]{max-width:160px}}@media(max-width:768px){.be-nav-footer .divided[data-v-d02fbda3]{width:100%}}.be-nav-footer .links__item[data-v-d02fbda3]{margin-bottom:8px}.be-nav-footer .links__item[data-v-d02fbda3]:hover{text-decoration:underline}@media(max-width:768px){.be-nav-footer .links__divided[data-v-d02fbda3]{-moz-column-gap:calc(100% - 290px);column-gap:calc(100% - 290px);-moz-columns:120px 2;columns:120px 2}}.be-nav-footer .link[data-v-d02fbda3]{color:#fff;font-size:.875rem;line-height:1.25}.be-nav-footer .footer__cta[data-v-d02fbda3]{display:flex;flex-direction:column;padding-top:64px}@media(min-width:768px){.be-nav-footer .footer__cta[data-v-d02fbda3]{padding-top:96px}}.be-nav-footer .footer__selectors[data-v-d02fbda3]{align-items:baseline;border-bottom:1px solid #fff;display:flex;flex-direction:column;margin-bottom:32px;padding-bottom:24px}@media(min-width:768px){.be-nav-footer .footer__selectors[data-v-d02fbda3]{flex-direction:row}}.be-nav-footer .footer__selectors--two-children[data-v-d02fbda3]{justify-content:space-between}.be-nav-footer__source-buttons[data-v-d02fbda3]{display:flex;flex-wrap:wrap;gap:8px 24px}.be-nav-footer__button[data-v-d02fbda3]{border:none;color:#fff;cursor:pointer;font-size:.875rem;font-weight:660;padding:0;text-decoration:underline}.be-nav-footer__button--black[data-v-d02fbda3]{color:#171321}.be-nav-footer .source p[data-v-d02fbda3]{font-size:.875rem;margin-bottom:8px}.be-nav-footer .source a[data-v-d02fbda3]{color:#fff;font-weight:660;text-decoration:underline}.be-nav-footer .source img[data-v-d02fbda3]{display:inline-block;margin-left:5px;vertical-align:middle}.be-nav-footer #ot-sdk-btn.optanon-show-settings[data-v-d02fbda3],.be-nav-footer #ot-sdk-btn.ot-sdk-show-settings[data-v-d02fbda3]{border:none!important;color:#fff!important;cursor:pointer;font-family:GitLab Sans,sans-serif!important;font-size:14px!important;padding:0!important;text-align:left!important}.be-nav-footer #ot-sdk-btn.optanon-show-settings[data-v-d02fbda3]:hover,.be-nav-footer #ot-sdk-btn.ot-sdk-show-settings[data-v-d02fbda3]:hover{background:inherit;text-decoration:underline}</style><style>.social__links[data-v-0b1e0911]{align-items:center;display:flex;gap:15px;margin-top:32px;width:100%}@media(min-width:829px){.social__links[data-v-0b1e0911]{margin-top:0;width:inherit}}.social__links--item a[data-v-0b1e0911]{color:#fff;transition:.3s}.social__links--item a[data-v-0b1e0911]:hover{color:#a989f5}.social__links--icon[data-v-0b1e0911]{height:auto;width:24px}.social__links--icon[data-v-0b1e0911] *{fill:currentColor}.social__links--icon-x[data-v-0b1e0911]{width:18px}</style><style>.footer-source[data-v-2b76ce90]{display:flex;flex-direction:column}.footer-source__source-buttons[data-v-2b76ce90]{display:flex;flex-wrap:wrap;gap:8px 24px}.footer-source p[data-v-2b76ce90]{font-size:.875rem;margin-bottom:8px}.footer-source a[data-v-2b76ce90]{color:#fff;font-weight:660;text-decoration:underline}.footer-source img[data-v-2b76ce90]{display:inline-block;margin-left:5px;vertical-align:middle}</style><style>.wrapper[data-v-43df3cc1]{display:grid;gap:64px;grid-template-columns:70% 30%}@media(max-width:1025px){.wrapper[data-v-43df3cc1]{grid-template-columns:1fr}}.wrapper .hero[data-v-43df3cc1]{display:flex;flex-direction:column;gap:16px}.wrapper .hero .date[data-v-43df3cc1]{align-items:center;display:flex;gap:8px}.wrapper .hero img[data-v-43df3cc1]{aspect-ratio:16/9;margin-top:24px;-o-object-fit:cover;object-fit:cover;width:100%}.wrapper .information[data-v-43df3cc1]{display:flex;flex-direction:column;gap:24px;padding-top:64px}@media(min-width:1025px){.wrapper .information[data-v-43df3cc1]{padding-top:120px}}.wrapper .information .author[data-v-43df3cc1]{align-items:center;display:flex;gap:8px}.wrapper .information .author img[data-v-43df3cc1]{border-radius:80px;height:80px;-o-object-fit:contain;object-fit:contain;width:80px}.wrapper .information .tags[data-v-43df3cc1]{display:flex;flex-wrap:wrap;gap:8px}.wrapper[data-v-43df3cc1] a{color:#7759c2}.container[data-v-43df3cc1]{position:relative}.container[data-v-43df3cc1]:after{background:linear-gradient(126deg,#eee7fd,#f6f3fed4 49.71%,#ffd4d873);content:"";height:60%;left:-40%;position:absolute;top:0;width:140vw;z-index:-1}@media(min-width:2100px){.container[data-v-43df3cc1]:after{left:-60%}}@media(min-width:1025px){.container[data-v-43df3cc1]:after{height:75%}}.pill[data-v-43df3cc1]{background:#fff;border:1px solid #d1d0d3;border-radius:20px;display:inline-block;padding:4px 16px}</style><style>.mobile-chevron[data-v-eccd9ac8]{display:none}@media(max-width:768px){.mobile-chevron[data-v-eccd9ac8]{display:block}}.breadcrumb[data-v-eccd9ac8]{display:flex;margin-bottom:32px;margin-top:32px}.breadcrumb__crumb[data-v-eccd9ac8]{color:#7759c2;display:flex}.breadcrumb__crumb[data-v-eccd9ac8]:active:not(.breadcrumb__crumb--disabled),.breadcrumb__crumb[data-v-eccd9ac8]:focus:not(.breadcrumb__crumb--disabled),.breadcrumb__crumb[data-v-eccd9ac8]:hover:not(.breadcrumb__crumb--disabled){text-decoration:underline}.breadcrumb__crumb--disabled[data-v-eccd9ac8]{color:#74717a}.breadcrumb__crumb a[data-v-eccd9ac8]{color:inherit}.breadcrumb__chevron[data-v-eccd9ac8]{margin:1px 10px 0 6px}.breadcrumb--hero[data-v-eccd9ac8]{margin-bottom:24px;margin-left:16px}@media(min-width:1025px){.breadcrumb--hero[data-v-eccd9ac8]{margin-left:86px;max-width:none}}.breadcrumb--dark .breadcrumb__crumb[data-v-eccd9ac8]{color:#fff}.breadcrumb--dark .breadcrumb__crumb[data-v-eccd9ac8]:active:not(.breadcrumb__crumb--disabled),.breadcrumb--dark .breadcrumb__crumb[data-v-eccd9ac8]:focus:not(.breadcrumb__crumb--disabled),.breadcrumb--dark .breadcrumb__crumb[data-v-eccd9ac8]:hover:not(.breadcrumb__crumb--disabled){color:#a989f5}.breadcrumb--dark .breadcrumb__crumb--disabled[data-v-eccd9ac8]{color:#a2a1a6}@media(max-width:768px){.hide-in-mobile[data-v-eccd9ac8]{display:none}}</style><style>.wrapper[data-v-612f8e63]{display:grid;gap:64px;grid-template-columns:70% 30%;padding-top:40px}@media(max-width:1025px){.wrapper[data-v-612f8e63]{grid-template-columns:1fr}}.social-icon-links[data-v-612f8e63]{display:flex;gap:28px;padding:14px 0}.newsletter-form[data-v-612f8e63]{padding:24px 0}.newsletter-form[data-v-612f8e63] .form_header{color:#171321;font-size:20px;font-weight:600;line-height:24px;text-align:left}@media(max-width:1025px){.newsletter-form[data-v-612f8e63] .form_header{text-align:center}}.newsletter-form[data-v-612f8e63] .form_container{border:1px solid rgba(0,0,0,.08);padding:24px}.newsletter-form[data-v-612f8e63] .required_label{display:none}.newsletter-form[data-v-612f8e63] #LblEmail,.newsletter-form[data-v-612f8e63] #LblCountry,.newsletter-form[data-v-612f8e63] #LblTitle{color:#000;font-size:16px;font-weight:400;line-height:20px;margin-bottom:8px;width:100%}@media(max-width:1025px){.newsletter-form[data-v-612f8e63] #Country{width:100%!important}}.newsletter-form[data-v-612f8e63] .mktoGutter.mktoHasWidth,.newsletter-form[data-v-612f8e63] .mktoOffset.mktoHasWidth{display:none}.newsletter-form[data-v-612f8e63] .mktoHtmlText.mktoHasWidth{color:#74717a;font-size:14px;font-weight:400;line-height:20px;margin-top:16px}.newsletter-form[data-v-612f8e63] .mktoHtmlText.mktoHasWidth a{color:#74717a;text-decoration:underline}.newsletter-form[data-v-612f8e63] .mktoField.mktoEmailField{height:44px}.newsletter-form[data-v-612f8e63] .mktoFieldWrap.mktoRequiredField{display:flex;flex-direction:column}.newsletter-form[data-v-612f8e63] .mktoErrorMsg{color:red!important}.newsletter-form[data-v-612f8e63] .mktoInvalid{border:1px solid red!important}.newsletter-form[data-v-612f8e63] .mktoButton{width:174px!important}.blog-promo-card[data-v-612f8e63]{align-items:center;align-self:stretch;background:#fff;border-radius:24px;box-shadow:0 2px 15px #00000026;display:none;flex-direction:column;gap:16px;justify-content:center;padding:16px;text-align:center}.blog-promo-card .image[data-v-612f8e63]{border-radius:8px;-o-object-fit:cover;object-fit:cover;width:100%}.blog-promo-card .footer-text[data-v-612f8e63]{color:#74717a;font-style:italic;text-align:center}@media(min-width:1025px){.blog-promo-card[data-v-612f8e63]{display:flex}}.sticky-container[data-v-612f8e63]{position:sticky;top:calc(var(--nav-height, 84px) + 16px)}</style><style>.wrapper[data-v-17b31dd6]{margin:64px auto}@media(min-width:1025px){.wrapper[data-v-17b31dd6]{margin:96px auto}}.header[data-v-17b31dd6]{display:flex;justify-content:space-between;margin-bottom:24px}@media(max-width:1025px){.header[data-v-17b31dd6]{flex-direction:column}}.header-cta[data-v-17b31dd6]{white-space:nowrap}.cards[data-v-17b31dd6]{display:grid;gap:30px;grid-template-columns:1fr}@media(min-width:1025px){.cards[data-v-17b31dd6]{grid-template-columns:repeat(3,1fr)}}.cards .card[data-v-17b31dd6]{align-items:flex-start;color:#171321;cursor:pointer;display:flex;flex-direction:column;gap:16px;padding:16px}.cards .card .title[data-v-17b31dd6]{flex-basis:100%}.cards .card .category[data-v-17b31dd6]{align-items:center;display:flex;gap:8px}.cards .card .category-title[data-v-17b31dd6]{text-transform:capitalize}.cards .card img[data-v-17b31dd6]{aspect-ratio:16/9;-o-object-fit:cover;object-fit:cover;width:318px}@media(max-width:1025px){.cards .card img[data-v-17b31dd6]{max-width:100%}}.cards .card[data-v-17b31dd6]:hover{background-color:#f6f3fe;border-radius:8px 8px 50px;transition:background .3s ease,border-radius .3s ease!important}</style><style>.contact-cta-container[data-v-baf99eb1]{align-items:flex-start;background:#f6f3fe;border-radius:4px 4px 64px;display:flex;flex-direction:column;gap:16px;margin:64px 0;padding:32px}@media(min-width:1025px){.contact-cta-container[data-v-baf99eb1]{margin:96px 0}}</style><style>.next-steps[data-v-9396c4f8]{background:url("data:image/svg+xml;charset=utf-8,%3Csvg xmlns='http://www.w3.org/2000/svg' width='1440' height='584' fill='none' viewBox='0 0 1440 584'%3E%3Cg clip-path='url(%23a)'%3E%3Cpath fill='%23fff' d='M0 0h1440v584H0z'/%3E%3Crect width='1632' height='1026' x='-96' y='-110' fill='%23edecef' rx='59.087'/%3E%3Cg filter='url(%23b)'%3E%3Cpath fill='url(%23c)' d='M705.093 548.282C500.026 393.66 83.088 460.741-102.985 517.758l30.532 304.473L1528.02 661.738 1468.6 69.21c-440.58 91.416-354.98 333.584-368.49 439.46-7.27 57.01-58.47 123.45-124.34 138.096-65.867 14.645-148.807-6.593-270.677-98.484'/%3E%3C/g%3E%3Cg filter='url(%23d)'%3E%3Cpath fill='url(%23e)' d='M779.021 670.794c-231.249-174.207-701.458-98.581-911.307-34.316l34.403 343.075L1707.06 798.556l-66.95-667.651c-496.87 103.049-400.36 375.915-415.6 495.216-8.21 64.24-65.96 139.109-140.24 155.618-74.29 16.508-167.819-7.415-305.249-110.945'/%3E%3C/g%3E%3C/g%3E%3Cdefs%3E%3Cfilter id='b' width='2021' height='1143.02' x='-297.984' y='-125.79' color-interpolation-filters='sRGB' filterUnits='userSpaceOnUse'%3E%3CfeFlood flood-opacity='0' result='BackgroundImageFix'/%3E%3CfeBlend in='SourceGraphic' in2='BackgroundImageFix' result='shape'/%3E%3CfeGaussianBlur result='effect1_foregroundBlur_2788_22869' stdDeviation='97.5'/%3E%3C/filter%3E%3Cfilter id='d' width='2039.35' height='1048.65' x='-232.285' y='30.904' color-interpolation-filters='sRGB' filterUnits='userSpaceOnUse'%3E%3CfeFlood flood-opacity='0' result='BackgroundImageFix'/%3E%3CfeBlend in='SourceGraphic' in2='BackgroundImageFix' result='shape'/%3E%3CfeGaussianBlur result='effect1_foregroundBlur_2788_22869' stdDeviation='25'/%3E%3C/filter%3E%3CradialGradient id='c' cx='0' cy='0' r='1' gradientTransform='rotate(-95.726 550.667 -92.927)scale(297.75 804.25)' gradientUnits='userSpaceOnUse'%3E%3Cstop stop-color='%23ff1493'/%3E%3Cstop offset='1' stop-color='%23fca326'/%3E%3C/radialGradient%3E%3ClinearGradient id='e' x1='1026.96' x2='854.877' y1='806.933' y2='392.373' gradientUnits='userSpaceOnUse'%3E%3Cstop stop-color='%23e24329'/%3E%3Cstop offset='1' stop-color='%23fc6d26'/%3E%3C/linearGradient%3E%3CclipPath id='a'%3E%3Cpath fill='%23fff' d='M0 0h1440v584H0z'/%3E%3C/clipPath%3E%3C/defs%3E%3C/svg%3E") no-repeat bottom/cover;position:relative;z-index:1}.next-steps__container[data-v-9396c4f8]{align-items:center;display:flex;flex-direction:column;gap:24px;padding:96px 0}@media(min-width:768px){.next-steps__container[data-v-9396c4f8]{max-width:100%;padding:165px 24px 193px}}@media(max-width:1025px){.next-steps__container[data-v-9396c4f8]{text-align:center}}.next-steps__container .description[data-v-9396c4f8]{color:#171321;margin-bottom:16px;text-align:center}.next-steps__container .buttons[data-v-9396c4f8]{display:flex;flex-direction:row;gap:24px}@media(max-width:1025px){.next-steps__container .buttons[data-v-9396c4f8]{flex-direction:column;width:100%}}.next-steps__container .buttons .main-button[data-v-9396c4f8],.next-steps__container .buttons .secondary-button[data-v-9396c4f8]{justify-content:center}.next-steps__header[data-v-9396c4f8]{color:#171321;text-align:center}.next-steps--compact[data-v-9396c4f8]{text-align:center}@media(min-width:768px){.next-steps--compact .next-steps__container[data-v-9396c4f8]{margin:0 auto;max-width:1040px;padding:120px 0 160px}}.next-steps--compact .next-steps__header[data-v-9396c4f8],.next-steps--compact .next-steps__header[data-v-9396c4f8] *{font-size:clamp(56px,5.6vw,96px);letter-spacing:-.03em;line-height:1.04}.next-steps--compact .description[data-v-9396c4f8]{color:#74717a;margin-bottom:0}@media(min-width:768px){.next-steps--compact .description[data-v-9396c4f8]{max-width:760px}}.next-steps--compact .description[data-v-9396c4f8],.next-steps--compact .description[data-v-9396c4f8] *{font-size:20px;font-weight:500;letter-spacing:normal;line-height:28px}.next-steps--compact .buttons[data-v-9396c4f8]{flex-wrap:wrap;gap:12px;justify-content:center;margin-top:16px}</style><style>.body[data-v-9c71bc40] h1{font-size:2.5rem;font-weight:660;letter-spacing:-.02em;line-height:1.05;margin-bottom:24px;margin-top:24px}@media(max-width:576px){.body[data-v-9c71bc40] h1{font-size:2rem;letter-spacing:-.02em;line-height:1.0625}}.body[data-v-9c71bc40] ol{list-style:decimal;margin-bottom:16px;padding-left:24px}.body[data-v-9c71bc40] ul{list-style:disc;margin-bottom:16px;padding-left:24px}.body[data-v-9c71bc40] ol+ul{padding-left:40px}.body[data-v-9c71bc40] table{border-collapse:collapse;width:100%}@media(max-width:576px){.body[data-v-9c71bc40] table{font-size:14px}}.body[data-v-9c71bc40] td,.body[data-v-9c71bc40] th{border:1px solid #d1d0d3;padding:8px;text-align:left}@media(max-width:576px){.body[data-v-9c71bc40] td,.body[data-v-9c71bc40] th{white-space:normal;word-wrap:break-word;overflow-wrap:break-word;word-break:break-all}}.body[data-v-9c71bc40] h2{font-size:2.5rem;font-weight:660;letter-spacing:-.03em;line-height:1.05;margin-bottom:24px;margin-top:24px}.body[data-v-9c71bc40] h3{font-size:2rem;font-weight:660;letter-spacing:-.02em;line-height:1.0625;margin-bottom:24px;margin-top:24px}.body[data-v-9c71bc40] h4{font-size:1.75rem;font-weight:660;letter-spacing:-.02em;line-height:1.0714285714;margin-bottom:24px;margin-top:24px}.body[data-v-9c71bc40] h5{font-size:1.375rem;font-weight:660;letter-spacing:-.02em;line-height:1.0909090909;margin-bottom:24px;margin-top:24px}.body[data-v-9c71bc40] li,.body[data-v-9c71bc40] p{font-size:1.125rem;font-weight:400;line-height:1.5555555556;margin-bottom:16px}.body[data-v-9c71bc40] a{color:#7759c2}.body[data-v-9c71bc40] li{margin-bottom:8px}.body[data-v-9c71bc40] li a,.body[data-v-9c71bc40] li p{margin-bottom:0}.body[data-v-9c71bc40] hr{margin-bottom:8px;margin-top:8px}.body[data-v-9c71bc40] .video_container iframe{height:460px;margin-bottom:24px;width:100%}@media(max-width:1025px){.body[data-v-9c71bc40] .video_container iframe{height:240px}}.body[data-v-9c71bc40] iframe[src*="player.vimeo.com"],.body[data-v-9c71bc40] iframe[src*="youtube-nocookie.com"],.body[data-v-9c71bc40] iframe[src*="youtube.com"]{display:none!important}.body[data-v-9c71bc40] .video-container>iframe,.body[data-v-9c71bc40] .video-wrapper>iframe{display:block!important}.body[data-v-9c71bc40] img{display:block;margin-inline:auto;max-width:100%;-o-object-fit:cover;object-fit:cover}.body[data-v-9c71bc40] figcaption{margin:8px 0 32px;text-align:center}.body[data-v-9c71bc40] pre{background-color:#f5f5f5;border:1px solid #ccc;display:flex;font-size:16px;line-height:1.5;margin:0 0 13.5px;padding:13px;white-space:pre-wrap;word-break:break-all}.body[data-v-9c71bc40] pre code,.body[data-v-9c71bc40] pre span{background-color:transparent;padding:0}.body[data-v-9c71bc40] blockquote{align-items:flex-start;align-self:stretch;background:#f6f3fe;border-radius:4px;display:flex;flex-direction:column;gap:16px;justify-content:center;margin:40px 0;padding:24px 40px}.body[data-v-9c71bc40] blockquote p{margin-bottom:0}.body[data-v-9c71bc40] h1,.body[data-v-9c71bc40] h2,.body[data-v-9c71bc40] h3,.body[data-v-9c71bc40] h4,.body[data-v-9c71bc40] h5,.body[data-v-9c71bc40] h6{position:relative}</style><style>.form_column[data-v-72af5bed]{grid-area:form}@media(max-width:1025px){.form_column[data-v-72af5bed]{margin-bottom:3rem}}.form_column iframe[data-v-72af5bed]{border:1px solid #f2f2f2;border-radius:4px;display:block;margin-left:auto;margin-right:auto;max-width:100%}.form_column iframe .sid-ltr[data-v-72af5bed]{background-color:#fff}.form_column .form_container[data-v-72af5bed]{background:inherit;border:inherit;border-radius:4px;padding:inherit;position:relative}.form_column .form_container .error[data-v-72af5bed]{padding:32px}.form_column .form_container .error-buttons[data-v-72af5bed]{margin:24px 0 0}.form_column .form_container .error-buttons .slp-btn[data-v-72af5bed]:first-of-type{margin-bottom:8px}.form_column .form_container .error-buttons .slp-btn .slp-icon[data-v-72af5bed]{margin-left:8px}@media(max-width:768px){.form_column .form_container .error-buttons .slp-btn[data-v-72af5bed]{justify-content:center;padding:11px 16px;width:100%}.form_column .form_container .error-buttons .slp-btn .slp-icon[data-v-72af5bed]{display:none}}.form_column .form_container[data-v-72af5bed] .mkto_consent a{text-decoration:underline}.form_column .form[data-v-72af5bed]{text-align:left}.form_header[data-v-72af5bed]{margin-top:16px}@media(min-width:768px){.form_header[data-v-72af5bed]{margin-top:0}}.form_header--align-left[data-v-72af5bed]{text-align:left}@media(max-width:576px){.form_header--align-left[data-v-72af5bed]{margin-left:10px}}hr[data-v-72af5bed]{background-color:#e8e7eb;border:none;height:1px;margin:16px 0}.required_label[data-v-72af5bed]{font-size:16px;margin:8px 0 16px}@media(max-width:576px){.required_label[data-v-72af5bed]{padding-left:10px}}.mktoForm_skeleton[data-v-72af5bed]{display:flex;flex-direction:column;gap:16px}.mktoForm_skeleton__field[data-v-72af5bed]{background:#f2f1f5;border-radius:4px;height:50px;margin-top:25px;overflow:hidden;position:relative;width:100%}.mktoForm_skeleton__field[data-v-72af5bed]:after{animation:mkto-form-skeleton-shimmer-72af5bed 1.4s ease-in-out infinite;background:linear-gradient(100deg,#fff0,#ffffff8c 50%,#fff0);content:"";inset:0;position:absolute;transform:translate(-100%)}@keyframes mkto-form-skeleton-shimmer-72af5bed{0%{transform:translate(-100%)}to{transform:translate(100%)}}@media(prefers-reduced-motion:reduce){.mktoForm_skeleton__field[data-v-72af5bed]:after{animation:none;transform:none}}.submitted_message[data-v-72af5bed]{scroll-margin-top:40px;text-align:center;width:100%}.submitted_message__default[data-v-72af5bed]{margin-top:32px}@media(max-width:1025px){.submitted_message__default[data-v-72af5bed]{padding:10px}}.submitted_message__icon[data-v-72af5bed]{margin:0 auto 24px}</style><style>.mktoLabel{text-transform:lowercase!important}.mktoLabel:first-letter{text-transform:uppercase}.mktoForm .mktoRequiredField label.mktoLabel,.mktoForm_container .mktoFieldWrap label{color:#171321;font-size:14px;font-weight:400!important}.assessment-form.form_column.form-multistep{display:flex;flex-direction:column;min-height:100%}.assessment-form.form_column.form-multistep .mktoFieldWrap:has(.mktoRadioList)>.mktoLabel{font-size:24px!important;font-style:normal!important;font-weight:700!important;line-height:28px!important;margin-bottom:32px!important}.assessment-form.form_column.form-multistep .mktoFieldWrap:has(.mktoEmailField)>.mktoLabel,.assessment-form.form_column.form-multistep .mktoFieldWrap:has(.mktoTextField)>.mktoLabel,.assessment-form.form_column.form-multistep .mktoFieldWrap:has(select)>.mktoLabel{font-weight:660!important;margin-bottom:0}.assessment-form.form_column.form-multistep .form_container{display:flex;flex:1;flex-direction:column}.assessment-form.form_column.form-multistep .mktoForm_container{flex:1}.assessment-form.form_column.form-multistep .mktoFormRow:has(>.fieldset-hidden){display:none;margin:0;padding:0}.assessment-form.form_column.form-multistep .fieldset-hidden{display:none}.assessment-form.form_column.form-multistep label{margin-top:4px!important}.assessment-form.form_column.form-multistep .mktoLabel{text-transform:none!important}.mktoForm .mktoRequiredField .mktoAsterix{display:none!important}.mktoFieldWrap input:not([type=checkbox]):not([type=radio]):not([type=hidden]),.mktoFieldWrap select,.mktoFieldWrap textarea,.mktoForm.mktoLayoutAbove .mktoField{border:1px solid #171321!important;border-radius:4px!important;color:#666!important;display:block!important;font-size:14px!important;margin-top:.4rem!important;outline:none!important;padding:1em!important;width:100%!important}.mktoFieldWrap input:not([type=checkbox]):not([type=radio]):not([type=hidden]):focus,.mktoFieldWrap select:focus,.mktoFieldWrap textarea:focus,.mktoForm.mktoLayoutAbove .mktoField:focus{box-shadow:0 0 4px 1px #00000040!important}.mktoForm_container .mktoButtonRow{width:100%!important}.mktoForm_container .mktoButtonRow .mktoButtonWrap{margin-left:0!important}.mktoButtonRow .mktoButtonWrap button,.mktoButtonRow .mktoButtonWrap.mktoSimple .mktoButton button,.mktoForm button.mktoButton button{background-color:#171321!important;background-image:none!important;border:1px solid #171321!important;border-radius:4px;color:#fff!important;display:inline-block;font-size:16px!important;font-weight:600;line-height:1;padding:16px!important;text-align:center;transition:background-color .3s ease-in-out,color .3s ease-in-out;vertical-align:middle;width:100%!important}.mktoButtonRow .mktoButtonWrap button:focus,.mktoButtonRow .mktoButtonWrap button:hover,.mktoButtonRow .mktoButtonWrap.mktoSimple .mktoButton button:focus,.mktoButtonRow .mktoButtonWrap.mktoSimple .mktoButton button:hover,.mktoForm button.mktoButton button:focus,.mktoForm button.mktoButton button:hover{background-color:#fff!important;color:#171321!important}.mktoButtonRow .mktoButtonWrap.mktoShadow button,.mktoButtonRow .mktoButtonWrap.mktoSimple .mktoButton.mktoShadow button,.mktoForm button.mktoButton.mktoShadow button{background-image:none!important;box-shadow:none!important;font-size:16px!important;padding:16px!important}.mktoForm .mktoError{position:unset!important}.mktoForm .mktoError div.mktoErrorMsg{background:transparent;border:none;box-shadow:none;color:#333;display:inline;font-size:12px;line-height:20px;padding-left:0;text-shadow:none}.mktoForm .mktoError div.mktoErrorMsg span.mktoErrorDetail{display:inline}.mktoForm_container{width:100%!important}.mktoForm_container a:not(.btn):not(.slp-btn),.mktoForm_container a:not(.btn):not(.slp-btn):active,.mktoForm_container a:not(.btn):not(.slp-btn):focus,.mktoForm_container a:not(.btn):not(.slp-btn):hover{color:#7d2bd9}.mktoForm_container .mktoField:not([type=radio]){min-height:48px}.mktoForm_container .mktoFormRow input:not([type=radio]){width:100%}.mktoForm_container .mktoHasWidth{width:100%!important}.mktoForm_container .mktoHtmlText{margin:16px 0}.mktoForm_container .mktoHtmlText a{text-decoration:underline}.mktoForm_container #commentCapture,.mktoForm_container .mktoFieldWrap,.mktoForm_container .mktoFormCol{margin-bottom:0;width:100%}.mktoForm_container .mktoLogicalField.mktoCheckboxList{display:flex}.mktoForm_container .mktoLogicalField.mktoCheckboxList input[type=checkbox]{width:30px!important}.mktoForm_container .mktoLogicalField.mktoCheckboxList label{align-items:center;display:flex;margin-bottom:4px;margin-left:.5rem}div.mktoErrorArrowWrap{display:none}div.mktoOffset{display:none!important}div.mktoGutter.mktoHasWidth{height:0!important;margin-bottom:8px!important}fieldset{border:0;margin-top:8px;padding:0}fieldset legend{color:#171321;font-size:14px;margin:0 0 8px;padding:0;text-transform:lowercase}fieldset legend:first-letter{text-transform:uppercase}.mktoForm fieldset .mktoFieldWrap:has(.mktoLogicalField){align-items:center;display:flex;flex-direction:row-reverse;gap:10px}.mktoForm fieldset .mktoFieldWrap:has(.mktoLogicalField) .mktoGutter{display:none}.mktoForm fieldset .mktoFieldWrap:has(.mktoLogicalField) .mktoLogicalField{margin:0;width:25px!important}.mktoForm fieldset .mktoFieldWrap:has(.mktoLogicalField) .mktoLogicalField input{min-height:0}.mktoForm .mktoFormCol{width:100%}.mktoForm .mktoFormCol:has(.mktoLogicalField){display:flex;flex-wrap:wrap;width:100%}.mktoForm .mktoFormCol:has(.mktoLogicalField) label{font-size:14px}.mktoForm .mktoFormCol:has(.mktoLogicalField) .mktoFormRow{width:100%}@media(min-width:768px){.mktoForm .mktoFormCol:has(.mktoLogicalField) .mktoFormRow{width:50%}}.mktoForm .mktoFormCol:has(.mktoLogicalField) .mktoFormRow .mktoOffset{display:none}.form_column .mktoFieldWrap select{-webkit-appearance:none!important;-moz-appearance:none!important;background-image:url(https://res.cloudinary.com/about-gitlab-com/image/upload/v1758224265/pwlj5uz290tau1ncdqmw.svg);background-position:calc(100% - 12px);background-repeat:no-repeat}.mktoForm_container:has(select[name=eventDateCode2]) .mktoRadioList{display:flex;flex-wrap:wrap;gap:8px;padding:0!important}.mktoForm_container:has(select[name=eventDateCode2]) .mktoRadioList input[type=radio]{height:1px;min-height:0;opacity:0;pointer-events:none;position:absolute;width:1px}.mktoForm_container:has(select[name=eventDateCode2]) .mktoRadioList input[type=radio]+label{align-items:center;background-color:#fff;border:1px solid #e8e7eb;border-radius:100px;color:#171321;cursor:pointer;display:inline-flex;font-size:16px;line-height:24px;margin:0!important;padding:10px 20px;transition:background-color .2s ease,border-color .2s ease}.mktoForm_container:has(select[name=eventDateCode2]) .mktoRadioList input[type=radio]:checked+label,.mktoForm_container:has(select[name=eventDateCode2]) .mktoRadioList input[type=radio]:hover+label{background-color:#e8e7eb}.mktoForm_container:has(select[name=eventDateCode2]) .mktoRadioList input[type=radio]:focus-visible+label{outline:2px solid #171321;outline-offset:2px}.form_column.form-multistep{background-color:#fff;border:1px solid #d1d0d3;border-radius:4px;padding:8px 32px 102px;position:relative;transition:padding .3s ease}.form_column.form-multistep.form_column--submitted{padding:32px}.form_column.form-multistep .submitted_message{text-align:left!important}.form_column.form-multistep .submitted_message__heading{font-size:32px!important;font-weight:500!important}.form_column.form-multistep .form_container{border-color:transparent!important;padding:0!important}.form_column.form-multistep .required_label{color:#74717a;font-size:14px;margin-bottom:0;padding:0!important}.form_column.form-multistep .mktoErrorMsg{text-shadow:none!important}.form_column.form-multistep .mktoField{transition:border-color .3s ease}.form_column.form-multistep .mktoFieldWrap:has(>.mktoError) .mktoInvalid,.form_column.form-multistep .mktoFieldWrap:has(>.mktoError) input.mktoField.mktoInvalid{border-color:#f24f63!important;margin-bottom:16px}.form_column.form-multistep .mktoFormRow{display:grid;grid-template-rows:1fr;overflow:hidden}.form_column.form-multistep .mktoFormRow:not(:has(>.fieldset-hidden)){transition:grid-template-rows .3s ease}.form_column.form-multistep .mktoFormRow:has(>.fieldset-hidden){grid-template-rows:0fr}.form_column.form-multistep .mktoFormRow .fieldset-hidden{min-height:0;visibility:hidden}.form_column.form-multistep legend:first-of-type{display:none}.form_column.form-multistep .form-multistep__actions{bottom:32px;display:flex;justify-content:end;left:32px;position:absolute;right:32px;width:calc(100% - 64px);z-index:0}.form_column.form-multistep .form-multistep__actions__back{margin-right:auto}.form_column.form-multistep .mktoFieldWrap .mktoLabel,.form_column.form-multistep .mktoFieldWrap label{font-size:16px!important;margin-bottom:8px}.form_column.form-multistep label{margin-top:24px}.form_column.form-multistep .mktoButtonRow{bottom:-72px;position:absolute;right:0;width:auto!important;z-index:1}.form_column.form-multistep .mktoButtonRow button{padding:14px!important}.form_column.form-multistep .mktoForm_container{padding:0}.form_column.form-multistep .mktoForm_container .mktoFormCol{padding:0!important}.form_column.form-multistep .mktoFormCol:has(.mktoLogicalField) .mktoFormRow{width:100%}.form_column.form-multistep .mktoFieldWrap:has(.mktoLogicalField){display:block}.form_column.form-multistep .mktoFieldWrap:has(.mktoLogicalField) .mktoLogicalField{align-items:center;padding:0;width:100%!important}.form_column.form-multistep .mktoFieldWrap:has(.mktoLogicalField) .mktoLogicalField label{margin-top:0!important}</style><style>.prose-img[data-v-216e27f8]{display:block}.prose-img__caption[data-v-216e27f8]{display:block;margin:8px 0 32px;text-align:center}</style><style>.code[data-v-84382b13]{background-color:#ececef;border-radius:4px;color:#18171d}</style><style>.code-block[data-v-588d8b59]{position:relative}.copy-btn[data-v-588d8b59]{background:#f5f6f8;border:1px solid #dcdfe6;border-radius:6px;color:#1f1f26;cursor:pointer;font-size:12px;line-height:1.4;padding:4px 8px;position:absolute;right:8px;top:8px;transition:background-color .15s ease,border-color .15s ease,color .15s ease}.copy-btn[data-v-588d8b59]:hover{background:#eceef3;border-color:#c5cad3}.copy-btn[data-v-588d8b59]:active{background:#e1e4eb;border-color:#b5bcc8}</style><link rel="stylesheet" href="/_nuxt/entry.Dd882eom.css" crossorigin><link rel="stylesheet" href="/_nuxt/Banner.BMJkmWbA.css" crossorigin><link rel="stylesheet" href="/_nuxt/_...DvB2Xm2x.css" crossorigin><link rel="preload" href="//page.gitlab.com/js/forms2/js/forms2.min.js" as="script" data-hid="mkto-preload-script"><link rel="preload" as="fetch" crossorigin="anonymous" href="/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/_payload.json?0834acfa-cb5b-41d1-8287-bd08f5a0480e"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/By4Z7nho.js"><link rel="preload" as="font" crossorigin href="/fonts/gitlab-sans/GitLabSans.woff2"><link rel="preload" as="font" crossorigin href="/fonts/gitlab-mono/GitLabMono.woff2"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/DAe6jp9T.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BiMSGXtC.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/Cmxf1l22.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/Cvh-IKuV.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/DPsacU1l.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/B0VHbU3h.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/Cbe7Jul6.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BMyR6M34.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BqFFb_eK.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/8V8CX2OR.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/G8wlJSRy.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/XGlb7PSy.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/CBOuVVw_.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/e3PC43bh.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/D4Lj3Z6e.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/DMGDMTDo.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/DFizG-ez.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/CRA9HQhj.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/d3LO55hk.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/CFqVtbCB.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/Q5rubXC4.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/B1a95iPH.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/HVY3QTxn.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/DVHIZA4n.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/Y0FTVmEZ.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/CJ-hnJZl.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/dA-J84ed.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BOPdMm77.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/Df5A5Q2V.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/-SOgpbLp.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/CQk8e7gi.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/JuvO3Rwh.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BGGQa44s.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/pHjadiuc.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/DZzF5X-t.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/DOwj7ZbP.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/DV3J4zpL.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/B1f3Teoo.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BqABBN1s.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/C07nSeAY.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/B5f7QnlC.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BKul2vYP.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BAoymWsp.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/CGRDWUT-.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BnEnyQ48.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/Bwhfb86Y.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BSCELZ9w.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/Dr73l3EC.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BHwje1c2.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/CnI4U7E9.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BovjNfW6.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BfsaH7vG.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/D1FF2tt-.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BrFk406r.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/CqRriLp5.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/CpBV0zsn.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/C8Xw8JgV.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/-enizZvc.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/DQaIELqQ.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/CqkleIqs.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/ChDLWnIl.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/D8_OCZC7.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BWDtx6r7.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/B-03DUlM.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/ByM3MyQR.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/CUU9HHKK.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BjxXX6cl.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/CkBxCTj9.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BrSp_lxJ.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/9T1i2qRn.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BLR4y9Gm.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/0KteTB1_.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/M6OaUIyA.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BEp5XiJo.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/2wcZrJF9.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/Dxc7EpBl.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/CN9mgHtb.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/0-nSsci1.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BUeMVcJ2.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/BjHmXoXM.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/DIuFRhnE.js"><link rel="modulepreload" as="script" crossorigin href="/_nuxt/Dh9xyc_c.js"><script src="https://cdn.bizible.com/scripts/bizible.js" defer type="text/plain" class="optanon-category-C0002"></script><script src="https://munchkin.marketo.net/munchkin.js" defer type="text/plain" class="optanon-category-C0002"></script><script src="//page.gitlab.com/js/forms2/js/forms2.min.js" type="text/javascript" defer fetchpriority="high" data-hid="mkto-forms2-script"></script><script type="module" src="/_nuxt/By4Z7nho.js" crossorigin></script><link rel="dns-prefetch" href="//page.gitlab.com" data-hid="mkto-dns-prefetch"><link rel="icon" type="image/x-icon" href="/images/ico/favicon.ico"><link rel="icon" type="image/png" sizes="32x32" href="/images/ico/favicon-32x32.png"><link rel="icon" type="image/png" sizes="192x192" href="/images/ico/favicon-192x192.png"><link rel="apple-touch-icon" href="/images/ico/favicon-192x192.png"><link rel="alternate" type="application/atom+xml" title="Major Releases" href="/releases.xml"><meta hid="contentSecurityPolicy" http-equiv="Content-Security-Policy" content="
|
||
default-src 'self' https: http:;
|
||
script-src 'self' 'unsafe-inline' 'unsafe-eval' https: http: *.googletagmanager.com *.mutinycdn.com;
|
||
style-src 'self' 'unsafe-inline' https: http:;
|
||
object-src https: http:;
|
||
base-uri 'self';
|
||
connect-src 'self' https: http: wss: ws: *.google-analytics.com *.analytics.google.com *.googletagmanager.com *.mutinyhq.com *.mutinyhq.io *.mutinycdn.com;
|
||
frame-src 'self' https: http:;
|
||
img-src 'self' https: http: data: *.google-analytics.com *.googletagmanager.com *.mutinycdn.com;
|
||
manifest-src 'self'; media-src 'self' https: http:;
|
||
child-src 'self' blob: https: http:;
|
||
font-src 'self' https: http: data:;
|
||
"><meta name="format-detection" content="telephone=no"><meta name="twitter:card" content="summary_large_image"><meta name="twitter:site" content="@GitLab"><meta name="robots" content="index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1"><link rel="alternate" hreflang="x-default" href="https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/"><link rel="alternate" hreflang="en" href="https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/"><link rel="alternate" hreflang="en-us" href="https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/"><link rel="canonical" href="https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/"><meta property="article:author" content="Matt Smiley"><meta property="article:tag" content="performance"><meta property="article:tag" content="tutorial"><meta property="article:tag" content="DevOps"><meta name="description" content="How we uncovered a three-phase cycle involving two distinct saturation points and a simple fix to break that cycle."><meta property="og:title" content="How we diagnosed and resolved Redis latency spikes with BPF and other tools"><meta property="og:description" content="How we uncovered a three-phase cycle involving two distinct saturation points and a simple fix to break that cycle."><meta property="og:image" content="https://res.cloudinary.com/about-gitlab-com/image/upload/v1749667913/Blog/Hero%20Images/clocks.jpg"><meta property="og:url" content="https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes"><meta property="og:type" content="article"><meta name="twitter:title" content="How we diagnosed and resolved Redis latency spikes with BPF and other tools"><meta name="twitter:image" content="https://res.cloudinary.com/about-gitlab-com/image/upload/v1749667913/Blog/Hero%20Images/clocks.jpg"><meta name="twitter:description" content="How we uncovered a three-phase cycle involving two distinct saturation points and a simple fix to break that cycle."><meta name="twitter:creator" content="@GitLab"><meta property="article:published_time" content="2022-11-28T00:00:00.000Z"><meta property="article:section" content="engineering"><meta property="og:site_name" content="GitLab"><script type="application/ld+json">{"@context":"https://schema.org/","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Blog","item":"https://about.gitlab.com/blog/"},{"@type":"ListItem","position":2,"name":"Engineering","item":"https://about.gitlab.com/blog/categories/engineering/"},{"@type":"ListItem","position":3,"name":"How we diagnosed and resolved Redis latency spikes with BPF and other tools","item":"https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/"}]}</script><meta property="og:locale" content="en_US"><script id="unhead:payload" type="application/json">{"title":"How we diagnosed and resolved Redis latency spikes with BPF and other tools"}</script></head><body><div id="__nuxt"><!--[--><div></div><div class="grid-wrapper grid-wrapper--has-release-banner" data-v-89b552f1><!--[--><div class="navigation__overlay" data-v-07f51b8b></div><section class="be-nav-search" style="display:none;" data-swiftype-index="false" data-v-07f51b8b data-v-25bdfcad><div class="slp-container be-nav-search__container" data-v-e594efe8 data-v-25bdfcad><!--[--><div class="slp-row be-nav-search__row" data-v-adc8d02f data-v-25bdfcad><!--[--><div class="slp-col-md-10" data-v-00b68f22 data-v-25bdfcad><!--[--><!--[--><div class="be-nav-search__close" data-v-25bdfcad><button class="slp-btn slp-btn-ghost" type="button" data-v-c6fd7cb9 data-v-25bdfcad><!--[--><span class="slp-text-body3 be-nav-search__close--text" data-v-64ad2128 data-v-25bdfcad><!--[-->Close<!--]--><!----></span><span class="slp-icon slp-icon--size-md slp-icon--color-color-text-300 slp-mr-16" style="" role="img" aria-hidden="true" data-v-25bdfcad><svg
|
||
width="16"
|
||
height="16"
|
||
viewBox="0 0 16 16"
|
||
preserveAspectRatio="xMidYMid meet"
|
||
fill="currentColor"
|
||
xmlns="http://www.w3.org/2000/svg"
|
||
>
|
||
<path
|
||
fill-rule="evenodd"
|
||
clip-rule="evenodd"
|
||
d="M4.28 3.22a.75.75 0 0 0-1.06 1.06L6.94 8l-3.72 3.72a.75.75 0 1 0 1.06 1.06L8 9.06l3.72 3.72a.75.75 0 1 0 1.06-1.06L9.06 8l3.72-3.72a.75.75 0 0 0-1.06-1.06L8 6.94 4.28 3.22Z"
|
||
/>
|
||
</svg>
|
||
</span><!--]--></button></div><div class="be-nav-search-content" data-v-25bdfcad><div class="be-nav-search-content__input slp-mb-16"><input value="" type="text" placeholder="Search"><!----></div><div class="slp-mb-48"><div class="slp-text-body3 slp-mb-8" data-v-64ad2128><!--[-->To search repositories and projects, login to <a class="be-nav-search-content__link" href="https://gitlab.com/users/sign_in/" target="_blank" data-ga-name="search login" data-ga-location="search">gitlab.com</a>. <!--]--><!----></div></div><div class="be-nav-suggestions" data-v-0823afcc><span class="slp-text-body1 slp-mb-24" data-v-64ad2128 data-v-0823afcc><!--[-->Suggestions<!--]--><!----></span><div class="be-nav-suggestions__default" data-v-0823afcc><!--[--><a class="be-nav-suggestions__default-btn slp-btn slp-btn-primary be-nav-suggestions__default-btn" data-ga-name="GitLab Duo Agent Platform" data-ga-location="search" target="_blank" href="/gitlab-duo-agent-platform/" data-v-c6fd7cb9 data-v-0823afcc><!--[-->GitLab Duo Agent Platform<!--]--></a><a class="be-nav-suggestions__default-btn slp-btn slp-btn-primary be-nav-suggestions__default-btn" data-ga-name="Code Suggestions (AI)" data-ga-location="search" target="_blank" href="/solutions/code-suggestions/" data-v-c6fd7cb9 data-v-0823afcc><!--[-->Code Suggestions (AI)<!--]--></a><a class="be-nav-suggestions__default-btn slp-btn slp-btn-primary be-nav-suggestions__default-btn" data-ga-name="CI/CD" data-ga-location="search" target="_blank" href="/solutions/continuous-integration/" data-v-c6fd7cb9 data-v-0823afcc><!--[-->CI/CD<!--]--></a><a class="be-nav-suggestions__default-btn slp-btn slp-btn-primary be-nav-suggestions__default-btn" data-ga-name="GitLab on AWS" data-ga-location="search" target="_blank" href="/partners/technology-partners/aws/" data-v-c6fd7cb9 data-v-0823afcc><!--[-->GitLab on AWS<!--]--></a><a class="be-nav-suggestions__default-btn slp-btn slp-btn-primary be-nav-suggestions__default-btn" data-ga-name="GitLab on Google Cloud" data-ga-location="search" target="_blank" href="/partners/technology-partners/google-cloud-platform/" data-v-c6fd7cb9 data-v-0823afcc><!--[-->GitLab on Google Cloud<!--]--></a><a class="be-nav-suggestions__default-btn slp-btn slp-btn-primary be-nav-suggestions__default-btn" data-ga-name="Why GitLab?" data-ga-location="search" target="_blank" href="/why-gitlab/" data-v-c6fd7cb9 data-v-0823afcc><!--[-->Why GitLab?<!--]--></a><!--]--></div><div data-v-0823afcc><!--[--><!--]--></div></div></div><!--]--><!--]--></div><!--]--></div><!--]--></div></section><header class="navigation" data-swiftype-index="false" data-v-07f51b8b><nav class="navigation__desktop" aria-label="Navigation" data-v-07f51b8b><div class="slp-flex" data-v-07f51b8b><a class="navigation__logo slp-btn slp-btn-icon navigation__logo" tabindex="0" aria-label="Home" data-ga-name="gitlab logo" data-ga-location="header" href="/" data-v-c6fd7cb9 data-v-07f51b8b><!--[--><span class="slp-icon slp-icon--size-lg slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-07f51b8b><svg fill="none" viewBox="0 0 32 32">
|
||
<path
|
||
fill="#E24329"
|
||
d="M31.462 12.779l-.045-.115-4.35-11.35a1.137 1.137 0 00-.447-.541 1.163 1.163 0 00-1.343.071c-.187.15-.322.356-.386.587l-2.94 9.001h-11.9l-2.941-9a1.138 1.138 0 00-1.045-.84 1.153 1.153 0 00-1.13.72L.579 12.68l-.045.113a8.09 8.09 0 002.68 9.34l.016.012.038.03 6.635 4.967 3.28 2.484 1.994 1.51a1.35 1.35 0 001.627 0l1.994-1.51 3.282-2.484 6.673-4.997.018-.013a8.088 8.088 0 002.69-9.352z"
|
||
/>
|
||
<path
|
||
fill="#FC6D26"
|
||
d="M31.462 12.779l-.045-.115a14.748 14.748 0 00-5.856 2.634l-9.553 7.24A11225.6 11225.6 0 0022.1 27.14l6.673-4.997.019-.013a8.09 8.09 0 002.67-9.352z"
|
||
/>
|
||
<path
|
||
fill="#FCA326"
|
||
d="M9.908 27.14l3.275 2.485 1.994 1.51a1.35 1.35 0 001.627 0l1.994-1.51 3.282-2.484s-2.835-2.14-6.092-4.603l-6.08 4.603z"
|
||
/>
|
||
<path
|
||
fill="#FC6D26"
|
||
d="M6.435 15.305A14.712 14.712 0 00.58 12.672l-.045.113a8.09 8.09 0 002.68 9.347l.016.012.038.03 6.635 4.967 6.105-4.603-9.573-7.233z"
|
||
/>
|
||
</svg>
|
||
</span><!--]--></a><ul class="navigation__menu" role="menubar" data-v-07f51b8b><!--[--><li class="navigation__menu-item" role="none" tabindex="-1" data-v-07f51b8b><!--[--><button id="navigation-submenu-trigger-0" class="navigation__menu-trigger slp-btn slp-btn-ghost navigation__menu-trigger" name="Platform" aria-expanded="false" aria-haspopup="true" aria-controls="navigation-submenu-0" data-ga-name="Platform" data-ga-location="header" data-nav-levelone="platform" role="menuitem" tabindex="0" type="button" data-v-c6fd7cb9 data-v-07f51b8b><!--[--><span data-v-07f51b8b>Platform</span><span class="slp-icon slp-icon--size-xs slp-icon--color-color-text-300 navigation__chevron" style="" role="img" aria-hidden="true" data-v-07f51b8b><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M2.22 5.22a.75.75 0 000 1.06l5.252 5.252a.75.75 0 001.06 0l5.252-5.252a.75.75 0 10-1.06-1.06L8.001 9.94 3.28 5.22a.75.75 0 00-1.06 0z"/></svg></span><!--]--></button><div id="navigation-submenu-0" class="navigation__dropdown" role="region" aria-labelledby="navigation-submenu-trigger-0" aria-label="Platform links" aria-live="polite" data-v-07f51b8b><div class="navigation__dropdown-content" data-v-07f51b8b><div style="flex:1;" data-v-07f51b8b><div class="navigation-dropdown-lists" data-v-07f51b8b data-v-70f8ea54><div class="navigation-dropdown-lists__columns" data-v-70f8ea54><!--[--><ul class="navigation-dropdown-lists__list" data-v-70f8ea54><h3 class="slp-text-body2-bold navigation-dropdown-lists__column-title" data-v-64ad2128 data-v-70f8ea54><!--[-->Execution & Workflows<!--]--><!----></h3><!--[--><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="CI/CD" data-nav-levelone="platform" data-nav-leveltwo="ci/cd" data-ga-name="ci/cd" data-ga-location="header" tabindex="0" href="/solutions/continuous-integration/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> CI/CD<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Source Code Management" data-nav-levelone="platform" data-nav-leveltwo="source code management" data-ga-name="source code management" data-ga-location="header" tabindex="0" href="/solutions/source-code-management/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Source Code Management<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Agile Delivery" data-nav-levelone="platform" data-nav-leveltwo="agile delivery" data-ga-name="agile delivery" data-ga-location="header" tabindex="0" href="/solutions/agile-delivery/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Agile Delivery<!--]--></a></li><!--]--></ul><ul class="navigation-dropdown-lists__list" data-v-70f8ea54><h3 class="slp-text-body2-bold navigation-dropdown-lists__column-title" data-v-64ad2128 data-v-70f8ea54><!--[-->Security & Governance<!--]--><!----></h3><!--[--><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Application Security Testing" data-nav-levelone="platform" data-nav-leveltwo="application security testing" data-ga-name="application security testing" data-ga-location="header" tabindex="0" href="/solutions/application-security-testing/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Application Security Testing<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Governance & Compliance" data-nav-levelone="platform" data-nav-leveltwo="governance and compliance" data-ga-name="governance and compliance" data-ga-location="header" tabindex="0" href="/solutions/software-compliance/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Governance & Compliance<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Supply Chain Security" data-nav-levelone="platform" data-nav-leveltwo="supply chain security" data-ga-name="supply chain security" data-ga-location="header" tabindex="0" href="/solutions/supply-chain/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Supply Chain Security<!--]--></a></li><!--]--></ul><ul class="navigation-dropdown-lists__list" data-v-70f8ea54><h3 class="slp-text-body2-bold navigation-dropdown-lists__column-title" data-v-64ad2128 data-v-70f8ea54><!--[-->Context & AI<!--]--><!----></h3><!--[--><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Agentic Orchestration" data-nav-levelone="platform" data-nav-leveltwo="agentic orchestration" data-ga-name="agentic orchestration" data-ga-location="header" tabindex="0" href="/gitlab-duo-agent-platform/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Agentic Orchestration<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Context Graph" data-nav-levelone="platform" data-nav-leveltwo="context graph" data-ga-name="context graph" data-ga-location="header" tabindex="0" href="/gitlab-orbit/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Context Graph<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Visibility & Measurement" data-nav-levelone="platform" data-nav-leveltwo="visibility and measurement" data-ga-name="visibility and measurement" data-ga-location="header" tabindex="0" href="/solutions/visibility-measurement/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Visibility & Measurement<!--]--></a></li><!--]--></ul><!--]--></div></div><a class="slp-mx-32 slp-my-16 slp-btn slp-btn-tertiary slp-mx-32 slp-my-16" name="Explore the Platform" data-ga-name="platform" data-ga-location="header" data-nav-level-one="platform" data-nav-level-two="Explore the Platform" tabindex="0" href="/platform/" data-v-c6fd7cb9 data-v-07f51b8b><!--[-->Explore the Platform <span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-07f51b8b><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M5.22 2.22a.75.75 0 011.06 0l5.252 5.252a.75.75 0 010 1.06L6.28 13.784a.75.75 0 11-1.06-1.06l4.72-4.723L5.22 3.28a.75.75 0 010-1.06z"/></svg></span><!--]--></a></div><aside class="navigation__dropdown-feature" data-v-07f51b8b data-v-f3a57a08><article class="navigation__feature" style="--background:url('https://res.cloudinary.com/about-gitlab-com/image/upload/v1777322348/qpq8yrgn8knii57omj0c.png');--textColor:#000;" data-v-f3a57a08><a href="/why-gitlab/" class="navigation__feature-link" aria-labelledby="feature-title-why gitlab" data-ga-name="why gitlab" data-ga-location="header" data-nav-levelone="platform" data-nav-leveltwo="Learn more" data-v-f3a57a08><div class="navigation__feature-content" data-v-f3a57a08><div data-v-f3a57a08><header id="feature-title-why gitlab" class="navigation__feature-title slp-mb-16" data-v-f3a57a08><span class="slp-text-heading3-bold" data-v-64ad2128 data-v-f3a57a08><!--[-->Why GitLab<!--]--><!----></span></header><!----><p class="slp-text-body1 navigation__feature-description" data-v-64ad2128 data-v-f3a57a08><!--[-->One platform for speed with control across your software lifecycle.<!--]--><!----></p></div><span class="slp-text-body1-bold navigation__feature-cta" data-v-64ad2128 data-v-f3a57a08><!--[-->Learn more <span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300 slp-ml-8" style="" role="img" aria-hidden="true" data-v-f3a57a08><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M5.22 2.22a.75.75 0 011.06 0l5.252 5.252a.75.75 0 010 1.06L6.28 13.784a.75.75 0 11-1.06-1.06l4.72-4.723L5.22 3.28a.75.75 0 010-1.06z"/></svg></span><!--]--><!----></span></div></a></article></aside></div></div><!--]--></li><li class="navigation__menu-item" role="none" tabindex="-1" data-v-07f51b8b><!--[--><button id="navigation-submenu-trigger-1" class="navigation__menu-trigger slp-btn slp-btn-ghost navigation__menu-trigger" name="Solutions" aria-expanded="false" aria-haspopup="true" aria-controls="navigation-submenu-1" data-ga-name="Solutions" data-ga-location="header" data-nav-levelone="solutions" role="menuitem" tabindex="0" type="button" data-v-c6fd7cb9 data-v-07f51b8b><!--[--><span data-v-07f51b8b>Solutions</span><span class="slp-icon slp-icon--size-xs slp-icon--color-color-text-300 navigation__chevron" style="" role="img" aria-hidden="true" data-v-07f51b8b><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M2.22 5.22a.75.75 0 000 1.06l5.252 5.252a.75.75 0 001.06 0l5.252-5.252a.75.75 0 10-1.06-1.06L8.001 9.94 3.28 5.22a.75.75 0 00-1.06 0z"/></svg></span><!--]--></button><div id="navigation-submenu-1" class="navigation__dropdown" role="region" aria-labelledby="navigation-submenu-trigger-1" aria-label="Solutions links" aria-live="polite" data-v-07f51b8b><div class="navigation__dropdown-content" data-v-07f51b8b><div style="flex:1;" data-v-07f51b8b><div class="navigation-dropdown-lists" data-v-07f51b8b data-v-70f8ea54><div class="navigation-dropdown-lists__columns" data-v-70f8ea54><!--[--><ul class="navigation-dropdown-lists__list" data-v-70f8ea54><h3 class="slp-text-body2-bold navigation-dropdown-lists__column-title" data-v-64ad2128 data-v-70f8ea54><!--[-->Outcomes<!--]--><!----></h3><!--[--><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="DevOps Modernization" data-nav-levelone="solutions" data-nav-leveltwo="devops modernization" data-ga-name="devops modernization" data-ga-location="header" tabindex="0" href="/assessments/devops-modernization-assessment/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> DevOps Modernization<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Security Modernization" data-nav-levelone="solutions" data-nav-leveltwo="security modernization" data-ga-name="security modernization" data-ga-location="header" tabindex="0" href="/assessments/security-modernization-assessment/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Security Modernization<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="AI Modernization" data-nav-levelone="solutions" data-nav-leveltwo="ai modernization" data-ga-name="ai modernization" data-ga-location="header" tabindex="0" href="/assessments/ai-modernization-assessment/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> AI Modernization<!--]--></a></li><!--]--></ul><ul class="navigation-dropdown-lists__list" data-v-70f8ea54><h3 class="slp-text-body2-bold navigation-dropdown-lists__column-title" data-v-64ad2128 data-v-70f8ea54><!--[-->By size<!--]--><!----></h3><!--[--><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Enterprise" data-nav-levelone="solutions" data-nav-leveltwo="enterprise" data-ga-name="enterprise" data-ga-location="header" tabindex="0" href="/enterprise/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Enterprise<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Small Business" data-nav-levelone="solutions" data-nav-leveltwo="small business" data-ga-name="small business" data-ga-location="header" tabindex="0" href="/small-business/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Small Business<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Startups" data-nav-levelone="solutions" data-nav-leveltwo="startups" data-ga-name="startups" data-ga-location="header" tabindex="0" href="/solutions/startups/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Startups<!--]--></a></li><!--]--></ul><ul class="navigation-dropdown-lists__list" data-v-70f8ea54><h3 class="slp-text-body2-bold navigation-dropdown-lists__column-title" data-v-64ad2128 data-v-70f8ea54><!--[-->Industries<!--]--><!----></h3><!--[--><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Financial Services" data-nav-levelone="solutions" data-nav-leveltwo="financial services" data-ga-name="financial services" data-ga-location="header" tabindex="0" href="/solutions/finance/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Financial Services<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Public Sector" data-nav-levelone="solutions" data-nav-leveltwo="public sector" data-ga-name="public sector" data-ga-location="header" tabindex="0" href="/solutions/public-sector/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Public Sector<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Telecommunications" data-nav-levelone="solutions" data-nav-leveltwo="telecommunications" data-ga-name="telecommunications" data-ga-location="header" tabindex="0" href="/solutions/telecommunications/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Telecommunications<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Automotive" data-nav-levelone="solutions" data-nav-leveltwo="automotive" data-ga-name="automotive" data-ga-location="header" tabindex="0" href="/solutions/automotive/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Automotive<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Education" data-nav-levelone="solutions" data-nav-leveltwo="education" data-ga-name="education" data-ga-location="header" tabindex="0" href="/solutions/education/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Education<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Aerospace" data-nav-levelone="solutions" data-nav-leveltwo="aerospace" data-ga-name="aerospace" data-ga-location="header" tabindex="0" href="/solutions/aerospace/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Aerospace<!--]--></a></li><!--]--></ul><!--]--></div></div><a class="slp-mx-32 slp-my-16 slp-btn slp-btn-tertiary slp-mx-32 slp-my-16" name="View all Solutions" data-ga-name="solutions" data-ga-location="header" data-nav-level-one="solutions" data-nav-level-two="View all Solutions" tabindex="0" href="/solutions/" data-v-c6fd7cb9 data-v-07f51b8b><!--[-->View all Solutions <span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-07f51b8b><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M5.22 2.22a.75.75 0 011.06 0l5.252 5.252a.75.75 0 010 1.06L6.28 13.784a.75.75 0 11-1.06-1.06l4.72-4.723L5.22 3.28a.75.75 0 010-1.06z"/></svg></span><!--]--></a></div><aside class="navigation__dropdown-feature" data-v-07f51b8b data-v-f3a57a08><article class="navigation__feature" style="--background:url('https://res.cloudinary.com/about-gitlab-com/image/upload/v1777322348/qpq8yrgn8knii57omj0c.png');--textColor:#000;" data-v-f3a57a08><a href="/blog/gitlab-transcend-announcements/" class="navigation__feature-link" aria-labelledby="feature-title-gitlab transcend" data-ga-name="gitlab transcend" data-ga-location="header" data-nav-levelone="solutions" data-nav-leveltwo="Read the blog" data-v-f3a57a08><div class="navigation__feature-content" data-v-f3a57a08><div data-v-f3a57a08><header id="feature-title-gitlab transcend" class="navigation__feature-title slp-mb-16" data-v-f3a57a08><span class="slp-text-heading3-bold" data-v-64ad2128 data-v-f3a57a08><!--[-->GitLab Transcend<!--]--><!----></span></header><!----><p class="slp-text-body1 navigation__feature-description" data-v-64ad2128 data-v-f3a57a08><!--[-->Catch our latest innovations announced at the last Transcend.<!--]--><!----></p></div><span class="slp-text-body1-bold navigation__feature-cta" data-v-64ad2128 data-v-f3a57a08><!--[-->Read the blog <span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300 slp-ml-8" style="" role="img" aria-hidden="true" data-v-f3a57a08><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M5.22 2.22a.75.75 0 011.06 0l5.252 5.252a.75.75 0 010 1.06L6.28 13.784a.75.75 0 11-1.06-1.06l4.72-4.723L5.22 3.28a.75.75 0 010-1.06z"/></svg></span><!--]--><!----></span></div></a></article></aside></div></div><!--]--></li><li class="navigation__menu-item" role="none" tabindex="-1" data-v-07f51b8b><a class="navigation__menu-link slp-btn slp-btn-ghost navigation__menu-link" name="Pricing" tabindex="0" data-ga-name="pricing" data-ga-location="header" data-nav-levelone="pricing" href="/pricing/" data-v-c6fd7cb9 data-v-07f51b8b><!--[-->Pricing<!--]--></a></li><li class="navigation__menu-item" role="none" tabindex="-1" data-v-07f51b8b><!--[--><button id="navigation-submenu-trigger-3" class="navigation__menu-trigger slp-btn slp-btn-ghost navigation__menu-trigger" name="Resources" aria-expanded="false" aria-haspopup="true" aria-controls="navigation-submenu-3" data-ga-name="Resources" data-ga-location="header" data-nav-levelone="resources" role="menuitem" tabindex="0" type="button" data-v-c6fd7cb9 data-v-07f51b8b><!--[--><span data-v-07f51b8b>Resources</span><span class="slp-icon slp-icon--size-xs slp-icon--color-color-text-300 navigation__chevron" style="" role="img" aria-hidden="true" data-v-07f51b8b><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M2.22 5.22a.75.75 0 000 1.06l5.252 5.252a.75.75 0 001.06 0l5.252-5.252a.75.75 0 10-1.06-1.06L8.001 9.94 3.28 5.22a.75.75 0 00-1.06 0z"/></svg></span><!--]--></button><div id="navigation-submenu-3" class="navigation__dropdown" role="region" aria-labelledby="navigation-submenu-trigger-3" aria-label="Resources links" aria-live="polite" data-v-07f51b8b><div class="navigation__dropdown-content" data-v-07f51b8b><div style="flex:1;" data-v-07f51b8b><div class="navigation-dropdown-lists" data-v-07f51b8b data-v-70f8ea54><div class="navigation-dropdown-lists__columns" data-v-70f8ea54><!--[--><ul class="navigation-dropdown-lists__list" data-v-70f8ea54><h3 class="slp-text-body2-bold navigation-dropdown-lists__column-title" data-v-64ad2128 data-v-70f8ea54><!--[-->Discover<!--]--><!----></h3><!--[--><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Docs" data-nav-levelone="resources" data-nav-leveltwo="docs" data-ga-name="docs" data-ga-location="header" tabindex="0" href="https://docs.gitlab.com/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Docs<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="University" data-nav-levelone="resources" data-nav-leveltwo="university" data-ga-name="university" data-ga-location="header" tabindex="0" href="https://university.gitlab.com/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> University<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Demo Series" data-nav-levelone="resources" data-nav-leveltwo="demo series" data-ga-name="demo series" data-ga-location="header" tabindex="0" href="/gitlab-technical-demo-series/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Demo Series<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Demo Hub" data-nav-levelone="resources" data-nav-leveltwo="demo hub" data-ga-name="demo hub" data-ga-location="header" tabindex="0" href="/demo-hub/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Demo Hub<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Services" data-nav-levelone="resources" data-nav-leveltwo="services" data-ga-name="services" data-ga-location="header" tabindex="0" href="/services/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Services<!--]--></a></li><!--]--></ul><ul class="navigation-dropdown-lists__list" data-v-70f8ea54><h3 class="slp-text-body2-bold navigation-dropdown-lists__column-title" data-v-64ad2128 data-v-70f8ea54><!--[-->Connect<!--]--><!----></h3><!--[--><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Blog" data-nav-levelone="resources" data-nav-leveltwo="blog" data-ga-name="blog" data-ga-location="header" tabindex="0" href="/blog/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Blog<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Community" data-nav-levelone="resources" data-nav-leveltwo="community" data-ga-name="community" data-ga-location="header" tabindex="0" href="/community/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Community<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Customers" data-nav-levelone="resources" data-nav-leveltwo="customers" data-ga-name="customers" data-ga-location="header" tabindex="0" href="/customers/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Customers<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Partners" data-nav-levelone="resources" data-nav-leveltwo="partners" data-ga-name="partners" data-ga-location="header" tabindex="0" href="/partners/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Partners<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Events" data-nav-levelone="resources" data-nav-leveltwo="events" data-ga-name="events" data-ga-location="header" tabindex="0" href="/events/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Events<!--]--></a></li><!--]--></ul><!--]--></div></div><a class="slp-mx-32 slp-my-16 slp-btn slp-btn-tertiary slp-mx-32 slp-my-16" name="View all resources" data-ga-name="resources" data-ga-location="header" data-nav-level-one="resources" data-nav-level-two="View all resources" tabindex="0" href="/resources/" data-v-c6fd7cb9 data-v-07f51b8b><!--[-->View all resources <span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-07f51b8b><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M5.22 2.22a.75.75 0 011.06 0l5.252 5.252a.75.75 0 010 1.06L6.28 13.784a.75.75 0 11-1.06-1.06l4.72-4.723L5.22 3.28a.75.75 0 010-1.06z"/></svg></span><!--]--></a></div><aside class="navigation__dropdown-feature" data-v-07f51b8b data-v-f3a57a08><article class="navigation__feature" style="--background:url('https://res.cloudinary.com/about-gitlab-com/image/upload/v1777322348/qpq8yrgn8knii57omj0c.png');--textColor:#000;" data-v-f3a57a08><a href="/whats-new/" class="navigation__feature-link" aria-labelledby="feature-title-whats new" data-ga-name="whats new" data-ga-location="header" data-nav-levelone="resources" data-nav-leveltwo="Read the latest" data-v-f3a57a08><div class="navigation__feature-content" data-v-f3a57a08><div data-v-f3a57a08><header id="feature-title-whats new" class="navigation__feature-title slp-mb-16" data-v-f3a57a08><span class="slp-text-heading3-bold" data-v-64ad2128 data-v-f3a57a08><!--[-->What’s new in GitLab<!--]--><!----></span></header><!----><p class="slp-text-body1 navigation__feature-description" data-v-64ad2128 data-v-f3a57a08><!--[-->Stay updated with our latest features and improvements.<!--]--><!----></p></div><span class="slp-text-body1-bold navigation__feature-cta" data-v-64ad2128 data-v-f3a57a08><!--[-->Read the latest <span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300 slp-ml-8" style="" role="img" aria-hidden="true" data-v-f3a57a08><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M5.22 2.22a.75.75 0 011.06 0l5.252 5.252a.75.75 0 010 1.06L6.28 13.784a.75.75 0 11-1.06-1.06l4.72-4.723L5.22 3.28a.75.75 0 010-1.06z"/></svg></span><!--]--><!----></span></div></a></article></aside></div></div><!--]--></li><li class="navigation__menu-item" role="none" tabindex="-1" data-v-07f51b8b><!--[--><button id="navigation-submenu-trigger-4" class="navigation__menu-trigger slp-btn slp-btn-ghost navigation__menu-trigger" name="Company" aria-expanded="false" aria-haspopup="true" aria-controls="navigation-submenu-4" data-ga-name="Company" data-ga-location="header" data-nav-levelone="company" role="menuitem" tabindex="0" type="button" data-v-c6fd7cb9 data-v-07f51b8b><!--[--><span data-v-07f51b8b>Company</span><span class="slp-icon slp-icon--size-xs slp-icon--color-color-text-300 navigation__chevron" style="" role="img" aria-hidden="true" data-v-07f51b8b><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M2.22 5.22a.75.75 0 000 1.06l5.252 5.252a.75.75 0 001.06 0l5.252-5.252a.75.75 0 10-1.06-1.06L8.001 9.94 3.28 5.22a.75.75 0 00-1.06 0z"/></svg></span><!--]--></button><div id="navigation-submenu-4" class="navigation__dropdown" role="region" aria-labelledby="navigation-submenu-trigger-4" aria-label="Company links" aria-live="polite" data-v-07f51b8b><div class="navigation__dropdown-content" data-v-07f51b8b><div style="flex:1;" data-v-07f51b8b><div class="navigation-dropdown-lists" data-v-07f51b8b data-v-70f8ea54><div class="navigation-dropdown-lists__columns" data-v-70f8ea54><!--[--><ul class="navigation-dropdown-lists__list" data-v-70f8ea54><h3 class="slp-text-body2-bold navigation-dropdown-lists__column-title" data-v-64ad2128 data-v-70f8ea54><!--[--><!--]--><!----></h3><!--[--><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="About" data-nav-levelone="company" data-nav-leveltwo="about" data-ga-name="about" data-ga-location="header" tabindex="0" href="/company/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> About<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Jobs" data-nav-levelone="company" data-nav-leveltwo="jobs" data-ga-name="jobs" data-ga-location="header" tabindex="0" href="/jobs/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Jobs<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Press" data-nav-levelone="company" data-nav-leveltwo="press" data-ga-name="press" data-ga-location="header" tabindex="0" href="/press/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Press<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Handbook" data-nav-levelone="company" data-nav-leveltwo="handbook" data-ga-name="handbook" data-ga-location="header" tabindex="0" href="https://handbook.gitlab.com/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Handbook<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Leadership" data-nav-levelone="company" data-nav-leveltwo="leadership" data-ga-name="leadership" data-ga-location="header" tabindex="0" href="/company/team/e-group/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Leadership<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Investor relations" data-nav-levelone="company" data-nav-leveltwo="investor relations" data-ga-name="investor relations" data-ga-location="header" tabindex="0" href="https://ir.gitlab.com/overview/default.aspx" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Investor relations<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Trust Center" data-nav-levelone="company" data-nav-leveltwo="trust center" data-ga-name="trust center" data-ga-location="header" tabindex="0" href="/security/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Trust Center<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="AI Transparency Center" data-nav-levelone="company" data-nav-leveltwo="ai transparency center" data-ga-name="ai transparency center" data-ga-location="header" tabindex="0" href="/ai-transparency-center/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> AI Transparency Center<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Newsletter" data-nav-levelone="company" data-nav-leveltwo="newsletter" data-ga-name="newsletter" data-ga-location="header" tabindex="0" href="/company/contact/#contact-forms" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Newsletter<!--]--></a></li><!--]--></ul><!--]--></div></div><!----></div><!----></div></div><!--]--></li><li class="navigation__menu-item" role="none" tabindex="-1" data-v-07f51b8b><!--[--><button id="navigation-submenu-trigger-5" class="navigation__menu-trigger slp-btn slp-btn-ghost navigation__menu-trigger" name="Contact us" aria-expanded="false" aria-haspopup="true" aria-controls="navigation-submenu-5" data-ga-name="Contact us" data-ga-location="header" data-nav-levelone="company" role="menuitem" tabindex="0" type="button" data-v-c6fd7cb9 data-v-07f51b8b><!--[--><span data-v-07f51b8b>Contact us</span><span class="slp-icon slp-icon--size-xs slp-icon--color-color-text-300 navigation__chevron" style="" role="img" aria-hidden="true" data-v-07f51b8b><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M2.22 5.22a.75.75 0 000 1.06l5.252 5.252a.75.75 0 001.06 0l5.252-5.252a.75.75 0 10-1.06-1.06L8.001 9.94 3.28 5.22a.75.75 0 00-1.06 0z"/></svg></span><!--]--></button><div id="navigation-submenu-5" class="navigation__dropdown" role="region" aria-labelledby="navigation-submenu-trigger-5" aria-label="Contact us links" aria-live="polite" data-v-07f51b8b><div class="navigation__dropdown-content" data-v-07f51b8b><div style="flex:1;" data-v-07f51b8b><div class="navigation-dropdown-lists" data-v-07f51b8b data-v-70f8ea54><div class="navigation-dropdown-lists__columns" data-v-70f8ea54><!--[--><ul class="navigation-dropdown-lists__list" data-v-70f8ea54><h3 class="slp-text-body2-bold navigation-dropdown-lists__column-title" data-v-64ad2128 data-v-70f8ea54><!--[--><!--]--><!----></h3><!--[--><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Talk to sales" data-nav-levelone="company" data-nav-leveltwo="talk to sales" data-ga-name="talk to sales" data-ga-location="header" tabindex="0" href="/sales/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Talk to sales<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Support portal" data-nav-levelone="company" data-nav-leveltwo="support portal" data-ga-name="support portal" data-ga-location="header" tabindex="0" href="https://support.gitlab.com/hc/en-us" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Support portal<!--]--></a></li><li class="navigation-dropdown-lists__item" data-v-70f8ea54><a class="navigation-dropdown-lists__link slp-btn slp-btn-ghost navigation-dropdown-lists__link" name="Customer portal" data-nav-levelone="company" data-nav-leveltwo="customer portal" data-ga-name="customer portal" data-ga-location="header" tabindex="0" href="https://customers.gitlab.com/customers/sign_in/" data-v-c6fd7cb9 data-v-70f8ea54><!--[--><!----> Customer portal<!--]--></a></li><!--]--></ul><!--]--></div></div><!----></div><!----></div></div><!--]--></li><!--]--></ul></div><div class="navigation__actions" data-v-07f51b8b><button tabindex="0" aria-label="Search" data-ga-name="site search" data-ga-location="header" class="navigation__search slp-btn slp-btn-icon navigation__search" type="button" data-v-c6fd7cb9 data-v-07f51b8b><!--[--><span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-07f51b8b><svg width="16" height="16" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
|
||
<path
|
||
fill-rule="evenodd"
|
||
clip-rule="evenodd"
|
||
d="M10.4765 11.8907C9.49572 12.5892 8.29583 13 7 13C3.68629 13 1 10.3137 1 7C1 3.68629 3.68629 1 7 1C10.3137 1 13 3.68629 13 7C13 8.29583 12.5892 9.49572 11.8907 10.4765L14.7071 13.2929C15.0976 13.6834 15.0976 14.3166 14.7071 14.7071C14.3166 15.0976 13.6834 15.0976 13.2929 14.7071L10.4765 11.8907ZM11 7C11 9.20914 9.20914 11 7 11C4.79086 11 3 9.20914 3 7C3 4.79086 4.79086 3 7 3C9.20914 3 11 4.79086 11 7Z"
|
||
fill="currentColor"
|
||
/>
|
||
</svg>
|
||
</span><!--]--></button><a tabindex="0" data-ga-name="sales" data-ga-location="header" name="Request a demo" class="slp-btn slp-btn-secondary" href="/sales/?contact-topic=request-demo" data-v-c6fd7cb9 data-v-07f51b8b><!--[-->Request a demo<!--]--></a><a tabindex="0" data-ga-name="free trial" data-ga-location="header" class="slp-btn slp-btn-primary" href="https://gitlab.com/-/trial_registrations/new?glm_source=about.gitlab.com&glm_content=default-saas-trial/" data-v-c6fd7cb9 data-v-07f51b8b><!--[-->Get free trial<!--]--></a><a class="navigation__login slp-btn slp-btn-ghost navigation__login" tabindex="0" data-ga-name="sign in" data-ga-location="header" href="https://gitlab.com/users/sign_in/" data-v-c6fd7cb9 data-v-07f51b8b><!--[-->Sign in<!--]--></a><!----></div></nav><nav class="navigation__mobile" aria-label="Navigation" data-v-07f51b8b><a class="navigation__mobile-trial-button slp-btn slp-btn-primary navigation__mobile-trial-button" name="Get free trial" data-ga-name="free trial" data-ga-location="header" href="https://gitlab.com/-/trial_registrations/new?glm_source=about.gitlab.com&glm_content=default-saas-trial/" data-v-c6fd7cb9 data-v-07f51b8b><!--[-->Get free trial<!--]--></a><a class="navigation__mobile-logo slp-btn slp-btn-icon navigation__mobile-logo" aria-label="Home" data-ga-name="gitlab logo" data-ga-location="header" href="/" data-v-c6fd7cb9 data-v-07f51b8b><!--[--><span class="slp-icon slp-icon--size-lg slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-07f51b8b><svg fill="none" viewBox="0 0 32 32">
|
||
<path
|
||
fill="#E24329"
|
||
d="M31.462 12.779l-.045-.115-4.35-11.35a1.137 1.137 0 00-.447-.541 1.163 1.163 0 00-1.343.071c-.187.15-.322.356-.386.587l-2.94 9.001h-11.9l-2.941-9a1.138 1.138 0 00-1.045-.84 1.153 1.153 0 00-1.13.72L.579 12.68l-.045.113a8.09 8.09 0 002.68 9.34l.016.012.038.03 6.635 4.967 3.28 2.484 1.994 1.51a1.35 1.35 0 001.627 0l1.994-1.51 3.282-2.484 6.673-4.997.018-.013a8.088 8.088 0 002.69-9.352z"
|
||
/>
|
||
<path
|
||
fill="#FC6D26"
|
||
d="M31.462 12.779l-.045-.115a14.748 14.748 0 00-5.856 2.634l-9.553 7.24A11225.6 11225.6 0 0022.1 27.14l6.673-4.997.019-.013a8.09 8.09 0 002.67-9.352z"
|
||
/>
|
||
<path
|
||
fill="#FCA326"
|
||
d="M9.908 27.14l3.275 2.485 1.994 1.51a1.35 1.35 0 001.627 0l1.994-1.51 3.282-2.484s-2.835-2.14-6.092-4.603l-6.08 4.603z"
|
||
/>
|
||
<path
|
||
fill="#FC6D26"
|
||
d="M6.435 15.305A14.712 14.712 0 00.58 12.672l-.045.113a8.09 8.09 0 002.68 9.347l.016.012.038.03 6.635 4.967 6.105-4.603-9.573-7.233z"
|
||
/>
|
||
</svg>
|
||
</span><!--]--></a><button class="navigation__mobile-menu-toggle slp-btn slp-btn-icon navigation__mobile-menu-toggle" aria-label="Main Menu" data-ga-name="open mobile menu" data-ga-location="header" type="button" data-v-c6fd7cb9 data-v-07f51b8b><!--[--><span class="slp-icon" style="color:#171321;fill:#171321;width:24px;height:24px;" role="img" aria-hidden="true" data-v-07f51b8b><svg width="16" height="16" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
|
||
<path
|
||
fill-rule="evenodd"
|
||
clip-rule="evenodd"
|
||
d="M0 3.75A.75.75 0 0 1 .75 3h14.5a.75.75 0 0 1 0 1.5H.75A.75.75 0 0 1 0 3.75ZM0 8a.75.75 0 0 1 .75-.75h14.5a.75.75 0 0 1 0 1.5H.75A.75.75 0 0 1 0 8Zm.75 3.5a.75.75 0 0 0 0 1.5h14.5a.75.75 0 0 0 0-1.5H.75Z"
|
||
/>
|
||
</svg>
|
||
</span><!--]--></button></nav><!----></header><!--]--><header role="banner" data-swiftype-index="false" class="banner" data-v-89b552f1 data-v-9f195807><a href="/events/transcend/virtual/#register" data-ga-name="transcend-october-2026-banner" data-ga-location="global-banner" class="banner--release banner" data-v-9f195807><div class="banner__desktop" data-v-9f195807><!----><div class="content" data-v-9f195807><!----><span class="slp-text-body2-bold text" data-v-64ad2128 data-v-9f195807><!--[-->Ship at agent speed. Prove every step. Transcend returns on October 6.<!--]--><!----></span><span class="banner__cta" data-v-9f195807><span class="banner__cta-text" data-v-9f195807>Register now</span><span class="icon" data-v-9f195807><span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-9f195807><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M10.159 10.72a.75.75 0 101.06 1.06l3.25-3.25L15 8l-.53-.53-3.25-3.25a.75.75 0 00-1.061 1.06l1.97 1.97H1.75a.75.75 0 100 1.5h10.379l-1.97 1.97z"/></svg></span></span></span></div></div><!----></a></header><!--[--><div id="page-blog-how-we-diagnosed-and-resolved-redis-latency-spikes"><!--[--><div class="slp-container container" data-v-e594efe8 data-v-43df3cc1><!--[--><div class="wrapper" data-v-43df3cc1><div class="hero" data-v-43df3cc1><nav aria-label="Breadcrumbs" data-v-43df3cc1 data-v-eccd9ac8><div class="slp-container breadcrumb" data-v-e594efe8 data-v-eccd9ac8><!--[--><!--[--><div class="breadcrumb__crumb hide-in-mobile" data-v-eccd9ac8><div data-v-eccd9ac8><span class="breadcrumb__chevron mobile-chevron" data-v-eccd9ac8><span class="slp-icon slp-icon--size-xs slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-eccd9ac8><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M10.78 2.22a.75.75 0 00-1.06 0L4.468 7.472a.75.75 0 000 1.06l5.252 5.252a.75.75 0 101.06-1.06L6.06 8.001l4.72-4.721a.75.75 0 000-1.06z"/></svg></span></span><a href="/blog/" data-ga-name="blog" data-ga-location="hero" data-v-eccd9ac8><span class="slp-text-body3" data-v-64ad2128 data-v-eccd9ac8><!--[-->Blog<!--]--><!----></span></a><span class="breadcrumb__chevron hide-in-mobile" data-v-eccd9ac8><span class="slp-icon slp-icon--size-xs slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-eccd9ac8><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M5.22 2.22a.75.75 0 011.06 0l5.252 5.252a.75.75 0 010 1.06L6.28 13.784a.75.75 0 11-1.06-1.06l4.72-4.723L5.22 3.28a.75.75 0 010-1.06z"/></svg></span></span></div></div><div class="breadcrumb__crumb" data-v-eccd9ac8><div data-v-eccd9ac8><span class="breadcrumb__chevron mobile-chevron" data-v-eccd9ac8><span class="slp-icon slp-icon--size-xs slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-eccd9ac8><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M10.78 2.22a.75.75 0 00-1.06 0L4.468 7.472a.75.75 0 000 1.06l5.252 5.252a.75.75 0 101.06-1.06L6.06 8.001l4.72-4.721a.75.75 0 000-1.06z"/></svg></span></span><a href="/blog/categories/engineering/" data-ga-name="engineering" data-ga-location="hero" data-v-eccd9ac8><span class="slp-text-body3" data-v-64ad2128 data-v-eccd9ac8><!--[-->Engineering<!--]--><!----></span></a><span class="breadcrumb__chevron hide-in-mobile" data-v-eccd9ac8><span class="slp-icon slp-icon--size-xs slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-eccd9ac8><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M5.22 2.22a.75.75 0 011.06 0l5.252 5.252a.75.75 0 010 1.06L6.28 13.784a.75.75 0 11-1.06-1.06l4.72-4.723L5.22 3.28a.75.75 0 010-1.06z"/></svg></span></span></div></div><div class="breadcrumb__crumb breadcrumb__crumb--disabled hide-in-mobile" data-v-eccd9ac8><div data-v-eccd9ac8><!----><a aria-current="page" data-v-eccd9ac8><span class="slp-text-body3" data-v-64ad2128 data-v-eccd9ac8><!--[-->How we diagnosed and resolved Redis latency spikes with BPF and other tools<!--]--><!----></span></a><!----></div></div><!--]--><!--]--></div></nav><div class="date" data-v-43df3cc1><p class="slp-text-body2" data-v-64ad2128 data-v-43df3cc1><!--[-->Published on: November 28, 2022<!--]--><!----></p><div class="pill" data-v-43df3cc1><p class="slp-text-body3" data-v-64ad2128 data-v-43df3cc1><!--[-->26 min read<!--]--><!----></p></div></div><h1 class="slp-text-heading2-bold" data-v-64ad2128 data-v-43df3cc1><!--[-->How we diagnosed and resolved Redis latency spikes with BPF and other tools<!--]--><!----></h1><p class="slp-text-body1" data-v-64ad2128 data-v-43df3cc1><!--[-->How we uncovered a three-phase cycle involving two distinct saturation points and a simple fix to break that cycle.<!--]--><!----></p><img data-nuxt-img srcset="https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto,c_lfill/v1749667913/Blog/Hero%20Images/clocks.jpg 1x, https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto,c_lfill/v1749667913/Blog/Hero%20Images/clocks.jpg 2x" onerror="this.setAttribute('data-error', 1)" alt fetch-priority="high" src="https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto,c_lfill/v1749667913/Blog/Hero%20Images/clocks.jpg" data-v-43df3cc1></div><div class="information" data-v-43df3cc1><!--[--><div class="author" data-v-43df3cc1><img data-nuxt-img srcset="https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto/v1749682529/Blog/Author%20Headshots/msmiley-headshot.jpg 1x, https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto/v1749682529/Blog/Author%20Headshots/msmiley-headshot.jpg 2x" onerror="this.setAttribute('data-error', 1)" alt="Matt Smiley" src="https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto/v1749682529/Blog/Author%20Headshots/msmiley-headshot.jpg" data-v-43df3cc1><a href="/blog/authors/matt-smiley/" data-ga-name="Matt Smiley link" data-ga-location="hero" data-v-43df3cc1><span class="slp-text-body1" data-v-64ad2128 data-v-43df3cc1><!--[-->Matt Smiley<!--]--><!----></span></a></div><!--]--><div class="tags" data-v-43df3cc1><!--[--><a class="pill" href="/blog/tags/performance/" data-ga-name="performance" data-ga-location="hero" data-v-43df3cc1><p class="slp-text-body3" data-v-64ad2128 data-v-43df3cc1><!--[-->performance<!--]--><!----></p></a><a class="pill" href="/blog/tags/tutorial/" data-ga-name="tutorial" data-ga-location="hero" data-v-43df3cc1><p class="slp-text-body3" data-v-64ad2128 data-v-43df3cc1><!--[-->tutorial<!--]--><!----></p></a><a class="pill" href="/blog/tags/devops/" data-ga-name="devops" data-ga-location="hero" data-v-43df3cc1><p class="slp-text-body3" data-v-64ad2128 data-v-43df3cc1><!--[-->DevOps<!--]--><!----></p></a><!--]--></div></div></div><!--]--></div><div class="slp-container" data-v-e594efe8><!--[--><div class="wrapper" data-v-612f8e63><div class="content" data-v-612f8e63><!--[--><div class="body" data-v-9c71bc40><div data-v-9c71bc40><p><!--[-->If you enjoy performance engineering and peeling back abstraction layers to ask underlying subsystems to explain themselves, this article’s for you. The context is a chronic Redis latency problem, and you are about to tour a practical example of using BPF and profiling tools in concert with standard metrics to reveal unintuitive behaviors of a complex system.<!--]--></p><p><!--[-->Beyond the tools and techniques, we also use an iterative hypothesis-testing approach to compose a behavior model of the system dynamics. This model tells us what factors influence the problem's severity and triggering conditions.<!--]--></p><p><!--[-->Ultimately, we find the root cause, and its remedy is delightfully boring and effective. We uncover a three-phase cycle involving two distinct saturation points and a simple fix to break that cycle. Along the way, we inspect aspects of the system’s behavior using stack sampling profiles, heat maps and flamegraphs, experimental tuning, source and binary analysis, instruction-level BPF instrumentation, and targeted latency injection under specific entry and exit conditions.<!--]--></p><p><!--[-->If you are short on time, the takeaways are summarized at the end. But the journey is the fun part, so let's dig in!<!--]--></p><h2 id="introducing-the-problem-chronic-latency" class=""><a class="header-anchor" href="#introducing-the-problem-chronic-latency" data-ga-name="header-anchor" data-ga-location="prose-heading"><svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path d="M12.2426 3.75736C11.4615 2.97631 10.1952 2.97631 9.41416 3.75736L7.99995 5.17157C7.60942 5.56209 6.97626 5.56209 6.58573 5.17157C6.19521 4.78105 6.19521 4.14788 6.58573 3.75736L7.99995 2.34314C9.56205 0.781046 12.0947 0.781046 13.6568 2.34314C15.2189 3.90524 15.2189 6.4379 13.6568 8L12.2426 9.41421C11.8521 9.80473 11.2189 9.80473 10.8284 9.41421C10.4379 9.02369 10.4379 8.39052 10.8284 8L12.2426 6.58578C13.0236 5.80473 13.0236 4.5384 12.2426 3.75736Z" fill="#333333"></path><path d="M10.5355 5.4645C10.926 5.85502 10.926 6.48819 10.5355 6.87871L6.87863 10.5356C6.4881 10.9261 5.85494 10.9261 5.46441 10.5356C5.07389 10.145 5.07389 9.51188 5.46441 9.12135L9.12127 5.4645C9.51179 5.07397 10.145 5.07397 10.5355 5.4645Z" fill="#333333"></path><path d="M3.75742 9.41422C2.97637 10.1953 2.97637 11.4616 3.75742 12.2426C4.53847 13.0237 5.8048 13.0237 6.58584 12.2426L8.00006 10.8284C8.39058 10.4379 9.02375 10.4379 9.41427 10.8284C9.8048 11.219 9.8048 11.8521 9.41427 12.2426L8.00006 13.6569C6.43796 15.219 3.9053 15.219 2.3432 13.6569C0.781107 12.0948 0.781107 9.56211 2.3432 8.00001L3.75742 6.5858C4.14794 6.19527 4.78111 6.19527 5.17163 6.5858C5.56216 6.97632 5.56215 7.60948 5.17163 8.00001L3.75742 9.41422Z" fill="#333333"></path></svg></a><!--[--><!--[-->Introducing the problem: Chronic latency<!--]--><!--]--></h2><p><!--[-->GitLab makes extensive use of Redis, and, on GitLab.com SaaS, we use separate Redis clusters for certain functions. This tale concerns a Redis instance acting exclusively as a least recently used (LRU) cache.<!--]--></p><p><!--[-->This cache had a chronic latency problem that started occurring intermittently over two years ago and in recent months had become significantly worse: Every few minutes, it suffered from bursts of very high latency and corresponding throughput drop, eating into its Service Level Objective (SLO). These latency spikes impacted user-facing response times and <a href="https://gitlab.com/gitlab-org/gitlab/-/issues/360578#note_966597336" rel><!--[-->burned error budgets<!--]--></a> for dependent features, and this is what we aimed to solve.<!--]--></p><p><!--[--><strong><!--[-->Graph:<!--]--></strong> Spikes in the rate of extremely slow (1 second) Redis requests, each corresponding to an eviction burst<!--]--></p><p><!--[--><img data-nuxt-img srcset="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397736/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/00_redis_slow_request_rate_spikes_during_each_eviction_burst.png 1x, https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397736/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/00_redis_slow_request_rate_spikes_during_each_eviction_burst.png 2x" onerror="this.setAttribute('data-error', 1)" alt="Graph showing spikes in the slow request rate every few minutes" loading="lazy" src="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397736/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/00_redis_slow_request_rate_spikes_during_each_eviction_burst.png" data-v-216e27f8><!--]--></p><p><!--[-->In prior work, we had already completed several mitigating optimizations. These sufficed for a while, but organic growth had resurfaced this as an important <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#why-is-it-important-to-get-to-the-root-of-the-latency-spikes" rel><!--[-->long-term scaling problem<!--]--></a>. We had also already ruled out externally triggered causes, such as request floods, connection rate spikes, host-level resource contention, etc. These latency spikes were consistently associated with memory usage reaching the eviction threshold (<code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code>), not by changes in client traffic patterns or other processes competing with Redis for CPU time, memory bandwidth, or network I/O.<!--]--></p><p><!--[-->We <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1567" rel><!--[-->initially thought<!--]--></a> that Redis 6.2’s new <a href="https://github.com/redis/redis/pull/7653" rel><!--[-->eviction throttling mechanism<!--]--></a> might alleviate our eviction burst overhead. It did not. That mechanism solves a different problem: It prevents a stall condition where a single call to <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code> could run arbitrarily long. In contrast, during this analysis we <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_977816216" rel><!--[-->discovered<!--]--></a> that our problem (both before and after upgrading Redis) was related to numerous calls collectively reducing Redis throughput, rather than a few extremely slow calls causing a complete stall.<!--]--></p><p><!--[-->To discover our bottleneck and its potential solutions, we needed to investigate Redis’s behavior during our workload’s eviction bursts.<!--]--></p><h2 id="a-little-background-on-redis-evictions" class=""><a class="header-anchor" href="#a-little-background-on-redis-evictions" data-ga-name="header-anchor" data-ga-location="prose-heading"><svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path d="M12.2426 3.75736C11.4615 2.97631 10.1952 2.97631 9.41416 3.75736L7.99995 5.17157C7.60942 5.56209 6.97626 5.56209 6.58573 5.17157C6.19521 4.78105 6.19521 4.14788 6.58573 3.75736L7.99995 2.34314C9.56205 0.781046 12.0947 0.781046 13.6568 2.34314C15.2189 3.90524 15.2189 6.4379 13.6568 8L12.2426 9.41421C11.8521 9.80473 11.2189 9.80473 10.8284 9.41421C10.4379 9.02369 10.4379 8.39052 10.8284 8L12.2426 6.58578C13.0236 5.80473 13.0236 4.5384 12.2426 3.75736Z" fill="#333333"></path><path d="M10.5355 5.4645C10.926 5.85502 10.926 6.48819 10.5355 6.87871L6.87863 10.5356C6.4881 10.9261 5.85494 10.9261 5.46441 10.5356C5.07389 10.145 5.07389 9.51188 5.46441 9.12135L9.12127 5.4645C9.51179 5.07397 10.145 5.07397 10.5355 5.4645Z" fill="#333333"></path><path d="M3.75742 9.41422C2.97637 10.1953 2.97637 11.4616 3.75742 12.2426C4.53847 13.0237 5.8048 13.0237 6.58584 12.2426L8.00006 10.8284C8.39058 10.4379 9.02375 10.4379 9.41427 10.8284C9.8048 11.219 9.8048 11.8521 9.41427 12.2426L8.00006 13.6569C6.43796 15.219 3.9053 15.219 2.3432 13.6569C0.781107 12.0948 0.781107 9.56211 2.3432 8.00001L3.75742 6.5858C4.14794 6.19527 4.78111 6.19527 5.17163 6.5858C5.56216 6.97632 5.56215 7.60948 5.17163 8.00001L3.75742 9.41422Z" fill="#333333"></path></svg></a><!--[--><!--[-->A little background on Redis evictions<!--]--><!--]--></h2><p><!--[-->At the time, our cache was oversubscribed, trying to hold more cache keys than the <a href="https://redis.io/docs/reference/eviction/" rel><!--[-->configured <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code> threshold<!--]--></a> could hold, so evictions from the LRU cache were expected. But the dense concentration of that eviction overhead was surprising and troubling.<!--]--></p><p><!--[-->Redis is essentially single-threaded. With a few exceptions, the “main” thread does almost all tasks serially, including handling client requests and evictions, among other things. Spending more time on X means there is less remaining time to do Y, so think about queuing behavior as the story unfolds.<!--]--></p><p><!--[-->Whenever Redis reaches its <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code> threshold, it frees memory by evicting some keys, aiming to do just enough evictions to get back under <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code>. However, contrary to expectation, the metrics for memory usage and eviction rate (shown below) indicated that instead of a continuous steady eviction rate, there were abrupt burst events that freed much more memory than expected. After each eviction burst, no evictions occurred until memory usage climbed back up to the <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code> threshold again.<!--]--></p><p><!--[--><strong><!--[-->Graph:<!--]--></strong> Redis memory usage drops by 300-500 MB during each eviction burst:<!--]--></p><p><!--[--><img data-nuxt-img srcset="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397736/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/01_redis_memory_usage_dips_during_eviction_bursts.png 1x, https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397736/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/01_redis_memory_usage_dips_during_eviction_bursts.png 2x" onerror="this.setAttribute('data-error', 1)" alt="Memory usage repeatedly rises gradually to 64 GB and then abruptly drops" loading="lazy" src="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397736/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/01_redis_memory_usage_dips_during_eviction_bursts.png" data-v-216e27f8><!--]--></p><p><!--[--><strong><!--[-->Graph:<!--]--></strong> Key eviction spikes match the timing and size of the memory usage dips shown above<!--]--></p><p><!--[--><img data-nuxt-img srcset="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397737/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/02_redis_eviction_bursts.png 1x, https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397737/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/02_redis_eviction_bursts.png 2x" onerror="this.setAttribute('data-error', 1)" alt="Eviction counter shows a large spike each time the previous graph showed a large memory usage drop" loading="lazy" src="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397737/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/02_redis_eviction_bursts.png" data-v-216e27f8><!--]--></p><p><!--[-->This apparent excess of evictions became the central mystery. Initially, we thought answering that question might reveal a way to smooth the eviction rate, spreading out the overhead and avoiding the latency spikes. Instead, we discovered that these bursts are an interaction effect that we need to avoid, but more on that later.<!--]--></p><h2 id="eviction-bursts-cause-cpu-saturation" class=""><a class="header-anchor" href="#eviction-bursts-cause-cpu-saturation" data-ga-name="header-anchor" data-ga-location="prose-heading"><svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path d="M12.2426 3.75736C11.4615 2.97631 10.1952 2.97631 9.41416 3.75736L7.99995 5.17157C7.60942 5.56209 6.97626 5.56209 6.58573 5.17157C6.19521 4.78105 6.19521 4.14788 6.58573 3.75736L7.99995 2.34314C9.56205 0.781046 12.0947 0.781046 13.6568 2.34314C15.2189 3.90524 15.2189 6.4379 13.6568 8L12.2426 9.41421C11.8521 9.80473 11.2189 9.80473 10.8284 9.41421C10.4379 9.02369 10.4379 8.39052 10.8284 8L12.2426 6.58578C13.0236 5.80473 13.0236 4.5384 12.2426 3.75736Z" fill="#333333"></path><path d="M10.5355 5.4645C10.926 5.85502 10.926 6.48819 10.5355 6.87871L6.87863 10.5356C6.4881 10.9261 5.85494 10.9261 5.46441 10.5356C5.07389 10.145 5.07389 9.51188 5.46441 9.12135L9.12127 5.4645C9.51179 5.07397 10.145 5.07397 10.5355 5.4645Z" fill="#333333"></path><path d="M3.75742 9.41422C2.97637 10.1953 2.97637 11.4616 3.75742 12.2426C4.53847 13.0237 5.8048 13.0237 6.58584 12.2426L8.00006 10.8284C8.39058 10.4379 9.02375 10.4379 9.41427 10.8284C9.8048 11.219 9.8048 11.8521 9.41427 12.2426L8.00006 13.6569C6.43796 15.219 3.9053 15.219 2.3432 13.6569C0.781107 12.0948 0.781107 9.56211 2.3432 8.00001L3.75742 6.5858C4.14794 6.19527 4.78111 6.19527 5.17163 6.5858C5.56216 6.97632 5.56215 7.60948 5.17163 8.00001L3.75742 9.41422Z" fill="#333333"></path></svg></a><!--[--><!--[-->Eviction bursts cause CPU saturation<!--]--><!--]--></h2><p><!--[-->As shown above, we had found that these latency spikes correlated perfectly with large spikes in the cache’s eviction rate, but we did not yet understand why the evictions were concentrated into bursts that last a few seconds and occur every few minutes.<!--]--></p><p><!--[-->As a first step, we wanted to verify a causal relationship between eviction bursts and latency spikes.<!--]--></p><p><!--[-->To test this, we used <a href="https://www.brendangregg.com/perf.html" rel><!--[--><code class="code" data-v-84382b13><!--[-->perf<!--]--></code><!--]--></a> to run a CPU sampling profile on the Redis main thread. Then we applied a filter to split that profile, isolating the samples where it was calling the <a href="https://github.com/redis/redis/blob/6.2.6/src/evict.c#L512" rel><!--[--><code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code> function<!--]--></a>. Using <a href="https://github.com/Netflix/flamescope" rel><!--[--><code class="code" data-v-84382b13><!--[-->flamescope<!--]--></code><!--]--></a>, we can visualize the profile’s CPU usage as a <a href="https://www.brendangregg.com/HeatMaps/subsecondoffset.html" rel><!--[-->subsecond offset heat map<!--]--></a>, where each second on the X axis is folded into a column of 20 msec buckets along the Y axis. This visualization style highlights sub-second activity patterns. Comparing these two heat maps confirmed that during an eviction burst, <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code> is starving all other main thread code paths for CPU time.<!--]--></p><p><!--[--><strong><!--[-->Graph:<!--]--></strong> Redis main thread CPU time, excluding calls to <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code><!--]--></p><p><!--[--><img data-nuxt-img srcset="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397737/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/03_heat_map_of_redis_main_thread_during_eviction_burst__excluding_performEvictions.png 1x, https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397737/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/03_heat_map_of_redis_main_thread_during_eviction_burst__excluding_performEvictions.png 2x" onerror="this.setAttribute('data-error', 1)" alt="Heat map shows one large gap and two small gaps in an otherwise uniform pattern of 70 percent to 80 percent CPU usage" loading="lazy" src="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397737/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/03_heat_map_of_redis_main_thread_during_eviction_burst__excluding_performEvictions.png" data-v-216e27f8><!--]--></p><p><!--[--><strong><!--[-->Graph:<!--]--></strong> Remainder of the same profile, showing only the calls to <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code><!--]--></p><p><!--[--><img data-nuxt-img srcset="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397737/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/04_heat_map_of_redis_main_thread_during_eviction_burst__only_performEvictions.png 1x, https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397737/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/04_heat_map_of_redis_main_thread_during_eviction_burst__only_performEvictions.png 2x" onerror="this.setAttribute('data-error', 1)" alt="This heat map shows the gaps in the previous heap map were CPU time spent performing evictions" loading="lazy" src="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397737/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/04_heat_map_of_redis_main_thread_during_eviction_burst__only_performEvictions.png" data-v-216e27f8><!--]--></p><p><!--[-->These results confirm that eviction bursts are causing CPU starvation on the main thread, which acts as a throughput bottleneck and increases Redis’s response time latency. These CPU utilization bursts typically lasted a few seconds, so they were too short-lived to trigger alerts but were still user impacting.<!--]--></p><p><!--[-->For context, the following flamegraph shows where <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code> spends its CPU time. There are a few interesting things here, but the most important takeaways are:<!--]--></p><ul><!--[--><li><!--[-->It gets called synchronously by <code class="code" data-v-84382b13><!--[-->processCommand<!--]--></code> (which handles all client requests).<!--]--></li><li><!--[-->It handles many of its own deletes. Despite its name, the <code class="code" data-v-84382b13><!--[-->dbAsyncDelete<!--]--></code> function only delegates deletes to a helper thread under certain conditions which turn out to be rare for this workload.<!--]--></li><!--]--></ul><p><!--[--><img data-nuxt-img srcset="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397750/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/05_flamegraph_of_redis_main_thread_during_eviction_burst__only_performEvictions.png 1x, https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397750/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/05_flamegraph_of_redis_main_thread_during_eviction_burst__only_performEvictions.png 2x" onerror="this.setAttribute('data-error', 1)" alt="Flamegraph of calls to function performEvictions, as described above" loading="lazy" src="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397750/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/05_flamegraph_of_redis_main_thread_during_eviction_burst__only_performEvictions.png" data-v-216e27f8><!--]--></p><p><!--[-->For more details on this analysis, see the <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_854745083" rel><!--[-->walkthrough and methodology<!--]--></a>.<!--]--></p><h2 id="how-fast-are-individual-calls-to-performevictions" class=""><a class="header-anchor" href="#how-fast-are-individual-calls-to-performevictions" data-ga-name="header-anchor" data-ga-location="prose-heading"><svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path d="M12.2426 3.75736C11.4615 2.97631 10.1952 2.97631 9.41416 3.75736L7.99995 5.17157C7.60942 5.56209 6.97626 5.56209 6.58573 5.17157C6.19521 4.78105 6.19521 4.14788 6.58573 3.75736L7.99995 2.34314C9.56205 0.781046 12.0947 0.781046 13.6568 2.34314C15.2189 3.90524 15.2189 6.4379 13.6568 8L12.2426 9.41421C11.8521 9.80473 11.2189 9.80473 10.8284 9.41421C10.4379 9.02369 10.4379 8.39052 10.8284 8L12.2426 6.58578C13.0236 5.80473 13.0236 4.5384 12.2426 3.75736Z" fill="#333333"></path><path d="M10.5355 5.4645C10.926 5.85502 10.926 6.48819 10.5355 6.87871L6.87863 10.5356C6.4881 10.9261 5.85494 10.9261 5.46441 10.5356C5.07389 10.145 5.07389 9.51188 5.46441 9.12135L9.12127 5.4645C9.51179 5.07397 10.145 5.07397 10.5355 5.4645Z" fill="#333333"></path><path d="M3.75742 9.41422C2.97637 10.1953 2.97637 11.4616 3.75742 12.2426C4.53847 13.0237 5.8048 13.0237 6.58584 12.2426L8.00006 10.8284C8.39058 10.4379 9.02375 10.4379 9.41427 10.8284C9.8048 11.219 9.8048 11.8521 9.41427 12.2426L8.00006 13.6569C6.43796 15.219 3.9053 15.219 2.3432 13.6569C0.781107 12.0948 0.781107 9.56211 2.3432 8.00001L3.75742 6.5858C4.14794 6.19527 4.78111 6.19527 5.17163 6.5858C5.56216 6.97632 5.56215 7.60948 5.17163 8.00001L3.75742 9.41422Z" fill="#333333"></path></svg></a><!--[--><!--[-->How fast are individual calls to <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code>?<!--]--><!--]--></h2><p><!--[-->Each incoming request to Redis is handled by a call to <code class="code" data-v-84382b13><!--[-->processCommand<!--]--></code>, and it always concludes by calling the <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code> function. That call to <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code> is frequently a no-op, returning immediately after checking that the <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code> threshold has not been breached. But when the threshold is exceeded, it will continue evicting keys until it either reaches its <code class="code" data-v-84382b13><!--[-->mem_tofree<!--]--></code> goal or exceeds its configured time limit per call.<!--]--></p><p><!--[-->The CPU heat maps shown earlier proved that <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code> calls were collectively consuming a large majority of CPU time for up to several seconds.<!--]--></p><p><!--[-->To complement that, we also measured the wall clock time of individual calls.<!--]--></p><p><!--[-->Using the <code class="code" data-v-84382b13><!--[-->funclatency<!--]--></code> CLI tool (part of the <a href="https://github.com/iovisor/bcc" rel><!--[-->BCC suite of BPF tools<!--]--></a>), we measured call duration by instrumenting entry and exit from the <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code> function and aggregated those measurements into a <a href="https://en.wikipedia.org/wiki/Histogram" rel><!--[-->histogram<!--]--></a> at 1-second intervals. When no evictions were occurring, the calls were consistently low latency (4-7 usecs/call). This is the no-op case described above (including 2.5 usecs/call of instrumentation overhead). But during an eviction burst, the results shift to a bimodal distribution, including a combination of the fast no-op calls along with much slower calls that are actively performing evictions:<!--]--></p><div class="code-block" meta style="" data-v-588d8b59><button class="copy-btn" type="button" data-v-588d8b59>Copy</button><pre class="language-shell shiki shiki-themes github-light" data-v-588d8b59> <!--[--><code><span class="line" line="1"><span class="s7eDp">$</span><span class="sYBdl"> sudo</span><span class="sYBdl"> funclatency-bpfcc</span><span class="sYu0t"> --microseconds</span><span class="sYu0t"> --timestamp</span><span class="sYu0t"> --interval</span><span class="sYu0t"> 1</span><span class="sYu0t"> --duration</span><span class="sYu0t"> 600</span><span class="sYu0t"> --pid</span><span class="sgsFI"> $( </span><span class="s7eDp">pgrep</span><span class="sYu0t"> -o</span><span class="sYBdl"> redis-server</span><span class="sgsFI"> ) </span><span class="sYBdl">'/opt/gitlab/embedded/bin/redis-server:performEvictions'
|
||
</span></span><span class="line" line="2"><span class="sYu0t">...
|
||
</span></span><span class="line" line="3"><span class="s7eDp">23:54:03
|
||
</span></span><span class="line" line="4"><span class="s7eDp"> usecs</span><span class="sYBdl"> :</span><span class="sYBdl"> count</span><span class="sYBdl"> distribution
|
||
</span></span><span class="line" line="5"><span class="s7eDp"> 0</span><span class="sgsFI"> -</span><span class="sD7c4">></span><span class="sYu0t"> 1</span><span class="sYBdl"> :</span><span class="sYu0t"> 0</span><span class="sD7c4"> |</span><span class="sD7c4"> |
|
||
</span></span><span class="line" line="6"><span class="s7eDp"> 2</span><span class="sgsFI"> -</span><span class="sD7c4">></span><span class="sYu0t"> 3</span><span class="sYBdl"> :</span><span class="sYu0t"> 576</span><span class="sD7c4"> |************</span><span class="sD7c4"> |
|
||
</span></span><span class="line" line="7"><span class="s7eDp"> 4</span><span class="sgsFI"> -</span><span class="sD7c4">></span><span class="sYu0t"> 7</span><span class="sYBdl"> :</span><span class="sYu0t"> 1896</span><span class="sD7c4"> |****************************************|
|
||
</span></span><span class="line" line="8"><span class="s7eDp"> 8</span><span class="sgsFI"> -</span><span class="sD7c4">></span><span class="sYu0t"> 15</span><span class="sYBdl"> :</span><span class="sYu0t"> 392</span><span class="sD7c4"> |********</span><span class="sD7c4"> |
|
||
</span></span><span class="line" line="9"><span class="s7eDp"> 16</span><span class="sgsFI"> -</span><span class="sD7c4">></span><span class="sYu0t"> 31</span><span class="sYBdl"> :</span><span class="sYu0t"> 84</span><span class="sD7c4"> |*</span><span class="sD7c4"> |
|
||
</span></span><span class="line" line="10"><span class="s7eDp"> 32</span><span class="sgsFI"> -</span><span class="sD7c4">></span><span class="sYu0t"> 63</span><span class="sYBdl"> :</span><span class="sYu0t"> 62</span><span class="sD7c4"> |*</span><span class="sD7c4"> |
|
||
</span></span><span class="line" line="11"><span class="s7eDp"> 64</span><span class="sgsFI"> -</span><span class="sD7c4">></span><span class="sYu0t"> 127</span><span class="sYBdl"> :</span><span class="sYu0t"> 94</span><span class="sD7c4"> |*</span><span class="sD7c4"> |
|
||
</span></span><span class="line" line="12"><span class="s7eDp"> 128</span><span class="sgsFI"> -</span><span class="sD7c4">></span><span class="sYu0t"> 255</span><span class="sYBdl"> :</span><span class="sYu0t"> 182</span><span class="sD7c4"> |***</span><span class="sD7c4"> |
|
||
</span></span><span class="line" line="13"><span class="s7eDp"> 256</span><span class="sgsFI"> -</span><span class="sD7c4">></span><span class="sYu0t"> 511</span><span class="sYBdl"> :</span><span class="sYu0t"> 826</span><span class="sD7c4"> |*****************</span><span class="sD7c4"> |
|
||
</span></span><span class="line" line="14"><span class="s7eDp"> 512</span><span class="sgsFI"> -</span><span class="sD7c4">></span><span class="sYu0t"> 1023</span><span class="sYBdl"> :</span><span class="sYu0t"> 750</span><span class="sD7c4"> |***************</span><span class="sD7c4"> |
|
||
</span></span></code><!--]-->
|
||
</pre></div><p><!--[-->This measurement also directly confirmed and quantified the throughput drop in Redis requests handled per second: The call rate to <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code> (and hence to <code class="code" data-v-84382b13><!--[-->processCommand<!--]--></code>) dropped to 20% of its norm from before the evictions began, from 25K to 5K calls per second.<!--]--></p><p><!--[-->This has a huge impact on clients: New requests are arriving at 5x the rate they are being completed. And crucially, we will see soon that this asymmetry is what drives the eviction burst.<!--]--></p><p><!--[-->For more details on this analysis, see the <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_857869826" rel><!--[-->safety check<!--]--></a> for instrumentation overhead and the <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_857907521" rel><!--[-->results walkthrough<!--]--></a>. And for more general reference, the BPF instrumentation overhead estimate is based on these <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1383" rel><!--[-->benchmark results<!--]--></a>.<!--]--></p><h2 id="experiment-can-tuning-mitigate-eviction-driven-cpu-saturation" class=""><a class="header-anchor" href="#experiment-can-tuning-mitigate-eviction-driven-cpu-saturation" data-ga-name="header-anchor" data-ga-location="prose-heading"><svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path d="M12.2426 3.75736C11.4615 2.97631 10.1952 2.97631 9.41416 3.75736L7.99995 5.17157C7.60942 5.56209 6.97626 5.56209 6.58573 5.17157C6.19521 4.78105 6.19521 4.14788 6.58573 3.75736L7.99995 2.34314C9.56205 0.781046 12.0947 0.781046 13.6568 2.34314C15.2189 3.90524 15.2189 6.4379 13.6568 8L12.2426 9.41421C11.8521 9.80473 11.2189 9.80473 10.8284 9.41421C10.4379 9.02369 10.4379 8.39052 10.8284 8L12.2426 6.58578C13.0236 5.80473 13.0236 4.5384 12.2426 3.75736Z" fill="#333333"></path><path d="M10.5355 5.4645C10.926 5.85502 10.926 6.48819 10.5355 6.87871L6.87863 10.5356C6.4881 10.9261 5.85494 10.9261 5.46441 10.5356C5.07389 10.145 5.07389 9.51188 5.46441 9.12135L9.12127 5.4645C9.51179 5.07397 10.145 5.07397 10.5355 5.4645Z" fill="#333333"></path><path d="M3.75742 9.41422C2.97637 10.1953 2.97637 11.4616 3.75742 12.2426C4.53847 13.0237 5.8048 13.0237 6.58584 12.2426L8.00006 10.8284C8.39058 10.4379 9.02375 10.4379 9.41427 10.8284C9.8048 11.219 9.8048 11.8521 9.41427 12.2426L8.00006 13.6569C6.43796 15.219 3.9053 15.219 2.3432 13.6569C0.781107 12.0948 0.781107 9.56211 2.3432 8.00001L3.75742 6.5858C4.14794 6.19527 4.78111 6.19527 5.17163 6.5858C5.56216 6.97632 5.56215 7.60948 5.17163 8.00001L3.75742 9.41422Z" fill="#333333"></path></svg></a><!--[--><!--[-->Experiment: Can tuning mitigate eviction-driven CPU saturation?<!--]--><!--]--></h2><p><!--[-->The analyses so far had shown that evictions were severely starving the Redis main thread for CPU time. There were still important unanswered questions (which we will return to shortly), but this was already enough info to <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_859236777" rel><!--[-->suggest some experiments<!--]--></a> to test potential mitigations:<!--]--></p><ul><!--[--><li><!--[-->Can we spread out the eviction overhead so it takes longer to reach its goal but consumes a smaller percentage of the main thread’s time?<!--]--></li><li><!--[-->Are evictions freeing more memory than expected due to scheduling a lot of keys to be asynchronously deleted by the <a href="https://github.com/redis/redis/blob/6.2.6/redis.conf#L1079" rel><!--[-->lazyfree mechanism<!--]--></a>? Lazyfree is an optional feature that lets the Redis main thread <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_859236777" rel><!--[-->delegate to an async helper thread<!--]--></a> the expensive task of deleting keys that have more than 64 elements. These async evictions do not count immediately towards the eviction loop’s memory goal, so if many keys qualify for lazyfree, this could potentially drive many extra iterations of the eviction loop.<!--]--></li><!--]--></ul><p><!--[-->The <a href="https://gitlab.com/gitlab-com/gl-infra/production/-/issues/7172#note_971197943" rel><!--[-->answers<!--]--></a> to both turned out to be no:<!--]--></p><ul><!--[--><li><!--[-->Reducing <code class="code" data-v-84382b13><!--[-->maxmemory-eviction-tenacity<!--]--></code> to its minimum setting still did not make <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code> cheap enough to avoid accumulating a request backlog. It did increase response rate, but arrival rate still far exceeded it, so this was not an effective mitigation.<!--]--></li><li><!--[-->Disabling <code class="code" data-v-84382b13><!--[-->lazyfree-lazy-eviction<!--]--></code> did not prevent the eviction burst from dropping memory usage far below <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code>. Those lazyfrees represent a small percentage of reclaimed memory. This rules out one of the potential explanations for the mystery of excessive memory being freed.<!--]--></li><!--]--></ul><p><!--[-->Having ruled out two potential mitigations and one candidate hypothesis, at this point we return to the pivotal question: Why are several hundred extra megabytes of memory being freed by the end of each eviction burst?<!--]--></p><h2 id="why-do-evictions-occur-in-bursts-and-free-too-much-memory" class=""><a class="header-anchor" href="#why-do-evictions-occur-in-bursts-and-free-too-much-memory" data-ga-name="header-anchor" data-ga-location="prose-heading"><svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path d="M12.2426 3.75736C11.4615 2.97631 10.1952 2.97631 9.41416 3.75736L7.99995 5.17157C7.60942 5.56209 6.97626 5.56209 6.58573 5.17157C6.19521 4.78105 6.19521 4.14788 6.58573 3.75736L7.99995 2.34314C9.56205 0.781046 12.0947 0.781046 13.6568 2.34314C15.2189 3.90524 15.2189 6.4379 13.6568 8L12.2426 9.41421C11.8521 9.80473 11.2189 9.80473 10.8284 9.41421C10.4379 9.02369 10.4379 8.39052 10.8284 8L12.2426 6.58578C13.0236 5.80473 13.0236 4.5384 12.2426 3.75736Z" fill="#333333"></path><path d="M10.5355 5.4645C10.926 5.85502 10.926 6.48819 10.5355 6.87871L6.87863 10.5356C6.4881 10.9261 5.85494 10.9261 5.46441 10.5356C5.07389 10.145 5.07389 9.51188 5.46441 9.12135L9.12127 5.4645C9.51179 5.07397 10.145 5.07397 10.5355 5.4645Z" fill="#333333"></path><path d="M3.75742 9.41422C2.97637 10.1953 2.97637 11.4616 3.75742 12.2426C4.53847 13.0237 5.8048 13.0237 6.58584 12.2426L8.00006 10.8284C8.39058 10.4379 9.02375 10.4379 9.41427 10.8284C9.8048 11.219 9.8048 11.8521 9.41427 12.2426L8.00006 13.6569C6.43796 15.219 3.9053 15.219 2.3432 13.6569C0.781107 12.0948 0.781107 9.56211 2.3432 8.00001L3.75742 6.5858C4.14794 6.19527 4.78111 6.19527 5.17163 6.5858C5.56216 6.97632 5.56215 7.60948 5.17163 8.00001L3.75742 9.41422Z" fill="#333333"></path></svg></a><!--[--><!--[-->Why do evictions occur in bursts and free too much memory?<!--]--><!--]--></h2><p><!--[-->Each round of eviction aims to free just barely enough memory to get back under the <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code> threshold.<!--]--></p><p><!--[-->With a steady rate of demand for new memory allocations, the eviction rate should be similarly steady. The rate of arriving cache writes does appear to be steady. So why are evictions happening in dense bursts, rather than smoothly? And why do they reduce memory usage on a scale of hundreds of megabytes rather than hundreds of bytes?<!--]--></p><p><!--[-->Some potential explanations to explore:<!--]--></p><ul><!--[--><li><!--[-->Do evictions only end when a large key gets evicted, spontaneously freeing enough memory to skip evictions for a while? No, the memory usage drop is far bigger than the largest keys in the dataset.<!--]--></li><li><!--[-->Do deferred lazyfree evictions cause the eviction loop to overshoot its goal, freeing more memory than intended? No, the above experiment disproved this hypothesis.<!--]--></li><li><!--[-->Is something causing the eviction loop to sometimes calculate an unexpectedly large value for its <code class="code" data-v-84382b13><!--[-->mem_tofree<!--]--></code> goal? We explore this next. The answer is no, but checking it led to a new insight.<!--]--></li><li><!--[-->Is a feedback loop causing evictions to become somehow self-amplifying? If so, what conditions lead to entering and leaving this state? This turned out to be correct.<!--]--></li><!--]--></ul><p><!--[-->These were all plausible and testable hypotheses, and each would point towards a different solution to the eviction-driven latency problem.<!--]--></p><p><!--[-->The first two hypotheses we have already eliminated.<!--]--></p><p><!--[-->To test the next two, we built custom BPF instrumentation to peek at the calculation of <code class="code" data-v-84382b13><!--[-->mem_tofree<!--]--></code> at the start of each call to <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code>.<!--]--></p><h2 id="observing-the-mem_tofree-calculation-with-bpftrace" class=""><a class="header-anchor" href="#observing-the-mem_tofree-calculation-with-bpftrace" data-ga-name="header-anchor" data-ga-location="prose-heading"><svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path d="M12.2426 3.75736C11.4615 2.97631 10.1952 2.97631 9.41416 3.75736L7.99995 5.17157C7.60942 5.56209 6.97626 5.56209 6.58573 5.17157C6.19521 4.78105 6.19521 4.14788 6.58573 3.75736L7.99995 2.34314C9.56205 0.781046 12.0947 0.781046 13.6568 2.34314C15.2189 3.90524 15.2189 6.4379 13.6568 8L12.2426 9.41421C11.8521 9.80473 11.2189 9.80473 10.8284 9.41421C10.4379 9.02369 10.4379 8.39052 10.8284 8L12.2426 6.58578C13.0236 5.80473 13.0236 4.5384 12.2426 3.75736Z" fill="#333333"></path><path d="M10.5355 5.4645C10.926 5.85502 10.926 6.48819 10.5355 6.87871L6.87863 10.5356C6.4881 10.9261 5.85494 10.9261 5.46441 10.5356C5.07389 10.145 5.07389 9.51188 5.46441 9.12135L9.12127 5.4645C9.51179 5.07397 10.145 5.07397 10.5355 5.4645Z" fill="#333333"></path><path d="M3.75742 9.41422C2.97637 10.1953 2.97637 11.4616 3.75742 12.2426C4.53847 13.0237 5.8048 13.0237 6.58584 12.2426L8.00006 10.8284C8.39058 10.4379 9.02375 10.4379 9.41427 10.8284C9.8048 11.219 9.8048 11.8521 9.41427 12.2426L8.00006 13.6569C6.43796 15.219 3.9053 15.219 2.3432 13.6569C0.781107 12.0948 0.781107 9.56211 2.3432 8.00001L3.75742 6.5858C4.14794 6.19527 4.78111 6.19527 5.17163 6.5858C5.56216 6.97632 5.56215 7.60948 5.17163 8.00001L3.75742 9.41422Z" fill="#333333"></path></svg></a><!--[--><!--[-->Observing the <code class="code" data-v-84382b13><!--[-->mem_tofree<!--]--></code> calculation with <code class="code" data-v-84382b13><!--[-->bpftrace<!--]--></code><!--]--><!--]--></h2><p><!--[-->This part of the investigation was a personal favorite and led to a critical realization about the nature of the problem.<!--]--></p><p><!--[-->As noted above, our two remaining hypotheses were:<!--]--></p><ul><!--[--><li><!--[-->an unexpectedly large <code class="code" data-v-84382b13><!--[-->mem_tofree<!--]--></code> goal<!--]--></li><li><!--[-->a self-amplifying feedback loop<!--]--></li><!--]--></ul><p><!--[-->To differentiate between them, we used <a href="https://github.com/iovisor/bpftrace" rel><!--[--><code class="code" data-v-84382b13><!--[-->bpftrace<!--]--></code><!--]--></a> to instrument the calculation of <code class="code" data-v-84382b13><!--[-->mem_tofree<!--]--></code>, looking at its input variables and results.<!--]--></p><p><!--[-->This set of measurements directly tests the following:<!--]--></p><ul><!--[--><li><!--[-->Does each call to <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code> aim to free a small amount of memory -- perhaps roughly the size of an average cache entry? If <code class="code" data-v-84382b13><!--[-->mem_tofree<!--]--></code> ever approaches hundreds of megabytes, that would confirm the first hypothesis and reveal what part of the calculation was causing that large value. Otherwise, it rules out the first hypothesis and makes the feedback loop hypothesis more likely.<!--]--></li><li><!--[-->Does the replication buffer size significantly influence <code class="code" data-v-84382b13><!--[-->mem_tofree<!--]--></code> as a feedback mechanism? Each eviction adds to this buffer, just like normal writes do. If this buffer grows large (possibly partly due to evictions) and then abruptly shrinks (due to the peer consuming it), that would cause a spontaneous large drop in memory usage, ending evictions and instantly reducing memory usage. This is one potential way for evictions to drive a feedback loop.<!--]--></li><!--]--></ul><p><!--[-->To peek at the values of the <code class="code" data-v-84382b13><!--[-->mem_tofree<!--]--></code> calculation (<a href="https://gitlab.com/gitlab-com/gl-infra/scalability/uploads/cab2cd03231f8dd4819f77b44d768cb9/redis_snoop.getMaxmemoryState.sha_25a228b839a93a1395907a03f83e1eee448b0f14.production_thresholds.bt" rel><!--[-->script<!--]--></a>), we needed to isolate the <a href="https://github.com/redis/redis/blob/6.2.6/src/evict.c#L523" rel><!--[-->correct call from <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code><!--]--></a> to the <a href="https://github.com/redis/redis/blob/6.2.6/src/evict.c#L374-L407" rel><!--[--><code class="code" data-v-84382b13><!--[-->getMaxmemoryState<!--]--></code><!--]--></a> function and reverse engineer its assembly to find the right instruction and register to instrument for each of the source code level variables that we wanted to capture. From that data we generate histograms for each of the following variables:<!--]--></p><div class="code-block" meta data-v-588d8b59><button class="copy-btn" type="button" data-v-588d8b59>Copy</button><pre class="language-text" data-v-588d8b59> <!--[--><code>mem_reported = zmalloc_used_memory() // All used memory tracked by jemalloc
|
||
overhead = freeMemoryGetNotCountedMemory() // Replication output buffers + AOF buffer
|
||
mem_used = mem_reported - overhead // Non-exempt used memory
|
||
mem_tofree = mem_used - maxmemory // Eviction goal
|
||
</code><!--]-->
|
||
</pre></div><p><!--[--><em><!--[-->Caveat:<!--]--></em> Our <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/uploads/cab2cd03231f8dd4819f77b44d768cb9/redis_snoop.getMaxmemoryState.sha_25a228b839a93a1395907a03f83e1eee448b0f14.production_thresholds.bt" rel><!--[-->custom BPF instrumentation<!--]--></a> is specific to this particular build of the <code class="code" data-v-84382b13><!--[-->redis-server<!--]--></code> binary, since it attaches to virtual addresses that are likely to change the next time Redis is compiled. But the approach is able to be generalized. Treat this as a concrete example of using BPF to inspect source code variables in the middle of a function call without having to rebuild the binary. Because we are peeking at the function’s intermediate state and because the compiler inlined this function call, we needed to do binary analysis to find the correct instrumentation points. In general, peeking at a function’s arguments or return value is easier and more portable, but in this case it would not suffice.<!--]--></p><p><!--[-->The results:<!--]--></p><ul><!--[--><li><!--[-->Ruled out the first hypothesis: Each call to <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code> had a small target value (<code class="code" data-v-84382b13><!--[-->mem_tofree<!--]--></code> < 2 MB). This means each call to <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code> did a small amount of work. Redis’s mysterious rapid drop in memory usage cannot have been caused by an abnormally large <code class="code" data-v-84382b13><!--[-->mem_tofree<!--]--></code> target evicting a big batch of keys all at once. Instead, there must be many calls collectively driving down memory usage.<!--]--></li><li><!--[-->The replication output buffers remained consistently small, ruling out one of the potential feedback loop mechanisms.<!--]--></li><li><!--[-->Surprisingly, <code class="code" data-v-84382b13><!--[-->mem_tofree<!--]--></code> was usually 16 KB to 64 KB, which is larger than a typical cache entry. This size discrepancy hints that cache keys may not be the main source of the memory pressure perpetuating the eviction burst once it begins.<!--]--></li><!--]--></ul><p><!--[-->All of the above results were consistent with the feedback loop hypothesis.<!--]--></p><p><!--[-->In addition to answering the initial questions, we got a bonus outcome: Concurrently measuring both <code class="code" data-v-84382b13><!--[-->mem_tofree<!--]--></code> and <code class="code" data-v-84382b13><!--[-->mem_used<!--]--></code> revealed a crucial new fact – <em><!--[-->the memory reclaim is a completely distinct phase from the eviction burst<!--]--></em>.<!--]--></p><p><!--[-->Reframing the pathology as exhibiting separate phases for evictions versus memory reclaim led to a series of realizations, described in the next section. From that emerged a coherent hypothesis explaining all the observed properties of the pathology.<!--]--></p><p><!--[-->For more details on this analysis, see <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_982498636" rel><!--[-->methodology notes<!--]--></a>, <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_982499538" rel><!--[-->build notes<!--]--></a> supporting the disassembly of the Redis binary, and <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_977994182" rel><!--[-->initial interpretations<!--]--></a>.<!--]--></p><h2 id="three-phase-cycle" class=""><a class="header-anchor" href="#three-phase-cycle" data-ga-name="header-anchor" data-ga-location="prose-heading"><svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path d="M12.2426 3.75736C11.4615 2.97631 10.1952 2.97631 9.41416 3.75736L7.99995 5.17157C7.60942 5.56209 6.97626 5.56209 6.58573 5.17157C6.19521 4.78105 6.19521 4.14788 6.58573 3.75736L7.99995 2.34314C9.56205 0.781046 12.0947 0.781046 13.6568 2.34314C15.2189 3.90524 15.2189 6.4379 13.6568 8L12.2426 9.41421C11.8521 9.80473 11.2189 9.80473 10.8284 9.41421C10.4379 9.02369 10.4379 8.39052 10.8284 8L12.2426 6.58578C13.0236 5.80473 13.0236 4.5384 12.2426 3.75736Z" fill="#333333"></path><path d="M10.5355 5.4645C10.926 5.85502 10.926 6.48819 10.5355 6.87871L6.87863 10.5356C6.4881 10.9261 5.85494 10.9261 5.46441 10.5356C5.07389 10.145 5.07389 9.51188 5.46441 9.12135L9.12127 5.4645C9.51179 5.07397 10.145 5.07397 10.5355 5.4645Z" fill="#333333"></path><path d="M3.75742 9.41422C2.97637 10.1953 2.97637 11.4616 3.75742 12.2426C4.53847 13.0237 5.8048 13.0237 6.58584 12.2426L8.00006 10.8284C8.39058 10.4379 9.02375 10.4379 9.41427 10.8284C9.8048 11.219 9.8048 11.8521 9.41427 12.2426L8.00006 13.6569C6.43796 15.219 3.9053 15.219 2.3432 13.6569C0.781107 12.0948 0.781107 9.56211 2.3432 8.00001L3.75742 6.5858C4.14794 6.19527 4.78111 6.19527 5.17163 6.5858C5.56216 6.97632 5.56215 7.60948 5.17163 8.00001L3.75742 9.41422Z" fill="#333333"></path></svg></a><!--[--><!--[-->Three-phase cycle<!--]--><!--]--></h2><p><!--[-->With the above results indicating a distinct separation between the evictions and the memory reclaim, we can now concisely characterize <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_982623949" rel><!--[-->three phases<!--]--></a> in the cycle of eviction-driven latency spikes.<!--]--></p><p><!--[--><strong><!--[-->Graph:<!--]--></strong> Diagram (not to scale) comparing memory and CPU usage to request and response rates during each of the three phases<!--]--></p><p><!--[--><img data-nuxt-img srcset="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397750/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/06_3_phase_cycle_of_eviction_bursts.png 1x, https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397750/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/06_3_phase_cycle_of_eviction_bursts.png 2x" onerror="this.setAttribute('data-error', 1)" alt="Diagram summarizes the text that follows, showing CPU and memory saturate in Phase 2 until request rate drops to match response rate, after which they recover" loading="lazy" src="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397750/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/06_3_phase_cycle_of_eviction_bursts.png" data-v-216e27f8><!--]--></p><p><!--[-->Phase 1: Not saturated (7-15 minutes)<!--]--></p><ul><!--[--><li><!--[-->Memory usage is below <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code>. No evictions occur during this phase.<!--]--></li><li><!--[-->Memory usage grows organically until reaching <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code>, which starts the next phase.<!--]--></li><!--]--></ul><p><!--[-->Phase 2: Saturated memory and CPU (6-8 seconds)<!--]--></p><ul><!--[--><li><!--[-->When memory usage reaches <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code>, evictions begin.<!--]--></li><li><!--[-->Evictions occur only during this phase, and they occur intermittently and frequently.<!--]--></li><li><!--[-->Demand for memory frequently exceeds free capacity, repeatedly pushing memory usage above <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code>. Throughout this phase, memory usage oscillates close to the <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code> threshold, evicting a small amount of memory at a time, just enough to get back under <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code>.<!--]--></li><!--]--></ul><p><!--[-->Phase 3: Rapid memory reclaim (30-60 seconds)<!--]--></p><ul><!--[--><li><!--[-->No evictions occur during this phase.<!--]--></li><li><!--[-->During this phase, something that had been holding a lot of memory starts quickly and steadily releasing it.<!--]--></li><li><!--[-->Without the overhead of running evictions, CPU time is again spent mostly on handling requests (starting with the backlog that accumulated during Phase 2).<!--]--></li><li><!--[-->Memory usage drops rapidly and steadily. By the time this phase ends, hundreds of megabytes have been freed. Afterwards, the cycle restarts with Phase 1.<!--]--></li><!--]--></ul><p><!--[-->At the transition between Phase 2 and Phase 3, evictions abruptly ended because memory usage stays below the <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code> threshold.<!--]--></p><p><!--[-->Reaching that transition point where memory pressure becomes negative signals that whatever was driving the memory demand in Phase 2 has started releasing memory faster than it is consuming it, shrinking the footprint it had accumulated during the previous phase.<!--]--></p><p><!--[-->What is this <strong><!--[-->mystery memory consumer<!--]--></strong> that bloats its demand during Phase 2 and frees it during Phase 3?<!--]--></p><h2 id="the-mystery-revealed" class=""><a class="header-anchor" href="#the-mystery-revealed" data-ga-name="header-anchor" data-ga-location="prose-heading"><svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path d="M12.2426 3.75736C11.4615 2.97631 10.1952 2.97631 9.41416 3.75736L7.99995 5.17157C7.60942 5.56209 6.97626 5.56209 6.58573 5.17157C6.19521 4.78105 6.19521 4.14788 6.58573 3.75736L7.99995 2.34314C9.56205 0.781046 12.0947 0.781046 13.6568 2.34314C15.2189 3.90524 15.2189 6.4379 13.6568 8L12.2426 9.41421C11.8521 9.80473 11.2189 9.80473 10.8284 9.41421C10.4379 9.02369 10.4379 8.39052 10.8284 8L12.2426 6.58578C13.0236 5.80473 13.0236 4.5384 12.2426 3.75736Z" fill="#333333"></path><path d="M10.5355 5.4645C10.926 5.85502 10.926 6.48819 10.5355 6.87871L6.87863 10.5356C6.4881 10.9261 5.85494 10.9261 5.46441 10.5356C5.07389 10.145 5.07389 9.51188 5.46441 9.12135L9.12127 5.4645C9.51179 5.07397 10.145 5.07397 10.5355 5.4645Z" fill="#333333"></path><path d="M3.75742 9.41422C2.97637 10.1953 2.97637 11.4616 3.75742 12.2426C4.53847 13.0237 5.8048 13.0237 6.58584 12.2426L8.00006 10.8284C8.39058 10.4379 9.02375 10.4379 9.41427 10.8284C9.8048 11.219 9.8048 11.8521 9.41427 12.2426L8.00006 13.6569C6.43796 15.219 3.9053 15.219 2.3432 13.6569C0.781107 12.0948 0.781107 9.56211 2.3432 8.00001L3.75742 6.5858C4.14794 6.19527 4.78111 6.19527 5.17163 6.5858C5.56216 6.97632 5.56215 7.60948 5.17163 8.00001L3.75742 9.41422Z" fill="#333333"></path></svg></a><!--[--><!--[-->The mystery revealed<!--]--><!--]--></h2><p><!--[--><a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_982651298" rel><!--[-->Modeling the phase transitions<!--]--></a> gave us some useful constraints that a viable hypothesis must satisfy. The mystery memory consumer must:<!--]--></p><ul><!--[--><li><!--[-->quickly bloat its footprint to hundreds of megabytes on a timescale of less than 10 seconds (the duration of Phase 2), under conditions triggered by the start of an eviction burst<!--]--></li><li><!--[-->quickly release its accumulated excess on a timescale of just tens of seconds (the duration of Phase 3), under the conditions immediately following an eviction burst<!--]--></li><!--]--></ul><p><!--[--><strong><!--[-->The answer:<!--]--></strong> The client input/output buffers meet those constraints to be the mystery memory consumer.<!--]--></p><p><!--[-->Here is how that hypothesis plays out:<!--]--></p><ul><!--[--><li><!--[-->During Phase 1 (healthy state), the Redis main thread’s CPU usage is already fairly high. At the start of Phase 2, when evictions begin, the eviction overhead saturates the main thread’s CPU capacity, quickly dropping response rate below the incoming request rate.<!--]--></li><li><!--[-->This throughput mismatch between arrivals versus responses <strong><!--[-->is itself the amplifier<!--]--></strong> that takes over driving the eviction burst. As the size of that rate gap increases, the proportion of time spent doing evictions also increases.<!--]--></li><li><!--[-->Accumulating a backlog of requests requires memory, and that backlog continues to grow until enough clients are stalled that the arrival rate drops to match the response rate. As clients stall, the arrival rate falls, and with it the memory pressure, eviction rate, and CPU overhead begin to reduce.<!--]--></li><li><!--[-->At the equilibrium point when arrival rate falls to match response rate, memory demand is satisfied and evictions stop (ending Phase 2). Without the eviction overhead, more CPU time is available to process the backlog, so response rate increases above request arrival rate. This recovery phase steadily consumes the request backlog, incrementally freeing memory as it goes (Phase 3).<!--]--></li><li><!--[-->Once the backlog is resolved, the arrival and response rates match again. CPU usage is back to its Phase 1 norm, and memory usage has temporarily dropped in proportion to the max size of Phase 2’s request backlog.<!--]--></li><!--]--></ul><p><!--[-->We confirmed this hypothesis via a <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_987049036" rel><!--[-->latency injection experiment<!--]--></a> showing that queuing alone explains the pathology. This outcome supports the conclusion that the extra memory demand originates from response rate falling below request arrival rate.<!--]--></p><h2 id="remedies-how-to-avoid-entering-the-eviction-burst-cycle" class=""><a class="header-anchor" href="#remedies-how-to-avoid-entering-the-eviction-burst-cycle" data-ga-name="header-anchor" data-ga-location="prose-heading"><svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path d="M12.2426 3.75736C11.4615 2.97631 10.1952 2.97631 9.41416 3.75736L7.99995 5.17157C7.60942 5.56209 6.97626 5.56209 6.58573 5.17157C6.19521 4.78105 6.19521 4.14788 6.58573 3.75736L7.99995 2.34314C9.56205 0.781046 12.0947 0.781046 13.6568 2.34314C15.2189 3.90524 15.2189 6.4379 13.6568 8L12.2426 9.41421C11.8521 9.80473 11.2189 9.80473 10.8284 9.41421C10.4379 9.02369 10.4379 8.39052 10.8284 8L12.2426 6.58578C13.0236 5.80473 13.0236 4.5384 12.2426 3.75736Z" fill="#333333"></path><path d="M10.5355 5.4645C10.926 5.85502 10.926 6.48819 10.5355 6.87871L6.87863 10.5356C6.4881 10.9261 5.85494 10.9261 5.46441 10.5356C5.07389 10.145 5.07389 9.51188 5.46441 9.12135L9.12127 5.4645C9.51179 5.07397 10.145 5.07397 10.5355 5.4645Z" fill="#333333"></path><path d="M3.75742 9.41422C2.97637 10.1953 2.97637 11.4616 3.75742 12.2426C4.53847 13.0237 5.8048 13.0237 6.58584 12.2426L8.00006 10.8284C8.39058 10.4379 9.02375 10.4379 9.41427 10.8284C9.8048 11.219 9.8048 11.8521 9.41427 12.2426L8.00006 13.6569C6.43796 15.219 3.9053 15.219 2.3432 13.6569C0.781107 12.0948 0.781107 9.56211 2.3432 8.00001L3.75742 6.5858C4.14794 6.19527 4.78111 6.19527 5.17163 6.5858C5.56216 6.97632 5.56215 7.60948 5.17163 8.00001L3.75742 9.41422Z" fill="#333333"></path></svg></a><!--[--><!--[-->Remedies: How to avoid entering the eviction burst cycle<!--]--><!--]--></h2><p><!--[-->Now that we understand the dynamics of the pathology, we can draw confident conclusions about viable solutions.<!--]--></p><p><!--[-->Redis evictions are only self-amplifying when all of the following conditions are present:<!--]--></p><ul><!--[--><li><!--[--><strong><!--[-->Memory saturation:<!--]--></strong> Memory usage reaches the <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code> limit, causing evictions to start.<!--]--></li><li><!--[--><strong><!--[-->CPU saturation:<!--]--></strong> The baseline CPU usage by the Redis main thread’s normal workload is close enough to a whole core that the eviction overhead pushes it to saturation. This reduces the response rate below request arrival rate, inducing self-amplification via increased memory demand for request buffering.<!--]--></li><li><!--[--><strong><!--[-->Many active clients:<!--]--></strong> The saturation only lasts as long as request arrival rate exceeds response rate. Stalled clients no longer contribute to that arrival rate, so the saturation lasts longer and has a greater impact if Redis has many active clients still sending requests.<!--]--></li><!--]--></ul><p><!--[-->Viable remedies include:<!--]--></p><ul><!--[--><li><!--[-->Avoid memory saturation by any combination of the following to make peak memory usage less than the <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code> limit:
|
||
<ul><!--[--><li><!--[-->Reduce cache time to live (TTL)<!--]--></li><li><!--[-->Increase <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code> (and host memory if needed, but watch out for <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1889" rel><!--[--><code class="code" data-v-84382b13><!--[-->numa_balancing<!--]--></code> CPU overhead<!--]--></a> on hosts with multiple NUMA nodes)<!--]--></li><li><!--[-->Adjust client behavior to avoid writing unnecessary cache entries<!--]--></li><li><!--[-->Split the cache among multiple instances (sharding or functional partitioning, helps avoid both memory and CPU saturation)<!--]--></li><!--]--></ul><!--]--></li><li><!--[-->Avoid CPU saturation by any combination of the following to make peak CPU usage for the workload plus eviction overhead be less than 1 CPU core:
|
||
<ul><!--[--><li><!--[-->Use the fastest processor available for single-threaded instructions per second<!--]--></li><li><!--[-->Isolate the redis-server process (particularly its main thread) from any other competing CPU-intensive processes (dedicated host, taskset, cpuset)<!--]--></li><li><!--[-->Adjust client behavior to avoid unnecessary cache lookups or writes<!--]--></li><li><!--[-->Split the cache among multiple instances (sharding or functional partitioning, helps avoid both memory and CPU saturation)<!--]--></li><li><!--[-->Offload work from the Redis main thread (io-threads, lazyfree)<!--]--></li><li><!--[-->Reduce eviction tenacity (only gives a minor benefit in our experiments)<!--]--></li><!--]--></ul><!--]--></li><!--]--></ul><p><!--[-->More exotic potential remedies could include a new Redis feature. One idea is to exempt ephemeral allocations like client buffers from counting towards the <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code> limit, instead applying that limit only to key storage. Alternatively, we could limit evictions to only consume at most a configurable percentage of the main thread’s time, so that most of its time is still spent on request throughput rather than eviction overhead.<!--]--></p><p><!--[-->Unfortunately, either of those features would trade one failure mode for another, reducing the risk of eviction-driven CPU saturation while increasing the risk of unbounded memory growth at the process level, which could potentially saturate the host or cgroup and lead to an OOM, or out of memory, kill. That trade-off may not be worthwhile, and in any case it is not currently an option.<!--]--></p><h2 id="our-solution" class=""><a class="header-anchor" href="#our-solution" data-ga-name="header-anchor" data-ga-location="prose-heading"><svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path d="M12.2426 3.75736C11.4615 2.97631 10.1952 2.97631 9.41416 3.75736L7.99995 5.17157C7.60942 5.56209 6.97626 5.56209 6.58573 5.17157C6.19521 4.78105 6.19521 4.14788 6.58573 3.75736L7.99995 2.34314C9.56205 0.781046 12.0947 0.781046 13.6568 2.34314C15.2189 3.90524 15.2189 6.4379 13.6568 8L12.2426 9.41421C11.8521 9.80473 11.2189 9.80473 10.8284 9.41421C10.4379 9.02369 10.4379 8.39052 10.8284 8L12.2426 6.58578C13.0236 5.80473 13.0236 4.5384 12.2426 3.75736Z" fill="#333333"></path><path d="M10.5355 5.4645C10.926 5.85502 10.926 6.48819 10.5355 6.87871L6.87863 10.5356C6.4881 10.9261 5.85494 10.9261 5.46441 10.5356C5.07389 10.145 5.07389 9.51188 5.46441 9.12135L9.12127 5.4645C9.51179 5.07397 10.145 5.07397 10.5355 5.4645Z" fill="#333333"></path><path d="M3.75742 9.41422C2.97637 10.1953 2.97637 11.4616 3.75742 12.2426C4.53847 13.0237 5.8048 13.0237 6.58584 12.2426L8.00006 10.8284C8.39058 10.4379 9.02375 10.4379 9.41427 10.8284C9.8048 11.219 9.8048 11.8521 9.41427 12.2426L8.00006 13.6569C6.43796 15.219 3.9053 15.219 2.3432 13.6569C0.781107 12.0948 0.781107 9.56211 2.3432 8.00001L3.75742 6.5858C4.14794 6.19527 4.78111 6.19527 5.17163 6.5858C5.56216 6.97632 5.56215 7.60948 5.17163 8.00001L3.75742 9.41422Z" fill="#333333"></path></svg></a><!--[--><!--[-->Our solution<!--]--><!--]--></h2><p><!--[-->We had already exhausted the low-hanging fruit for CPU efficiency, so we focused our attention on avoiding memory saturation.<!--]--></p><p><!--[-->To improve the cache’s memory efficiency, we <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_990891708" rel><!--[-->evaluated<!--]--></a> which types of cache keys were using the most space and how much <a href="https://redis.io/commands/object-idletime/" rel><!--[--><code class="code" data-v-84382b13><!--[-->IDLETIME<!--]--></code><!--]--></a> they had accrued since last access. This memory usage profile identified some rarely used cache entries (which waste space), helped inform the TTL, or time to live, tuning by first focusing on keys with a high idle time, and highlighted some useful potential cutpoints for functionally partitioning the cache.<!--]--></p><p><!--[-->We <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_1014582669" rel><!--[-->decided<!--]--></a> to concurrently pursue several cache efficiency improvements and opened an <a href="https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/764" rel><!--[-->epic<!--]--></a> for it. The goal was to avoid chronic memory saturation, and the main action items were:<!--]--></p><ul><!--[--><li><!--[-->Iteratively reduce the cache’s <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1854" rel><!--[-->default TTL<!--]--></a> from 2 weeks to 8 hours (helped a lot!)<!--]--></li><li><!--[-->Switch to <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_1026821730" rel><!--[-->client-side caching<!--]--></a> for certain cache keys (efficiently avoids spending shared cache space on non-shared cache entries)<!--]--></li><li><!--[--><a href="https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/762" rel><!--[-->Partition<!--]--></a> a set of cache keys to a separate Redis instance<!--]--></li><!--]--></ul><p><!--[-->The TTL reduction was the simplest solution and turned out to be a big win. One of our main concerns with TTL reduction was that the additional cache misses could potentially increase workload on other parts of the infrastructure. Some cache misses are more expensive than others, and our metrics are not granular enough to quantify the cost of cache misses per type of cache entry. This concern is why we applied the TTL adjustment incrementally and monitored for SLO violations. Fortunately, our inference was correct: Reducing TTL did not significantly reduce the cache hit rate, and the additional cache misses did not cause noticeable impact to downstream subsystems.<!--]--></p><p><!--[-->The TTL reduction turned out to be sufficient to drop memory usage consistently a little below its saturation point.<!--]--></p><p><!--[-->Increasing <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code> had initially not been feasible because the original peak memory demand (prior to the efficiency improvements) was expected to be larger than the max size of the VMs we use for Redis. However, once we dropped memory demand below saturation, then we could confidently <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1868" rel><!--[-->provision headroom<!--]--></a> for future growth and re-enable <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1883" rel><!--[-->saturation alerting<!--]--></a>.<!--]--></p><h2 id="results" class=""><a class="header-anchor" href="#results" data-ga-name="header-anchor" data-ga-location="prose-heading"><svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path d="M12.2426 3.75736C11.4615 2.97631 10.1952 2.97631 9.41416 3.75736L7.99995 5.17157C7.60942 5.56209 6.97626 5.56209 6.58573 5.17157C6.19521 4.78105 6.19521 4.14788 6.58573 3.75736L7.99995 2.34314C9.56205 0.781046 12.0947 0.781046 13.6568 2.34314C15.2189 3.90524 15.2189 6.4379 13.6568 8L12.2426 9.41421C11.8521 9.80473 11.2189 9.80473 10.8284 9.41421C10.4379 9.02369 10.4379 8.39052 10.8284 8L12.2426 6.58578C13.0236 5.80473 13.0236 4.5384 12.2426 3.75736Z" fill="#333333"></path><path d="M10.5355 5.4645C10.926 5.85502 10.926 6.48819 10.5355 6.87871L6.87863 10.5356C6.4881 10.9261 5.85494 10.9261 5.46441 10.5356C5.07389 10.145 5.07389 9.51188 5.46441 9.12135L9.12127 5.4645C9.51179 5.07397 10.145 5.07397 10.5355 5.4645Z" fill="#333333"></path><path d="M3.75742 9.41422C2.97637 10.1953 2.97637 11.4616 3.75742 12.2426C4.53847 13.0237 5.8048 13.0237 6.58584 12.2426L8.00006 10.8284C8.39058 10.4379 9.02375 10.4379 9.41427 10.8284C9.8048 11.219 9.8048 11.8521 9.41427 12.2426L8.00006 13.6569C6.43796 15.219 3.9053 15.219 2.3432 13.6569C0.781107 12.0948 0.781107 9.56211 2.3432 8.00001L3.75742 6.5858C4.14794 6.19527 4.78111 6.19527 5.17163 6.5858C5.56216 6.97632 5.56215 7.60948 5.17163 8.00001L3.75742 9.41422Z" fill="#333333"></path></svg></a><!--[--><!--[-->Results<!--]--><!--]--></h2><p><!--[-->The following graph shows Redis memory usage transitioning out of its chronically saturated state, with annotations describing the milestones when latency spikes ended and when saturation margin became wide enough to be considered safe:<!--]--></p><p><!--[--><img data-nuxt-img srcset="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397750/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/07_epic_results__memory_saturation_avoided_by_TTL_reductions.png 1x, https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397750/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/07_epic_results__memory_saturation_avoided_by_TTL_reductions.png 2x" onerror="this.setAttribute('data-error', 1)" alt="Redis memory usage stops showing a flat top saturation" loading="lazy" src="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397750/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/07_epic_results__memory_saturation_avoided_by_TTL_reductions.png" data-v-216e27f8><!--]--></p><p><!--[-->Zooming into the days when we rolled out the TTL adjustments, we can see the harmful eviction-driven latency spikes vanish as we drop memory usage below its saturation point, exactly as predicted:<!--]--></p><p><!--[--><img data-nuxt-img srcset="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397749/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/08_results__redis_memory_usage_stops_saturating.png 1x, https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397749/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/08_results__redis_memory_usage_stops_saturating.png 2x" onerror="this.setAttribute('data-error', 1)" alt="Redis memory usage starts as a flat line and then falls below that saturation line" loading="lazy" src="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397749/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/08_results__redis_memory_usage_stops_saturating.png" data-v-216e27f8><!--]--></p><p><!--[--><img data-nuxt-img srcset="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397750/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/09_results__redis_latency_spikes_end.png 1x, https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397750/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/09_results__redis_latency_spikes_end.png 2x" onerror="this.setAttribute('data-error', 1)" alt="Redis response time spikes stop occurring at the exact point when memory stops being saturated" loading="lazy" src="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397750/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/09_results__redis_latency_spikes_end.png" data-v-216e27f8><!--]--></p><p><!--[-->These eviction-driven latency spikes had been the biggest cause of slowess in Redis cache.<!--]--></p><p><!--[-->Solving this source of slowness significantly improved the user experience. This 1-year lookback shows only the long-tail portion of the improvement, not even the full benefit. Each weekday had roughly 2 million Redis requests slower than 1 second, until our fix in mid-August:<!--]--></p><p><!--[--><img data-nuxt-img srcset="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397750/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/10_results__1_year_retrospective_of_slow_redis_requests_per_day.png 1x, https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397750/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/10_results__1_year_retrospective_of_slow_redis_requests_per_day.png 2x" onerror="this.setAttribute('data-error', 1)" alt="Graph of the daily count of Redis cache requests slower than 1 second, showing roughly 2 million slow requests per day on weekdays until mid-August, when the TTL adjustments were applied" loading="lazy" src="https://res.cloudinary.com/about-gitlab-com/image/upload/v1782397750/blog/Content%20Images/2022-11-28-diagnosing-redis-latency-spikes-with-bpf-and-friends/10_results__1_year_retrospective_of_slow_redis_requests_per_day.png" data-v-216e27f8><!--]--></p><h2 id="conclusions" class=""><a class="header-anchor" href="#conclusions" data-ga-name="header-anchor" data-ga-location="prose-heading"><svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path d="M12.2426 3.75736C11.4615 2.97631 10.1952 2.97631 9.41416 3.75736L7.99995 5.17157C7.60942 5.56209 6.97626 5.56209 6.58573 5.17157C6.19521 4.78105 6.19521 4.14788 6.58573 3.75736L7.99995 2.34314C9.56205 0.781046 12.0947 0.781046 13.6568 2.34314C15.2189 3.90524 15.2189 6.4379 13.6568 8L12.2426 9.41421C11.8521 9.80473 11.2189 9.80473 10.8284 9.41421C10.4379 9.02369 10.4379 8.39052 10.8284 8L12.2426 6.58578C13.0236 5.80473 13.0236 4.5384 12.2426 3.75736Z" fill="#333333"></path><path d="M10.5355 5.4645C10.926 5.85502 10.926 6.48819 10.5355 6.87871L6.87863 10.5356C6.4881 10.9261 5.85494 10.9261 5.46441 10.5356C5.07389 10.145 5.07389 9.51188 5.46441 9.12135L9.12127 5.4645C9.51179 5.07397 10.145 5.07397 10.5355 5.4645Z" fill="#333333"></path><path d="M3.75742 9.41422C2.97637 10.1953 2.97637 11.4616 3.75742 12.2426C4.53847 13.0237 5.8048 13.0237 6.58584 12.2426L8.00006 10.8284C8.39058 10.4379 9.02375 10.4379 9.41427 10.8284C9.8048 11.219 9.8048 11.8521 9.41427 12.2426L8.00006 13.6569C6.43796 15.219 3.9053 15.219 2.3432 13.6569C0.781107 12.0948 0.781107 9.56211 2.3432 8.00001L3.75742 6.5858C4.14794 6.19527 4.78111 6.19527 5.17163 6.5858C5.56216 6.97632 5.56215 7.60948 5.17163 8.00001L3.75742 9.41422Z" fill="#333333"></path></svg></a><!--[--><!--[-->Conclusions<!--]--><!--]--></h2><p><!--[-->We solved a long-standing latency problem that had been worsening as the workload grew, and we learned a lot along the way. This article focuses mostly on the Redis discoveries, since those are general behaviors that some of you may encounter in your travels. We also developed some novel tools and analytical methods and uncovered several useful environment-specific facts about our workload, infrastructure, and observability, leading to several additional improvements and proposals not mentioned above.<!--]--></p><p><!--[-->Overall, we made several efficiency improvements and broke the cycle that was driving the pathology. Memory demand now stays well below the saturation point, eliminating the latency spikes that were burning error budgets for the development teams and causing intermittent slowness for users. All stakeholders are happy, and we came away with deeper domain knowledge and sharper skills!<!--]--></p><h2 id="key-insights-summary" class=""><a class="header-anchor" href="#key-insights-summary" data-ga-name="header-anchor" data-ga-location="prose-heading"><svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path d="M12.2426 3.75736C11.4615 2.97631 10.1952 2.97631 9.41416 3.75736L7.99995 5.17157C7.60942 5.56209 6.97626 5.56209 6.58573 5.17157C6.19521 4.78105 6.19521 4.14788 6.58573 3.75736L7.99995 2.34314C9.56205 0.781046 12.0947 0.781046 13.6568 2.34314C15.2189 3.90524 15.2189 6.4379 13.6568 8L12.2426 9.41421C11.8521 9.80473 11.2189 9.80473 10.8284 9.41421C10.4379 9.02369 10.4379 8.39052 10.8284 8L12.2426 6.58578C13.0236 5.80473 13.0236 4.5384 12.2426 3.75736Z" fill="#333333"></path><path d="M10.5355 5.4645C10.926 5.85502 10.926 6.48819 10.5355 6.87871L6.87863 10.5356C6.4881 10.9261 5.85494 10.9261 5.46441 10.5356C5.07389 10.145 5.07389 9.51188 5.46441 9.12135L9.12127 5.4645C9.51179 5.07397 10.145 5.07397 10.5355 5.4645Z" fill="#333333"></path><path d="M3.75742 9.41422C2.97637 10.1953 2.97637 11.4616 3.75742 12.2426C4.53847 13.0237 5.8048 13.0237 6.58584 12.2426L8.00006 10.8284C8.39058 10.4379 9.02375 10.4379 9.41427 10.8284C9.8048 11.219 9.8048 11.8521 9.41427 12.2426L8.00006 13.6569C6.43796 15.219 3.9053 15.219 2.3432 13.6569C0.781107 12.0948 0.781107 9.56211 2.3432 8.00001L3.75742 6.5858C4.14794 6.19527 4.78111 6.19527 5.17163 6.5858C5.56216 6.97632 5.56215 7.60948 5.17163 8.00001L3.75742 9.41422Z" fill="#333333"></path></svg></a><!--[--><!--[-->Key insights summary<!--]--><!--]--></h2><p><!--[-->The following notes summarize what we learned about Redis eviction behavior (current as of version 6.2):<!--]--></p><ul><!--[--><li><!--[-->The same memory budget (<code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code>) is shared by key storage and client connection buffers. A spike in demand for client connection buffers counts towards the <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code> limit, in the same way that a spike in key inserts or key size would.<!--]--></li><li><!--[-->Redis performs evictions in the foreground on its main thread. All time spent in <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code> is time not spent handling client requests. Consequently, during an eviction burst, Redis has a lower throughput ceiling.<!--]--></li><li><!--[-->If eviction overhead saturates the main thread’s CPU, then response rate falls below request arrival rate. Redis accumulates a request backlog (which consumes memory), and clients experience this as slowness.<!--]--></li><li><!--[-->The memory used for pending requests requires more evictions, driving the eviction burst until enough clients are stalled that arrival rate falls back below response rate. At that equilibrium point, evictions stop, eviction overhead vanishes, Redis rapidly handles its request backlog, and that backlog’s memory gets freed.<!--]--></li><li><!--[-->Triggering this cycle requires all of the following:
|
||
<ul><!--[--><li><!--[-->Redis is configured with a <code class="code" data-v-84382b13><!--[-->maxmemory<!--]--></code> limit, and its memory demand exceeds that size. This memory saturation causes evictions to begin.<!--]--></li><li><!--[-->Redis main thread’s CPU utilization is high enough under its normal workload that having to also perform evictions drives it to CPU saturation. This reduces response rate below request rate, causing a growing request backlog and high latency.<!--]--></li><li><!--[-->Many active clients are connected. The duration of the eviction burst and the size of memory spent on client connection buffers increases proportionally to the number of active clients.<!--]--></li><!--]--></ul><!--]--></li><li><!--[-->Prevent this cycle by avoiding either memory or CPU saturation. In our case, avoiding memory saturation was easier (mainly by reducing cache TTL).<!--]--></li><!--]--></ul><h2 id="further-reading" class=""><a class="header-anchor" href="#further-reading" data-ga-name="header-anchor" data-ga-location="prose-heading"><svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false"><path d="M12.2426 3.75736C11.4615 2.97631 10.1952 2.97631 9.41416 3.75736L7.99995 5.17157C7.60942 5.56209 6.97626 5.56209 6.58573 5.17157C6.19521 4.78105 6.19521 4.14788 6.58573 3.75736L7.99995 2.34314C9.56205 0.781046 12.0947 0.781046 13.6568 2.34314C15.2189 3.90524 15.2189 6.4379 13.6568 8L12.2426 9.41421C11.8521 9.80473 11.2189 9.80473 10.8284 9.41421C10.4379 9.02369 10.4379 8.39052 10.8284 8L12.2426 6.58578C13.0236 5.80473 13.0236 4.5384 12.2426 3.75736Z" fill="#333333"></path><path d="M10.5355 5.4645C10.926 5.85502 10.926 6.48819 10.5355 6.87871L6.87863 10.5356C6.4881 10.9261 5.85494 10.9261 5.46441 10.5356C5.07389 10.145 5.07389 9.51188 5.46441 9.12135L9.12127 5.4645C9.51179 5.07397 10.145 5.07397 10.5355 5.4645Z" fill="#333333"></path><path d="M3.75742 9.41422C2.97637 10.1953 2.97637 11.4616 3.75742 12.2426C4.53847 13.0237 5.8048 13.0237 6.58584 12.2426L8.00006 10.8284C8.39058 10.4379 9.02375 10.4379 9.41427 10.8284C9.8048 11.219 9.8048 11.8521 9.41427 12.2426L8.00006 13.6569C6.43796 15.219 3.9053 15.219 2.3432 13.6569C0.781107 12.0948 0.781107 9.56211 2.3432 8.00001L3.75742 6.5858C4.14794 6.19527 4.78111 6.19527 5.17163 6.5858C5.56216 6.97632 5.56215 7.60948 5.17163 8.00001L3.75742 9.41422Z" fill="#333333"></path></svg></a><!--[--><!--[-->Further reading<!--]--><!--]--></h2><p><!--[-->The following lists summarize the analytical tools and methods cited in this article. These tools are all highly versatile and any of them can provide a massive level-up when working on performance engineering problems.<!--]--></p><p><!--[-->Tools:<!--]--></p><ul><!--[--><li><!--[--><a href="https://www.brendangregg.com/perf.html" rel><!--[-->perf<!--]--></a> - A Linux performance analysis multitool. In this article, we used <code class="code" data-v-84382b13><!--[-->perf<!--]--></code> as a sampling profiler, capturing periodic stack traces of the <code class="code" data-v-84382b13><!--[-->redis-server<!--]--></code> process's main thread when it is actively running on a CPU.<!--]--></li><li><!--[--><a href="https://github.com/Netflix/flamescope" rel><!--[-->Flamescope<!--]--></a> - A visualization tool for rendering a <code class="code" data-v-84382b13><!--[-->perf<!--]--></code> profile (and other formats) into an interactive subsecond heat map. This tool invites the user to explore the timeline for microbursts of activity or inactivity and render flamegraphs of those interesting timespans to explore what code paths were active.<!--]--></li><li><!--[--><a href="https://github.com/iovisor/bcc" rel><!--[-->BCC<!--]--></a> - BCC is a framework for building BPF tools, and it ships with many useful tools out of the box. In this article, we used <code class="code" data-v-84382b13><!--[-->funclatency<!--]--></code> to measure the call durations of a specific Redis function and render the results as a histogram.<!--]--></li><li><!--[--><a href="https://github.com/iovisor/bpftrace" rel><!--[-->bpftrace<!--]--></a> - Another BPF framework, ideal for answering ad-hoc questions about your system's behavior. It uses an <code class="code" data-v-84382b13><!--[-->awk<!--]--></code>-like syntax and is <a href="https://github.com/iovisor/bpftrace#readme" rel><!--[-->quick to learn<!--]--></a>. In this article, we wrote a <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/uploads/cab2cd03231f8dd4819f77b44d768cb9/redis_snoop.getMaxmemoryState.sha_25a228b839a93a1395907a03f83e1eee448b0f14.production_thresholds.bt" rel><!--[-->custom <code class="code" data-v-84382b13><!--[-->bpftrace<!--]--></code> script<!--]--></a> for observing the variables used in computing how much memory to free during each round of evictions. This script's instrumentation points are specific to our particular build of <code class="code" data-v-84382b13><!--[-->redis-server<!--]--></code>, but the <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_982498636" rel><!--[-->approach is able to be generalized<!--]--></a> and illustrates how versatile this tool can be.<!--]--></li><!--]--></ul><p><!--[-->Usage examples:<!--]--></p><ul><!--[--><li><!--[--><a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_854745083" rel><!--[-->Example<!--]--></a> - Walkthrough of using <code class="code" data-v-84382b13><!--[-->perf<!--]--></code> and <code class="code" data-v-84382b13><!--[-->flamescope<!--]--></code> to capture, filter, and visualize the stack sampling CPU profiles of the Redis main thread.<!--]--></li><li><!--[--><a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_857869826" rel><!--[-->Example<!--]--></a> - Walkthrough (including safety check) of using <code class="code" data-v-84382b13><!--[-->funclatency<!--]--></code> to measure the durations of the frequent calls to function <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code>.<!--]--></li><li><!--[--><a href="https://gitlab.com/gitlab-com/gl-infra/production/-/issues/7172#note_971197943" rel><!--[-->Example<!--]--></a> - Experiment for adjusting Redis settings <code class="code" data-v-84382b13><!--[-->lazyfree-lazy-eviction<!--]--></code> and <code class="code" data-v-84382b13><!--[-->maxmemory-eviction-tenacity<!--]--></code> and observing the results using <code class="code" data-v-84382b13><!--[-->perf<!--]--></code>, <code class="code" data-v-84382b13><!--[-->funclatency<!--]--></code>, <code class="code" data-v-84382b13><!--[-->funcslower<!--]--></code>, and the Redis metrics for eviction count and memory usage.<!--]--></li><li><!--[--><a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_982498636" rel><!--[-->Example<!--]--></a> - This is a working example (script included) of using <code class="code" data-v-84382b13><!--[-->bpftrace<!--]--></code> to observe the values of a function's variables. In this case we inspected the <code class="code" data-v-84382b13><!--[-->mem_tofree<!--]--></code> calculation at the start of <code class="code" data-v-84382b13><!--[-->performEvictions<!--]--></code>. Also, these <a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_982499538" rel><!--[-->companion notes<!--]--></a> discuss some build-specific considerations.<!--]--></li><li><!--[--><a href="https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1601#note_987049036" rel><!--[-->Example<!--]--></a> - Describes the latency injection experiment (the first of the three ideas). This experiment confirmed that memory demand increases at the predicted rate when we slow response rate to below request arrival rate, in the same way evictions do. This result confirmed the request queuing itself is the source of the memory pressure that amplifies the eviction burst once it begins.<!--]--></li><!--]--></ul><style>html pre.shiki code .s7eDp, html code.shiki .s7eDp{--shiki-default:#6F42C1}html pre.shiki code .sYBdl, html code.shiki .sYBdl{--shiki-default:#032F62}html pre.shiki code .sYu0t, html code.shiki .sYu0t{--shiki-default:#005CC5}html pre.shiki code .sgsFI, html code.shiki .sgsFI{--shiki-default:#24292E}html pre.shiki code .sD7c4, html code.shiki .sD7c4{--shiki-default:#D73A49}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}</style></div></div><!--]--></div><div class="share" data-v-612f8e63><div class="sticky-container" data-v-612f8e63><p class="slp-text-body1-bold" data-v-64ad2128 data-v-612f8e63><!--[-->Share this article<!--]--><!----></p><div class="social-icon-links" data-v-612f8e63><a href="https://twitter.com/intent/tweet?text=Check%20out%20How%20we%20diagnosed%20and%20resolved%20Redis%20latency%20spikes%20with%20BPF%20and%20other%20tools&url=https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/" class="social-link twitter-link twitter-share-button" target="_blank" data-toggle="tooltip" title="GitLab on Twitter" data-ga-name="twitter" data-ga-location="body" data-v-612f8e63><span class="slp-icon slp-icon--size-sm" style="color:#000;fill:#000;" role="img" aria-hidden="true" data-v-612f8e63><svg fill="currentColor" viewBox="0 0 16 16">
|
||
<path
|
||
fill="currentColor"
|
||
stroke="none"
|
||
d="M9.31264 6.77143L15.1379 0H13.7575L8.69942 5.87954L4.65953 0H0L6.1091 8.8909L0 15.9918H1.38049L6.72197 9.78279L10.9884 15.9918H15.6479L9.3123 6.77143H9.31264ZM7.42187 8.96923L6.80289 8.0839L1.87789 1.03921H3.99824L7.97277 6.7245L8.59175 7.60983L13.7582 14.9998H11.6378L7.42187 8.96957V8.96923Z"
|
||
/>
|
||
</svg>
|
||
</span></a><a href="https://www.linkedin.com/shareArticle?mini=true&url=https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/" class="social-link linkedin-link" target="_blank" data-toggle="tooltip" title="GitLab on LinkedIn" data-ga-name="linkedin" data-ga-location="body" data-v-612f8e63><span class="slp-icon slp-icon--size-sm" style="color:#000;fill:#000;" role="img" aria-hidden="true" data-v-612f8e63><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3 1a2 2 0 00-2 2v10a2 2 0 002 2h10a2 2 0 002-2V3a2 2 0 00-2-2H3zm1.102 4.297a1.195 1.195 0 100-2.39 1.195 1.195 0 000 2.39zm1 7.516V6.234h-2v6.579h2zM6.43 6.234h2v.881c.295-.462.943-1.084 2.148-1.084 1.438 0 2.219.953 2.219 2.766 0 .087.008.484.008.484v3.531h-2v-3.53c0-.485-.102-1.438-1.18-1.438-1.079 0-1.17 1.198-1.195 1.982v2.986h-2V6.234z"/></svg></span></a><a href="https://www.facebook.com/sharer/sharer.php?u=https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/" class="social-link facebook-link" target="_blank" data-toggle="tooltip" title="GitLab on Facebook" data-ga-name="facebook" data-ga-location="body" data-v-612f8e63><span class="slp-icon slp-icon--size-sm" style="color:#000;fill:#000;" role="img" aria-hidden="true" data-v-612f8e63><svg fill="none" viewBox="0 0 17 16">
|
||
<path
|
||
fill="currentColor"
|
||
d="M16.6479 8C16.6479 3.58172 13.0662 0 8.64795 0C4.22967 0 0.647949 3.58172 0.647949 8C0.647949 11.993 3.57342 15.3027 7.39795 15.9028V10.3125H5.3667V8H7.39795V6.2375C7.39795 4.2325 8.59232 3.125 10.4197 3.125C11.2947 3.125 12.2104 3.28125 12.2104 3.28125V5.25H11.2017C10.2079 5.25 9.89795 5.86672 9.89795 6.5V8H12.1167L11.762 10.3125H9.89795V15.9028C13.7225 15.3027 16.6479 11.993 16.6479 8Z"
|
||
/>
|
||
</svg>
|
||
</span></a><a href="https://news.ycombinator.com/submitlink?u=https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/&t=Check%20out%20How%20we%20diagnosed%20and%20resolved%20Redis%20latency%20spikes%20with%20BPF%20and%20other%20tools" class="social-link ycombinator-link" target="_blank" data-toggle="tooltip" title="GitLab on Y Combinator" data-ga-name="hacker news" data-ga-location="body" data-v-612f8e63><span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-612f8e63><svg fill="none" viewBox="0 0 17 16">
|
||
<g>
|
||
<path fill="#FB651E" d="M16.6479 0H0.647919V16H16.6479V0Z" />
|
||
<path
|
||
fill="white"
|
||
d="M8.10878 9.0465L5.36249 3.90186H6.61758L8.23305 7.15764C8.2579 7.21563 8.2869 7.2757 8.32003 7.33783C8.35317 7.39996 8.38217 7.46417 8.40702 7.53044C8.42359 7.5553 8.43602 7.57808 8.4443 7.59879C8.45259 7.6195 8.46087 7.63814 8.46915 7.65471C8.51058 7.73755 8.54786 7.81833 8.58099 7.89703C8.61413 7.97573 8.64313 8.04822 8.66798 8.1145C8.73426 7.97366 8.80675 7.82247 8.88545 7.66092C8.96415 7.49937 9.04492 7.33162 9.12777 7.15764L10.7681 3.90186H11.9362L9.16505 9.10863V12.4265H8.10878V9.0465Z"
|
||
/>
|
||
</g>
|
||
</svg>
|
||
</span></a><a href="https://www.threads.net/intent/post?text=https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/" class="social-link" data-toggle="tooltip" title="GitLab on Threads" data-ga-name="threads" data-ga-location="body" data-v-612f8e63><span class="slp-icon slp-icon--size-sm" style="color:#000;fill:#000;" role="img" aria-hidden="true" data-v-612f8e63><svg fill="none" viewBox="0 0 15 16">
|
||
<g>
|
||
<path
|
||
fill="black"
|
||
d="M7.79511 16H7.79031C5.40311 15.984 3.56791 15.1968 2.33431 13.6608C1.23831 12.2928 0.67191 10.3904 0.65271 8.0064V7.9952C0.67191 5.6096 1.23831 3.7088 2.33591 2.3408C3.56791 0.8032 5.40471 0.016 7.79031 0H7.79511H7.79991C9.63031 0.0128 11.1615 0.4832 12.3503 1.3984C13.4687 2.2592 14.2559 3.4848 14.6895 5.0432L13.3295 5.4224C12.5935 2.7824 10.7311 1.4336 7.79351 1.4128C5.85431 1.4272 4.38711 2.0368 3.43351 3.224C2.54231 4.336 2.08151 5.9424 2.06391 8C2.08151 10.0576 2.54231 11.664 3.43511 12.776C4.38871 13.9648 5.85591 14.5744 7.79511 14.5872C9.54391 14.5744 10.7007 14.1664 11.6623 13.224C12.7599 12.1488 12.7407 10.8288 12.3887 10.0256C12.1823 9.552 11.8063 9.1584 11.2991 8.8592C11.1711 9.76 10.8847 10.4896 10.4431 11.04C9.85271 11.7744 9.01591 12.176 7.95671 12.2336C7.15511 12.2768 6.38231 12.088 5.78391 11.6992C5.07511 11.24 4.66071 10.5392 4.61591 9.7232C4.57271 8.9296 4.88791 8.2 5.50231 7.6688C6.08951 7.1616 6.91511 6.864 7.89111 6.808C8.60951 6.768 9.28311 6.8 9.90391 6.9024C9.82071 6.408 9.65431 6.0144 9.40471 5.7312C9.06231 5.3408 8.53271 5.1424 7.83191 5.1376C7.82551 5.1376 7.81911 5.1376 7.81271 5.1376C7.24951 5.1376 6.48471 5.2928 5.99831 6.0176L4.82711 5.232C5.47991 4.2624 6.53911 3.728 7.81271 3.728C7.82231 3.728 7.83191 3.728 7.84151 3.728C9.97111 3.7408 11.2399 5.0448 11.3663 7.32C11.4383 7.3504 11.5103 7.3824 11.5807 7.4144C12.5743 7.8816 13.3007 8.5888 13.6831 9.4608C14.2143 10.6752 14.2639 12.6544 12.6511 14.2336C11.4175 15.44 9.92151 15.9856 7.79991 16H7.79511ZM8.46391 8.2064C8.30231 8.2064 8.13911 8.2112 7.97111 8.2208C6.74711 8.2896 5.98391 8.8512 6.02711 9.6496C6.07191 10.4864 6.99511 10.8752 7.88311 10.8272C8.69911 10.784 9.76151 10.4656 9.94071 8.3536C9.48951 8.256 8.99511 8.2064 8.46391 8.2064Z"
|
||
/>
|
||
</g>
|
||
</svg>
|
||
</span></a></div><div class="form_column form-step--0 newsletter-form" data-v-612f8e63 data-v-72af5bed><div class="form_container" data-v-72af5bed><div data-v-72af5bed><!--[--><!--]--><h2 class="slp-text-heading5-bold form_header" data-v-64ad2128 data-v-72af5bed><!--[-->Stay in the know with GitLab's monthly newsletter<!--]--><!----></h2><p class="slp-text-body1 required_label" data-v-64ad2128 data-v-72af5bed><!--[-->All fields required<!--]--><!----></p><div class="mktoForm_skeleton" aria-hidden="true" data-v-72af5bed><!--[--><div class="mktoForm_skeleton__field" data-v-72af5bed></div><div class="mktoForm_skeleton__field" data-v-72af5bed></div><div class="mktoForm_skeleton__field" data-v-72af5bed></div><!--]--></div><form id="mktoForm_1077" class="mktoForm_container" data-v-72af5bed></form><!----><!--[--><!--]--></div></div><!----><!----></div></div></div></div><div class="wrapper" data-swiftype-index="false" data-v-17b31dd6><div class="header" data-v-17b31dd6><h2 class="slp-text-heading2-bold" data-v-64ad2128 data-v-17b31dd6><!--[-->More to explore<!--]--><!----></h2><a class="header-cta slp-btn slp-btn-tertiary header-cta" data-ga-location="resources" data-ga-name="View all blog posts" href="/blog/" data-v-c6fd7cb9 data-v-17b31dd6><!--[-->View all blog posts <span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-17b31dd6><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M5.22 2.22a.75.75 0 011.06 0l5.252 5.252a.75.75 0 010 1.06L6.28 13.784a.75.75 0 11-1.06-1.06l4.72-4.723L5.22 3.28a.75.75 0 010-1.06z"/></svg></span><!--]--></a></div><div class="cards" data-v-17b31dd6><!--[--><a class="card" data-ga-location="resources" data-ga-name="Co-Create: Building GitLab with our users" href="/blog/co-create-h1-2026/" data-v-17b31dd6><div class="category" data-v-17b31dd6><span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-17b31dd6><svg fill="currentColor" viewBox="0 0 32 32">
|
||
<path d="M2 31.96V6H4V26.9L8.56 22H28V5C28 3.34 26.66 2 25 2H2V0H25C27.76 0 30 2.24 30 5V24H9.44L2 31.96Z" />
|
||
<path d="M24 14H8V6H24V14ZM10 12H22V8H10V12Z" />
|
||
<path d="M24 16H8V18H24V16Z" />
|
||
</svg>
|
||
</span><p class="slp-text-body2 category-title" data-v-64ad2128 data-v-17b31dd6><!--[-->Engineering<!--]--><!----></p></div><img data-nuxt-img srcset="https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto/v1750099203/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2820%29_2bJGC5ZP3WheoqzlLT05C5_1750099203484.png 1x, https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto/v1750099203/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2820%29_2bJGC5ZP3WheoqzlLT05C5_1750099203484.png 2x" onerror="this.setAttribute('data-error', 1)" alt src="https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto/v1750099203/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2820%29_2bJGC5ZP3WheoqzlLT05C5_1750099203484.png" data-v-17b31dd6><h3 class="slp-text-heading5-bold title" data-v-64ad2128 data-v-17b31dd6><!--[-->Co-Create: Building GitLab with our users<!--]--><!----></h3><button class="slp-btn slp-btn-tertiary" type="button" data-v-c6fd7cb9 data-v-17b31dd6><!--[-->Read the blog <span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-17b31dd6><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M5.22 2.22a.75.75 0 011.06 0l5.252 5.252a.75.75 0 010 1.06L6.28 13.784a.75.75 0 11-1.06-1.06l4.72-4.723L5.22 3.28a.75.75 0 010-1.06z"/></svg></span><!--]--></button></a><a class="card" data-ga-location="resources" data-ga-name="Git was built for humans — agents need an upgrade" href="/blog/gitlab-next-gen-scm/" data-v-17b31dd6><div class="category" data-v-17b31dd6><span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-17b31dd6><svg fill="currentColor" viewBox="0 0 32 32">
|
||
<path d="M2 31.96V6H4V26.9L8.56 22H28V5C28 3.34 26.66 2 25 2H2V0H25C27.76 0 30 2.24 30 5V24H9.44L2 31.96Z" />
|
||
<path d="M24 14H8V6H24V14ZM10 12H22V8H10V12Z" />
|
||
<path d="M24 16H8V18H24V16Z" />
|
||
</svg>
|
||
</span><p class="slp-text-body2 category-title" data-v-64ad2128 data-v-17b31dd6><!--[-->Engineering<!--]--><!----></p></div><img data-nuxt-img srcset="https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto/v1773856365/gsx2c0vqlswox3ldmq88.jpg 1x, https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto/v1773856365/gsx2c0vqlswox3ldmq88.jpg 2x" onerror="this.setAttribute('data-error', 1)" alt src="https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto/v1773856365/gsx2c0vqlswox3ldmq88.jpg" data-v-17b31dd6><h3 class="slp-text-heading5-bold title" data-v-64ad2128 data-v-17b31dd6><!--[-->Git was built for humans — agents need an upgrade<!--]--><!----></h3><button class="slp-btn slp-btn-tertiary" type="button" data-v-c6fd7cb9 data-v-17b31dd6><!--[-->Read the blog <span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-17b31dd6><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M5.22 2.22a.75.75 0 011.06 0l5.252 5.252a.75.75 0 010 1.06L6.28 13.784a.75.75 0 11-1.06-1.06l4.72-4.723L5.22 3.28a.75.75 0 010-1.06z"/></svg></span><!--]--></button></a><a class="card" data-ga-location="resources" data-ga-name="From chaos to context: Building an AI dev workflow" href="/blog/building-an-ai-dev-workflow/" data-v-17b31dd6><div class="category" data-v-17b31dd6><span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-17b31dd6><svg fill="currentColor" viewBox="0 0 32 32">
|
||
<path d="M2 31.96V6H4V26.9L8.56 22H28V5C28 3.34 26.66 2 25 2H2V0H25C27.76 0 30 2.24 30 5V24H9.44L2 31.96Z" />
|
||
<path d="M24 14H8V6H24V14ZM10 12H22V8H10V12Z" />
|
||
<path d="M24 16H8V18H24V16Z" />
|
||
</svg>
|
||
</span><p class="slp-text-body2 category-title" data-v-64ad2128 data-v-17b31dd6><!--[-->Engineering<!--]--><!----></p></div><img data-nuxt-img srcset="https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto/v1750099203/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2820%29_2bJGC5ZP3WheoqzlLT05C5_1750099203484.png 1x, https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto/v1750099203/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2820%29_2bJGC5ZP3WheoqzlLT05C5_1750099203484.png 2x" onerror="this.setAttribute('data-error', 1)" alt src="https://res.cloudinary.com/about-gitlab-com/image/upload/f_auto,q_auto/v1750099203/Blog/Hero%20Images/Blog/Hero%20Images/blog-image-template-1800x945%20%2820%29_2bJGC5ZP3WheoqzlLT05C5_1750099203484.png" data-v-17b31dd6><h3 class="slp-text-heading5-bold title" data-v-64ad2128 data-v-17b31dd6><!--[-->From chaos to context: Building an AI dev workflow<!--]--><!----></h3><button class="slp-btn slp-btn-tertiary" type="button" data-v-c6fd7cb9 data-v-17b31dd6><!--[-->Read the blog <span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-17b31dd6><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M5.22 2.22a.75.75 0 011.06 0l5.252 5.252a.75.75 0 010 1.06L6.28 13.784a.75.75 0 11-1.06-1.06l4.72-4.723L5.22 3.28a.75.75 0 010-1.06z"/></svg></span><!--]--></button></a><!--]--></div></div><div class="contact-cta-container" data-v-baf99eb1><span class="slp-icon slp-icon--size-md slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-baf99eb1><svg fill="currentColor" fill-opacity="0" viewBox="0 0 32 33">
|
||
<g clip-path="url(#clip0_7608_14965)">
|
||
<path
|
||
stroke="currentColor"
|
||
stroke-linecap="round"
|
||
stroke-linejoin="round"
|
||
stroke-width="1.27982"
|
||
d="M4.97144 17.0143C5.80529 19.2113 7.31272 21.0883 9.27801 22.3767C9.56103 22.5663 9.7927 22.823 9.95237 23.1239C10.112 23.4248 10.1947 23.7606 10.1931 24.1013V27.144C10.1914 28.521 10.7363 29.8423 11.7082 30.8178C12.68 31.7933 13.9994 32.3431 15.3763 32.3465V32.3465C16.0595 32.3469 16.736 32.2126 17.3672 31.9512C17.9984 31.6899 18.5719 31.3067 19.0548 30.8234C19.5377 30.3402 19.9206 29.7665 20.1816 29.1352C20.4425 28.5038 20.5764 27.8272 20.5756 27.144V24.1013C20.5744 23.6371 20.73 23.1861 21.0171 22.8214"
|
||
/>
|
||
<path
|
||
stroke="currentColor"
|
||
stroke-linecap="round"
|
||
stroke-linejoin="round"
|
||
stroke-width="1.27982"
|
||
d="M24.7044 7.04123C23.6472 5.41078 22.1807 4.08619 20.4515 3.19968C18.7223 2.31318 16.7908 1.89578 14.8498 1.98915C13.8133 2.03543 12.7888 2.22936 11.8071 2.56507"
|
||
/>
|
||
<path
|
||
stroke="currentColor"
|
||
stroke-linecap="round"
|
||
stroke-linejoin="round"
|
||
stroke-width="1.27982"
|
||
d="M10.1765 24.1012H20.575"
|
||
/>
|
||
<path
|
||
stroke="currentColor"
|
||
stroke-linecap="round"
|
||
stroke-linejoin="round"
|
||
stroke-width="1.27982"
|
||
d="M9.39221 26.8496H21.3585"
|
||
/>
|
||
<path
|
||
stroke="currentColor"
|
||
stroke-linecap="round"
|
||
stroke-linejoin="round"
|
||
stroke-width="1.27982"
|
||
d="M10.0903 29.5979H20.8375"
|
||
/>
|
||
<path
|
||
stroke="currentColor"
|
||
stroke-linecap="round"
|
||
stroke-linejoin="round"
|
||
stroke-width="1.27982"
|
||
d="M11.6615 13.2099C10.4499 12.1377 8.88795 11.5458 7.27009 11.5458C5.65224 11.5458 4.09028 12.1377 2.87872 13.2099"
|
||
/>
|
||
<path
|
||
stroke="currentColor"
|
||
stroke-linecap="round"
|
||
stroke-linejoin="round"
|
||
stroke-width="1.27982"
|
||
d="M7.26612 9.2461C8.33708 9.2461 9.20527 8.37792 9.20527 7.30695C9.20527 6.23599 8.33708 5.3678 7.26612 5.3678C6.19515 5.3678 5.32697 6.23599 5.32697 7.30695C5.32697 8.37792 6.19515 9.2461 7.26612 9.2461Z"
|
||
/>
|
||
<path
|
||
stroke="currentColor"
|
||
stroke-linecap="round"
|
||
stroke-linejoin="round"
|
||
stroke-width="1.27982"
|
||
d="M7.28982 14.9312C10.9618 14.9312 13.9385 11.9545 13.9385 8.28255C13.9385 4.61061 10.9618 1.63391 7.28982 1.63391C3.61787 1.63391 0.641174 4.61061 0.641174 8.28255C0.641174 11.9545 3.61787 14.9312 7.28982 14.9312Z"
|
||
/>
|
||
<path
|
||
stroke="currentColor"
|
||
stroke-linecap="round"
|
||
stroke-linejoin="round"
|
||
stroke-width="1.27982"
|
||
d="M29.076 19.2284C27.8637 18.1526 26.2991 17.5586 24.6783 17.5586C23.0575 17.5586 21.4929 18.1526 20.2805 19.2284"
|
||
/>
|
||
<path
|
||
stroke="currentColor"
|
||
stroke-linecap="round"
|
||
stroke-linejoin="round"
|
||
stroke-width="1.27982"
|
||
d="M24.6801 15.2069C25.751 15.2069 26.6192 14.3387 26.6192 13.2678C26.6192 12.1968 25.751 11.3286 24.6801 11.3286C23.6091 11.3286 22.7409 12.1968 22.7409 13.2678C22.7409 14.3387 23.6091 15.2069 24.6801 15.2069Z"
|
||
/>
|
||
<path
|
||
stroke="currentColor"
|
||
stroke-linecap="round"
|
||
stroke-linejoin="round"
|
||
stroke-width="1.27982"
|
||
d="M24.7013 20.9496C28.3733 20.9496 31.35 17.9729 31.35 14.301C31.35 10.629 28.3733 7.65234 24.7013 7.65234C21.0294 7.65234 18.0527 10.629 18.0527 14.301C18.0527 17.9729 21.0294 20.9496 24.7013 20.9496Z"
|
||
/>
|
||
</g>
|
||
<defs>
|
||
<clipPath id="clip0_7608_14965">
|
||
<rect width="31.9954" height="31.9954" fill="white" transform="translate(0 0.990845)" />
|
||
</clipPath>
|
||
</defs>
|
||
</svg>
|
||
</span><h2 class="slp-text-heading3-bold" data-v-64ad2128 data-v-baf99eb1><!--[-->We want to hear from you<!--]--><!----></h2><p class="slp-text-heading5" data-v-64ad2128 data-v-baf99eb1><!--[-->Enjoyed reading this blog post or have questions or feedback? Share your thoughts by creating a new topic in the GitLab community forum.<!--]--><!----></p><a data-ga-location="contact" data-ga-name="Share your feedback" class="slp-btn slp-btn-tertiary" href="https://forum.gitlab.com/" data-v-c6fd7cb9 data-v-baf99eb1><!--[-->Share your feedback <span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300" style="" role="img" aria-hidden="true" data-v-baf99eb1><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M5.22 2.22a.75.75 0 011.06 0l5.252 5.252a.75.75 0 010 1.06L6.28 13.784a.75.75 0 11-1.06-1.06l4.72-4.723L5.22 3.28a.75.75 0 010-1.06z"/></svg></span><!--]--></a></div><!--]--></div><!----><section class="next-steps" data-v-9396c4f8><div class="slp-container" data-v-e594efe8 data-v-9396c4f8><!--[--><div class="next-steps__container" data-v-9396c4f8><h2 class="slp-text-display1 next-steps__header" data-v-64ad2128 data-v-9396c4f8><!--[--><div data-v-9396c4f8><p><!--[-->Start building faster today<!--]--></p></div><!--]--><!----></h2><div class="slp-text-heading5 description" data-v-64ad2128 data-v-9396c4f8><!--[--><div data-v-9396c4f8><p><!--[-->See what your team can do with the intelligent orchestration platform for DevSecOps.<!--]--></p></div><!--]--><!----></div><div class="buttons" data-v-9396c4f8><a data-ga-name="free trial" class="main-button slp-btn slp-btn-primary main-button" data-ga-location="feature" href="https://gitlab.com/-/trial_registrations/new?glm_content=default-saas-trial&glm_source=about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes" data-v-c6fd7cb9 data-v-9396c4f8><!--[-->Get your free trial <!----><!--]--></a><a data-ga-name="sales" class="secondary-button slp-btn slp-btn-tertiary secondary-button" data-ga-location="feature" href="/sales/" data-v-c6fd7cb9 data-v-9396c4f8><!--[-->Contact sales<span class="slp-icon slp-icon--color-color-text-300 slp-ml-8" style="width:16px;height:16px;" role="img" aria-hidden="true" data-v-9396c4f8><svg width="16" height="16" viewBox="0 0 16 16" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M5.22 2.22a.75.75 0 011.06 0l5.252 5.252a.75.75 0 010 1.06L6.28 13.784a.75.75 0 11-1.06-1.06l4.72-4.723L5.22 3.28a.75.75 0 010-1.06z"/></svg></span><!--]--></a></div></div><!--]--></div></section><!--]--></div><!--]--><footer id="be-footer" class="be-nav-footer" data-swiftype-index="false" data-v-89b552f1 data-v-d02fbda3><div class="slp-container footer__container" data-v-e594efe8 data-v-d02fbda3><!--[--><div class="slp-row footer__navigation" data-v-adc8d02f data-v-d02fbda3><!--[--><div class="slp-col-md-2 logo" data-v-00b68f22 data-v-d02fbda3><!--[--><span data-v-d02fbda3><svg width="134" height="32" viewBox="0 0 134 32" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||
<path
|
||
d="M57.26 12.62H61.4258C60.7308 8.1943 56.8949 5.1098 51.9034 5.1098C45.997 5.1098 41.5657 9.44842 41.5657 16.7181C41.5657 23.8531 45.7979 28.2809 52.0237 28.2809C57.6127 28.2809 61.6125 24.6906 61.6125 18.8761V16.1668H52.4552V19.359H57.6645C57.5982 22.5741 55.4489 24.6118 52.0631 24.6118C48.2832 24.6118 45.6941 21.7823 45.6941 16.6725C45.6941 11.596 48.3268 8.7768 51.976 8.7768C54.6813 8.7768 56.5318 10.2299 57.26 12.62Z"
|
||
fill="white"
|
||
/>
|
||
<path
|
||
d="M64.3655 27.97H68.355V11.057H64.3655V27.97ZM66.3779 8.37879C67.6455 8.37879 68.6807 7.40867 68.6807 6.22088C68.6807 5.0331 67.6434 4.05054 66.3779 4.05054C65.1124 4.05054 64.0626 5.02066 64.0626 6.22088C64.0626 7.4211 65.0937 8.37879 66.3779 8.37879Z"
|
||
fill="white"
|
||
/>
|
||
<path
|
||
d="M80.3004 11.057H76.9582V7.00446H72.9729V11.057H70.5706V14.1436H72.9729V23.5484C72.9501 26.7304 75.2654 28.2933 78.2611 28.2063C79.0595 28.1924 79.8512 28.0582 80.6095 27.8083L79.9373 24.6989C79.5557 24.7974 79.1635 24.8496 78.7693 24.8544C77.7673 24.8544 76.9624 24.502 76.9624 22.8934V14.1436H80.3004V11.057Z"
|
||
fill="white"
|
||
/>
|
||
<path d="M83.3895 27.9699H97.42V24.5455H87.4765V5.41864H83.3895V27.9699Z" fill="white" />
|
||
<path
|
||
d="M105.474 28.312C108.129 28.312 109.716 27.0682 110.442 25.6462H110.567V27.97H114.403V16.6498C114.403 12.1805 110.756 10.8456 107.525 10.8456C103.965 10.8456 101.233 12.4314 100.351 15.5138L104.086 16.0424C104.484 14.8857 105.606 13.8948 107.559 13.8948C109.409 13.8948 110.424 14.8422 110.424 16.5047V16.571C110.424 17.7152 109.222 17.7712 106.235 18.0904C102.951 18.4428 99.8037 19.4233 99.8037 23.2333C99.8079 26.55 102.243 28.312 105.474 28.312ZM106.785 25.383C105.125 25.383 103.93 24.6222 103.93 23.1587C103.93 21.6268 105.264 20.9884 107.042 20.7355C108.09 20.5924 110.183 20.3209 110.702 19.9063V21.8984C110.708 23.7868 109.187 25.383 106.785 25.383Z"
|
||
fill="white"
|
||
/>
|
||
<path
|
||
d="M117.353 27.9699H121.276V25.3062H121.508C122.131 26.55 123.448 28.2684 126.357 28.2684C130.346 28.2684 133.333 25.1072 133.333 19.5352C133.333 13.8969 130.259 10.829 126.346 10.829C123.359 10.829 122.114 12.622 121.508 13.8451H121.342V5.41864H117.353V27.9699ZM121.265 19.5145C121.265 16.2331 122.676 14.1062 125.245 14.1062C127.9 14.1062 129.267 16.3637 129.267 19.5145C129.267 22.6653 127.877 24.9974 125.245 24.9974C122.697 24.9974 121.265 22.7959 121.265 19.5145Z"
|
||
fill="white"
|
||
/>
|
||
<path
|
||
d="M32.6927 12.6656L32.647 12.5474L28.1203 0.748383C28.0297 0.516137 27.8668 0.319172 27.6555 0.186622C27.4445 0.0517062 27.1961 -0.0128847 26.946 0.00213248C26.6957 0.0142693 26.4558 0.105532 26.2608 0.262756C26.0658 0.41998 25.9258 0.635038 25.861 0.876905L22.8114 10.2154H10.4468L7.38467 0.876905C7.31922 0.63649 7.17901 0.423073 6.98427 0.267466C6.78824 0.112294 6.54882 0.0217724 6.29909 0.00840475C6.04936 -0.00496289 5.80164 0.059483 5.59014 0.192841C5.37948 0.326054 5.21672 0.522805 5.12543 0.754603L0.600739 12.5474L0.555098 12.6656C-0.096379 14.3666 -0.176704 16.2332 0.326232 17.9838C0.829169 19.7344 1.8881 21.2742 3.34335 22.371L3.35788 22.3814L3.39937 22.4125L10.2933 27.5699L13.7039 30.1486L15.7785 31.7157C16.0217 31.9002 16.3186 32 16.6239 32C16.9292 32 17.2261 31.9002 17.4693 31.7157L19.5439 30.1486L22.9545 27.5699L29.8878 22.3876L29.9065 22.3752C31.362 21.2776 32.4209 19.737 32.9234 17.9856C33.426 16.2342 33.345 14.367 32.6927 12.6656Z"
|
||
fill="#E24329"
|
||
/>
|
||
<path
|
||
d="M32.6926 12.6656L32.647 12.5474C30.4419 12.9998 28.3641 13.9334 26.5622 15.2816L16.6228 22.7897L22.9545 27.5699L29.8878 22.3876L29.9064 22.3751C31.3619 21.2776 32.4208 19.737 32.9234 17.9856C33.426 16.2342 33.345 14.367 32.6926 12.6656Z"
|
||
fill="#FC6D26"
|
||
/>
|
||
<path
|
||
d="M10.2932 27.5699L13.7038 30.1486L15.7784 31.7157C16.0216 31.9002 16.3185 32 16.6238 32C16.9291 32 17.2261 31.9002 17.4692 31.7157L19.5438 30.1486L22.9545 27.5699L16.6228 22.7897L10.2932 27.5699Z"
|
||
fill="#FCA326"
|
||
/>
|
||
<path
|
||
d="M6.68345 15.2816C4.88252 13.9331 2.80532 12.9994 0.600739 12.5474L0.555098 12.6656C-0.096379 14.3666 -0.176704 16.2332 0.326232 17.9838C0.829169 19.7344 1.8881 21.2742 3.34335 22.371L3.35788 22.3814L3.39937 22.4124L10.2933 27.5699L16.6228 22.7897L6.68345 15.2816Z"
|
||
fill="#FC6D26"
|
||
/>
|
||
</svg>
|
||
</span><span data-v-d02fbda3>®</span><!--]--></div><div class="slp-col-md-10 content" data-v-00b68f22 data-v-d02fbda3><!--[--><h2 class="sr-only" data-v-d02fbda3>Footer links</h2><!--[--><div class="column" data-v-d02fbda3><div class="" data-v-d02fbda3><section data-v-d02fbda3><header data-v-d02fbda3><h3 class="slp-text-body1-bold column__heading" data-v-64ad2128 data-v-d02fbda3><!--[-->Pricing<!--]--><!----></h3></header><ul class="links" data-v-d02fbda3><!--[--><li class="links__item" data-v-d02fbda3><a href="/pricing/" class="link" data-ga-location="footer" data-ga-name="view plans" data-v-d02fbda3>View plans</a></li><li class="links__item" data-v-d02fbda3><a href="/pricing/premium/" class="link" data-ga-location="footer" data-ga-name="why premium" data-v-d02fbda3>Why Premium?</a></li><li class="links__item" data-v-d02fbda3><a href="/pricing/ultimate/" class="link" data-ga-location="footer" data-ga-name="why ultimate" data-v-d02fbda3>Why Ultimate?</a></li><!--]--></ul></section></div><div class="column-wrapper" data-v-d02fbda3><!--[--><div class="column" data-v-d02fbda3><section data-v-d02fbda3><header data-v-d02fbda3><h3 class="slp-text-body1-bold column__heading" data-v-64ad2128 data-v-d02fbda3><!--[-->Contact Us<!--]--><!----></h3><ul class="links" data-v-d02fbda3><!--[--><li class="links__item" data-v-d02fbda3><a href="/sales/" class="link" data-ga-location="footer" data-ga-name="sales" data-v-d02fbda3>Contact sales</a></li><li class="links__item" data-v-d02fbda3><a href="https://support.gitlab.com/hc/en-us" class="link" data-ga-location="footer" data-ga-name="support portal" data-v-d02fbda3>Support portal</a></li><li class="links__item" data-v-d02fbda3><a href="https://customers.gitlab.com/customers/sign_in/" class="link" data-ga-location="footer" data-ga-name="customer portal" data-v-d02fbda3>Customer portal</a></li><li class="links__item" data-v-d02fbda3><a href="https://status.gitlab.com/" class="link" data-ga-location="footer" data-ga-name="status" data-v-d02fbda3>Status</a></li><li class="links__item" data-v-d02fbda3><a href="/terms/" class="link" data-ga-location="footer" data-ga-name="terms of use" data-v-d02fbda3>Terms of use</a></li><li class="links__item" data-v-d02fbda3><a href="/privacy/" class="link" data-ga-location="footer" data-ga-name="privacy statement" data-v-d02fbda3>Privacy statement</a></li><li class="links__item" data-v-d02fbda3><button id="ot-sdk-btn" class="ot-sdk-show-settings" data-ga-location="footer" data-ga-name="cookie preferences" data-v-d02fbda3></button></li><!--]--></ul></header></section></div><!--]--></div></div><div class="column" data-v-d02fbda3><div class="" data-v-d02fbda3><section data-v-d02fbda3><header data-v-d02fbda3><h3 class="slp-text-body1-bold column__heading" data-v-64ad2128 data-v-d02fbda3><!--[-->Platform<!--]--><!----></h3></header><ul class="links" data-v-d02fbda3><!--[--><li class="links__item" data-v-d02fbda3><a href="/platform/" class="link" data-ga-location="footer" data-ga-name="platform" data-v-d02fbda3>Explore the platform</a></li><li class="links__item" data-v-d02fbda3><a href="/gitlab-duo-agent-platform/" class="link" data-ga-location="footer" data-ga-name="agentic orchestration" data-v-d02fbda3>Agentic Orchestration</a></li><li class="links__item" data-v-d02fbda3><a href="/solutions/continuous-integration/" class="link" data-ga-location="footer" data-ga-name="ci/cd" data-v-d02fbda3>CI/CD</a></li><li class="links__item" data-v-d02fbda3><a href="/gitlab-orbit/" class="link" data-ga-location="footer" data-ga-name="context graph" data-v-d02fbda3>Context Graph</a></li><li class="links__item" data-v-d02fbda3><a href="/why-gitlab/" class="link" data-ga-location="footer" data-ga-name="why gitlab" data-v-d02fbda3>Why GitLab</a></li><!--]--></ul></section></div><div class="column-wrapper" data-v-d02fbda3><!--[--><div class="column" data-v-d02fbda3><section data-v-d02fbda3><header data-v-d02fbda3><h3 class="slp-text-body1-bold column__heading" data-v-64ad2128 data-v-d02fbda3><!--[-->Topics<!--]--><!----></h3><ul class="links" data-v-d02fbda3><!--[--><li class="links__item" data-v-d02fbda3><a href="/topics/ci-cd/" class="link" data-ga-location="footer" data-ga-name="cicd" data-v-d02fbda3>CI/CD</a></li><li class="links__item" data-v-d02fbda3><a href="/topics/gitops/" class="link" data-ga-location="footer" data-ga-name="gitops" data-v-d02fbda3>GitOps</a></li><li class="links__item" data-v-d02fbda3><a href="/topics/devops/" class="link" data-ga-location="footer" data-ga-name="devops" data-v-d02fbda3>DevOps</a></li><li class="links__item" data-v-d02fbda3><a href="/topics/version-control/" class="link" data-ga-location="footer" data-ga-name="version control" data-v-d02fbda3>Version Control</a></li><li class="links__item" data-v-d02fbda3><a href="/topics/devsecops/" class="link" data-ga-location="footer" data-ga-name="devsecops" data-v-d02fbda3>DevSecOps</a></li><li class="links__item" data-v-d02fbda3><a href="/topics/cloud-native/" class="link" data-ga-location="footer" data-ga-name="cloud native" data-v-d02fbda3>Cloud Native</a></li><li class="links__item" data-v-d02fbda3><a href="/topics/devops/ai-for-coding/" class="link" data-ga-location="footer" data-ga-name="ai for coding" data-v-d02fbda3>AI for Coding</a></li><li class="links__item" data-v-d02fbda3><a href="/topics/agentic-ai/" class="link" data-ga-location="footer" data-ga-name="agentic ai" data-v-d02fbda3>Agentic AI</a></li><!--]--></ul></header></section></div><!--]--></div></div><div class="column column__nested" data-v-d02fbda3><div class="divided" data-v-d02fbda3><section data-v-d02fbda3><header data-v-d02fbda3><h3 class="slp-text-body1-bold column__heading" data-v-64ad2128 data-v-d02fbda3><!--[-->Solutions<!--]--><!----></h3></header><ul class="links links__divided" data-v-d02fbda3><!--[--><li class="links__item" data-v-d02fbda3><a href="/assessments/devops-modernization-assessment/" class="link" data-ga-location="footer" data-ga-name="devops modernization" data-v-d02fbda3>DevOps modernization</a></li><li class="links__item" data-v-d02fbda3><a href="/assessments/security-modernization-assessment/" class="link" data-ga-location="footer" data-ga-name="security modernization" data-v-d02fbda3>Security modernization</a></li><li class="links__item" data-v-d02fbda3><a href="/assessments/ai-modernization-assessment/" class="link" data-ga-location="footer" data-ga-name="ai modernization" data-v-d02fbda3>AI modernization</a></li><li class="links__item" data-v-d02fbda3><a href="/solutions/finance/" class="link" data-ga-location="footer" data-ga-name="financial services" data-v-d02fbda3>Financial services</a></li><li class="links__item" data-v-d02fbda3><a href="/solutions/public-sector/" class="link" data-ga-location="footer" data-ga-name="public sector" data-v-d02fbda3>Public sector</a></li><li class="links__item" data-v-d02fbda3><a href="/solutions/telecommunications/" class="link" data-ga-location="footer" data-ga-name="telecommunications" data-v-d02fbda3>Telecommunications</a></li><li class="links__item" data-v-d02fbda3><a href="/solutions/automotive/" class="link" data-ga-location="footer" data-ga-name="automotive" data-v-d02fbda3>Automotive</a></li><li class="links__item" data-v-d02fbda3><a href="/solutions/education/" class="link" data-ga-location="footer" data-ga-name="education" data-v-d02fbda3>Education</a></li><li class="links__item" data-v-d02fbda3><a href="/solutions/aerospace/" class="link" data-ga-location="footer" data-ga-name="aerospace" data-v-d02fbda3>Aerospace</a></li><li class="links__item" data-v-d02fbda3><a href="/enterprise/" class="link" data-ga-location="footer" data-ga-name="enterprise" data-v-d02fbda3>Enterprise</a></li><li class="links__item" data-v-d02fbda3><a href="/small-business/" class="link" data-ga-location="footer" data-ga-name="small business" data-v-d02fbda3>Small business</a></li><li class="links__item" data-v-d02fbda3><a href="/solutions/startups/" class="link" data-ga-location="footer" data-ga-name="startups" data-v-d02fbda3>Startups</a></li><li class="links__item" data-v-d02fbda3><a href="/solutions/gitops/" class="link" data-ga-location="footer" data-ga-name="gitops" data-v-d02fbda3>GitOps</a></li><li class="links__item" data-v-d02fbda3><a href="/solutions/supply-chain/" class="link" data-ga-location="footer" data-ga-name="software composition analysis" data-v-d02fbda3>Software composition analysis</a></li><li class="links__item" data-v-d02fbda3><a href="/solutions/value-stream-management/" class="link" data-ga-location="footer" data-ga-name="value stream management" data-v-d02fbda3>Value stream management</a></li><li class="links__item" data-v-d02fbda3><a href="/solutions/" class="link" data-ga-location="footer" data-ga-name="all solutions" data-v-d02fbda3>All solutions</a></li><!--]--></ul></section></div><!----></div><div class="column column__nested" data-v-d02fbda3><div class="divided" data-v-d02fbda3><section data-v-d02fbda3><header data-v-d02fbda3><h3 class="slp-text-body1-bold column__heading" data-v-64ad2128 data-v-d02fbda3><!--[-->Resources<!--]--><!----></h3></header><ul class="links links__divided" data-v-d02fbda3><!--[--><li class="links__item" data-v-d02fbda3><a href="/install/" class="link" data-ga-location="footer" data-ga-name="install" data-v-d02fbda3>Install</a></li><li class="links__item" data-v-d02fbda3><a href="/get-started/" class="link" data-ga-location="footer" data-ga-name="quick setup checklists" data-v-d02fbda3>Quick start guides</a></li><li class="links__item" data-v-d02fbda3><a href="https://docs.gitlab.com/" class="link" data-ga-location="footer" data-ga-name="docs" data-v-d02fbda3>Docs</a></li><li class="links__item" data-v-d02fbda3><a href="https://university.gitlab.com/" class="link" data-ga-location="footer" data-ga-name="university" data-v-d02fbda3>University</a></li><li class="links__item" data-v-d02fbda3><a href="/gitlab-technical-demo-series/" class="link" data-ga-location="footer" data-ga-name="demo series" data-v-d02fbda3>Demo Series</a></li><li class="links__item" data-v-d02fbda3><a href="/demo-hub/" class="link" data-ga-location="footer" data-ga-name="demo hub" data-v-d02fbda3>Demo Hub</a></li><li class="links__item" data-v-d02fbda3><a href="/services/" class="link" data-ga-location="footer" data-ga-name="services" data-v-d02fbda3>Services</a></li><li class="links__item" data-v-d02fbda3><a href="/blog/" class="link" data-ga-location="footer" data-ga-name="blog" data-v-d02fbda3>Blog</a></li><li class="links__item" data-v-d02fbda3><a href="/community/" class="link" data-ga-location="footer" data-ga-name="community" data-v-d02fbda3>Community</a></li><li class="links__item" data-v-d02fbda3><a href="/customers/" class="link" data-ga-location="footer" data-ga-name="customers" data-v-d02fbda3>Customers</a></li><li class="links__item" data-v-d02fbda3><a href="/partners/" class="link" data-ga-location="footer" data-ga-name="partners" data-v-d02fbda3>Partners</a></li><li class="links__item" data-v-d02fbda3><a href="/events/" class="link" data-ga-location="footer" data-ga-name="events" data-v-d02fbda3>Events</a></li><li class="links__item" data-v-d02fbda3><a href="/company/contact/#contact-forms" class="link" data-ga-location="footer" data-ga-name="newsletter" data-v-d02fbda3>Newsletter</a></li><li class="links__item" data-v-d02fbda3><a href="/whats-new/" class="link" data-ga-location="footer" data-ga-name="whats new" data-v-d02fbda3>What's new</a></li><li class="links__item" data-v-d02fbda3><a href="/resources/" class="link" data-ga-location="footer" data-ga-name="all resources" data-v-d02fbda3>All resources</a></li><!--]--></ul></section></div><!----></div><div class="column column__nested" data-v-d02fbda3><div class="divided" data-v-d02fbda3><section data-v-d02fbda3><header data-v-d02fbda3><h3 class="slp-text-body1-bold column__heading" data-v-64ad2128 data-v-d02fbda3><!--[-->Company<!--]--><!----></h3></header><ul class="links links__divided" data-v-d02fbda3><!--[--><li class="links__item" data-v-d02fbda3><a href="/company/" class="link" data-ga-location="footer" data-ga-name="company" data-v-d02fbda3>About</a></li><li class="links__item" data-v-d02fbda3><a href="/jobs/" class="link" data-ga-location="footer" data-ga-name="jobs" data-v-d02fbda3>Jobs</a></li><li class="links__item" data-v-d02fbda3><a href="/press/" class="link" data-ga-location="footer" data-ga-name="press" data-v-d02fbda3>Press</a></li><li class="links__item" data-v-d02fbda3><a href="https://handbook.gitlab.com/" class="link" data-ga-location="footer" data-ga-name="handbook" data-v-d02fbda3>Handbook</a></li><li class="links__item" data-v-d02fbda3><a href="/company/team/e-group/" class="link" data-ga-location="footer" data-ga-name="leadership" data-v-d02fbda3>Leadership</a></li><li class="links__item" data-v-d02fbda3><a href="https://ir.gitlab.com/overview/default.aspx" class="link" data-ga-location="footer" data-ga-name="investor relations" data-v-d02fbda3>Investor relations</a></li><li class="links__item" data-v-d02fbda3><a href="/security/" class="link" data-ga-location="footer" data-ga-name="trust center" data-v-d02fbda3>Trust Center</a></li><li class="links__item" data-v-d02fbda3><a href="/ai-transparency-center/" class="link" data-ga-location="footer" data-ga-name="ai transparency center" data-v-d02fbda3>AI Transparency Center</a></li><li class="links__item" data-v-d02fbda3><a href="/sustainability/" class="link" data-ga-location="footer" data-ga-name="Sustainability" data-v-d02fbda3>Sustainability</a></li><li class="links__item" data-v-d02fbda3><a href="/diversity-inclusion-belonging/" class="link" data-ga-location="footer" data-ga-name="Diversity, inclusion and belonging" data-v-d02fbda3>Diversity, inclusion and belonging (DIB)</a></li><li class="links__item" data-v-d02fbda3><a href="https://handbook.gitlab.com/handbook/legal/modern-slavery-act-transparency-statement/" class="link" data-ga-location="footer" data-ga-name="modern slavery transparency statement" data-v-d02fbda3>Modern Slavery Transparency Statement</a></li><!--]--></ul></section></div><!----></div><!--]--><!--]--></div><!--]--></div><div class="footer__cta" data-v-d02fbda3><div class="footer__selectors" data-v-d02fbda3><!----><div data-v-d02fbda3 data-v-0b1e0911><ul class="social__links" data-v-0b1e0911><li class="social__links--item" data-v-0b1e0911><a href="https://twitter.com/gitlab" data-ga-name="twitter-x" data-ga-location="footer" aria-label="Twitter" data-v-0b1e0911><span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300 social__links--icon social__links--icon-x" style="" role="img" aria-hidden="true" data-v-0b1e0911><svg fill="currentColor" viewBox="0 0 16 16">
|
||
<path
|
||
fill="currentColor"
|
||
stroke="none"
|
||
d="M9.31264 6.77143L15.1379 0H13.7575L8.69942 5.87954L4.65953 0H0L6.1091 8.8909L0 15.9918H1.38049L6.72197 9.78279L10.9884 15.9918H15.6479L9.3123 6.77143H9.31264ZM7.42187 8.96923L6.80289 8.0839L1.87789 1.03921H3.99824L7.97277 6.7245L8.59175 7.60983L13.7582 14.9998H11.6378L7.42187 8.96957V8.96923Z"
|
||
/>
|
||
</svg>
|
||
</span></a></li><li class="social__links--item" data-v-0b1e0911><a href="https://www.facebook.com/gitlab" data-ga-name="facebook" data-ga-location="footer" aria-label="Facebook" data-v-0b1e0911><span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300 social__links--icon" style="" role="img" aria-hidden="true" data-v-0b1e0911><svg fill="none" viewBox="0 0 17 16">
|
||
<path
|
||
fill="currentColor"
|
||
d="M16.6479 8C16.6479 3.58172 13.0662 0 8.64795 0C4.22967 0 0.647949 3.58172 0.647949 8C0.647949 11.993 3.57342 15.3027 7.39795 15.9028V10.3125H5.3667V8H7.39795V6.2375C7.39795 4.2325 8.59232 3.125 10.4197 3.125C11.2947 3.125 12.2104 3.28125 12.2104 3.28125V5.25H11.2017C10.2079 5.25 9.89795 5.86672 9.89795 6.5V8H12.1167L11.762 10.3125H9.89795V15.9028C13.7225 15.3027 16.6479 11.993 16.6479 8Z"
|
||
/>
|
||
</svg>
|
||
</span></a></li><li class="social__links--item" data-v-0b1e0911><a href="https://www.youtube.com/channel/UCnMGQ8QHMAnVIsI3xJrihhg" data-ga-name="youtube" data-ga-location="footer" aria-label="YouTube" data-v-0b1e0911><span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300 social__links--icon" style="" role="img" aria-hidden="true" data-v-0b1e0911><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 576 512">
|
||
<!-- Font Awesome Free 5.15.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) -->
|
||
<path
|
||
fill="currentColor"
|
||
d="M549.655 124.083c-6.281-23.65-24.787-42.276-48.284-48.597C458.781 64 288 64 288 64S117.22 64 74.629 75.486c-23.497 6.322-42.003 24.947-48.284 48.597-11.412 42.867-11.412 132.305-11.412 132.305s0 89.438 11.412 132.305c6.281 23.65 24.787 41.5 48.284 47.821C117.22 448 288 448 288 448s170.78 0 213.371-11.486c23.497-6.321 42.003-24.171 48.284-47.821 11.412-42.867 11.412-132.305 11.412-132.305s0-89.438-11.412-132.305zm-317.51 213.508V175.185l142.739 81.205-142.739 81.201z"
|
||
/>
|
||
</svg>
|
||
</span></a></li><li class="social__links--item" data-v-0b1e0911><a href="https://www.linkedin.com/company/gitlab-com" data-ga-name="linkedin" data-ga-location="footer" aria-label="LinkedIn" data-v-0b1e0911><span class="slp-icon slp-icon--size-sm slp-icon--color-color-text-300 social__links--icon" style="" role="img" aria-hidden="true" data-v-0b1e0911><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512">
|
||
<!-- Font Awesome Free 5.15.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) -->
|
||
<path
|
||
fill="currentColor"
|
||
d="M100.28 448H7.4V148.9h92.88zM53.79 108.1C24.09 108.1 0 83.5 0 53.8a53.79 53.79 0 0 1 107.58 0c0 29.7-24.1 54.3-53.79 54.3zM447.9 448h-92.68V302.4c0-34.7-.7-79.2-48.29-79.2-48.29 0-55.69 37.7-55.69 76.7V448h-92.78V148.9h89.08v40.8h1.3c12.4-23.5 42.69-48.3 87.88-48.3 94 0 111.28 61.9 111.28 142.3V448z"
|
||
/>
|
||
</svg>
|
||
</span></a></li></ul></div></div><div class="footer-source" data-v-d02fbda3 data-v-2b76ce90><p class="slp-text-body1" data-v-64ad2128 data-v-2b76ce90><!--[-->Git is a trademark of Software Freedom Conservancy and our use of 'GitLab' is under license<!--]--><!----></p><p class="footer-source__source-buttons" data-v-2b76ce90><a target="_blank" class="footer-source__button" href="https://gitlab.com/gitlab-com/marketing/digital-experience/about-gitlab-com/" data-ga-name="page source" data-ga-location="footer" data-v-2b76ce90>View page source</a><a target="_blank" class="be-nav-footer__button" href="https://gitlab.com/gitlab-com/marketing/digital-experience/about-gitlab-com/-/blob/main/content/en-us/blog/how-we-diagnosed-and-resolved-redis-latency-spikes.md" data-ga-name="web ide" data-ga-location="footer" data-v-2b76ce90>Edit this page</a><a target="_blank" href="https://gitlab.com/gitlab-com/marketing/digital-experience/about-gitlab-com/-/blob/main/CONTRIBUTING.md/" data-ga-name="please contribute" data-ga-location="footer" data-v-2b76ce90>Please contribute</a><!----></p><p data-v-2b76ce90>© 2026 GitLab Inc.</p></div></div><!--]--></div></footer><!----></div><!--]--></div><div id="teleports"></div><script>window.__NUXT__={};window.__NUXT__.config={public:{projectId:"55044057",siteEnv:"production",mdc:{components:{prose:true,map:{},customElements:[]},headings:{anchorLinks:{h1:false,h2:true,h3:true,h4:true,h5:false,h6:false}},highlight:{noApiRoute:false,theme:"github-light",langs:["awk","bash","c","cpp","csharp","css","csv","cypher","diff","docker","erb","go","graphql","groovy","hcl","html","ini","java","javascript","json","liquid","log","markdown","mermaid","objective-c","perl","php","python","ruby","rust","sass","scss","shellsession","swift","terraform","toml","typescript","vue","xml","yaml","awk","bash","c","cpp","csharp","css","csv","cypher","diff","docker","erb","go","graphql","groovy","hcl","html","ini","java","javascript","json","liquid","log","markdown","mermaid","objective-c","perl","php","python","ruby","rust","sass","scss","shellsession","swift","terraform","toml","typescript","vue","xml","yaml"],highlighter:"shiki",shikiEngine:"oniguruma"}},"nuxt-scripts":{version:"",prefix:"/_scripts",defaultScriptOptions:{trigger:"onNuxtReady"},googleStaticMapsProxy:"",endpoints:{}},"seo-utils":{canonicalQueryWhitelist:["page","sort","filter","search","q","category","tag"],canonicalLowercase:true},content:{wsUrl:""},i18n:{baseUrl:"",defaultLocale:"en-us",rootRedirect:"",redirectStatusCode:302,skipSettingLocaleOnNavigate:false,locales:[{code:"en-us",language:"en-US",label:"English",langLabel:"Language"},{code:"de-de",language:"de-DE",label:"Deutsch",englishLabel:"German",langLabel:"Sprache"},{code:"es",language:"es",label:"Español",englishLabel:"Spanish",langLabel:"Idioma"},{code:"fr-fr",language:"fr-FR",label:"Français",englishLabel:"French",langLabel:"Langue"},{code:"it-it",language:"it-IT",label:"Italiano",englishLabel:"Italian",langLabel:"Lingua"},{code:"ja-jp",language:"ja-JP",label:"日本語",englishLabel:"Japanese",langLabel:"言語"},{code:"ko-kr",language:"ko-KR",label:"한국어",englishLabel:"Korean",langLabel:"언어"},{code:"pt-br",language:"pt-BR",label:"Português",englishLabel:"Portuguese",langLabel:"Idioma"}],detectBrowserLanguage:false,experimental:{localeDetector:"",typedPages:true,typedOptionsAndMessages:false,alternateLinkCanonicalQueries:true,devCache:false,cacheLifetime:"",stripMessagesPayload:false,preload:false,strictSeo:false,nitroContextDetection:true,httpCacheDuration:10},domainLocales:{"en-us":{domain:""},"de-de":{domain:""},es:{domain:""},"fr-fr":{domain:""},"it-it":{domain:""},"ja-jp":{domain:""},"ko-kr":{domain:""},"pt-br":{domain:""}}},"nuxt-robots":{version:"5.7.1",isNuxtContentV2:false,debug:false,credits:true,groups:[{userAgent:["*"],disallow:["/search/","/de-de/search/","/es/search/","/fr-fr/search/","/it-it/search/","/ja-jp/search/","/ko-kr/search/","/pt-br/search/","/api/"],allow:[],contentUsage:[],contentSignal:[],_indexable:true,_rules:[{pattern:"/search/",allow:false},{pattern:"/de-de/search/",allow:false},{pattern:"/es/search/",allow:false},{pattern:"/fr-fr/search/",allow:false},{pattern:"/it-it/search/",allow:false},{pattern:"/ja-jp/search/",allow:false},{pattern:"/ko-kr/search/",allow:false},{pattern:"/pt-br/search/",allow:false},{pattern:"/api/",allow:false}],_normalized:true}],sitemap:["https://about.gitlab.com/sitemap.xml","/sitemap_index.xml"],header:true,robotsEnabledValue:"index, follow, max-image-preview:large, max-snippet:-1, max-video-preview:-1",robotsDisabledValue:"noindex, nofollow",cacheControl:"max-age=14400, must-revalidate",botDetection:true,pageMetaRobots:{}}},app:{baseURL:"/",buildId:"0834acfa-cb5b-41d1-8287-bd08f5a0480e",buildAssetsDir:"/_nuxt/",cdnURL:""}}</script><script type="application/ld+json" data-nuxt-schema-org="true" data-hid="schema-org-graph">{"@context":"https://schema.org","@graph":[{"@id":"https://about.gitlab.com/#website","@type":"WebSite","inLanguage":"en-US","name":"about.gitlab.com","url":"https://about.gitlab.com/","publisher":{"@id":"https://about.gitlab.com/#identity"},"workTranslation":[{"@id":"https://about.gitlab.com/#website"},{"@id":"https://about.gitlab.com/#website"},{"@id":"https://about.gitlab.com/#website"},{"@id":"https://about.gitlab.com/#website"},{"@id":"https://about.gitlab.com/#website"},{"@id":"https://about.gitlab.com/#website"},{"@id":"https://about.gitlab.com/#website"},{"@id":"https://about.gitlab.com/#website"}]},{"@id":"https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/#webpage","@type":"WebPage","dateModified":"2022-11-28T00:00:00.000Z","datePublished":"2022-11-28T00:00:00.000Z","description":"How we uncovered a three-phase cycle involving two distinct saturation points and a simple fix to break that cycle.","name":"How we diagnosed and resolved Redis latency spikes with BPF and other tools","url":"https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/","about":{"@id":"https://about.gitlab.com/#identity"},"isPartOf":{"@id":"https://about.gitlab.com/#website"},"potentialAction":[{"@type":"ReadAction","target":["https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/"]}],"primaryImageOfPage":{"@id":"https://about.gitlab.com/#logo"}},{"@id":"https://about.gitlab.com/#identity","@type":"Organization","name":"GitLab","url":"https://about.gitlab.com"},{"@context":"https://schema.org","@id":"https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/#article","articleSection":"engineering","dateModified":"2022-11-28T00:00:00.000Z","datePublished":"2022-11-28T00:00:00.000Z","description":"How we uncovered a three-phase cycle involving two distinct saturation points and a simple fix to break that cycle.","headline":"How we diagnosed and resolved Redis latency spikes with BPF and other tools","inLanguage":"en","thumbnailUrl":"https://res.cloudinary.com/about-gitlab-com/image/upload/v1749667913/Blog/Hero%20Images/clocks.jpg","timeRequired":"PT26M","@type":["Article","BlogPosting"],"author":{"@id":"https://about.gitlab.com/#/schema/person/1"},"image":{"@id":"https://about.gitlab.com/#/schema/image/1"},"isPartOf":{"@id":"https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/#webpage"},"keywords":["performance","tutorial","DevOps"],"mainEntityOfPage":{"@type":"WebPage","@id":"https://about.gitlab.com/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/"},"publisher":{"@id":"https://about.gitlab.com/#organization"}},{"@id":"https://about.gitlab.com/#/schema/person/1","@type":"Person","name":"Matt Smiley","url":"https://about.gitlab.com/blog/authors/matt-smiley"},{"@id":"https://about.gitlab.com/#logo","@type":"ImageObject","caption":"GitLab","contentUrl":"https://res.cloudinary.com/about-gitlab-com/image/upload/v1750705064/crnigurqmalqj1uwhlg8.png","inLanguage":"en","url":"https://res.cloudinary.com/about-gitlab-com/image/upload/v1750705064/crnigurqmalqj1uwhlg8.png"},{"@id":"https://about.gitlab.com/#organization","@type":"Organization","logo":"https://res.cloudinary.com/about-gitlab-com/image/upload/v1750705064/crnigurqmalqj1uwhlg8.png","name":"GitLab","url":"https://about.gitlab.com"},{"@id":"https://about.gitlab.com/#/schema/image/1","@type":"ImageObject","contentUrl":"https://res.cloudinary.com/about-gitlab-com/image/upload/v1749667913/Blog/Hero%20Images/clocks.jpg","inLanguage":"en","url":"https://res.cloudinary.com/about-gitlab-com/image/upload/v1749667913/Blog/Hero%20Images/clocks.jpg"}]}</script><script type="application/json" data-nuxt-data="nuxt-app" data-ssr="true" id="__NUXT_DATA__" data-src="/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/_payload.json?0834acfa-cb5b-41d1-8287-bd08f5a0480e">[{"state":1,"once":35,"_errors":36,"serverRendered":6,"path":38,"prerenderedAt":39},["Reactive",2],{"$si18n:cached-locale-configs":3,"$si18n:resolved-locale":21,"$savailableLanguages":22,"$snuxt-seo-utils:routeRules":24,"$ssite-config":25},{"en-us":4,"de-de":7,"es":9,"fr-fr":11,"it-it":13,"ja-jp":15,"ko-kr":17,"pt-br":19},{"fallbacks":5,"cacheable":6},[],true,{"fallbacks":8,"cacheable":6},[],{"fallbacks":10,"cacheable":6},[],{"fallbacks":12,"cacheable":6},[],{"fallbacks":14,"cacheable":6},[],{"fallbacks":16,"cacheable":6},[],{"fallbacks":18,"cacheable":6},[],{"fallbacks":20,"cacheable":6},[],"",[23],"en-us",{"head":-1,"seoMeta":-1},{"_priority":26,"currentLocale":31,"defaultLocale":31,"env":32,"name":33,"trailingSlash":6,"url":34},{"name":27,"env":28,"url":29,"trailingSlash":29,"defaultLocale":30,"currentLocale":30},-10,0,-3,-2,"en-US","production","about.gitlab.com","https://about.gitlab.com",["Set"],["ShallowReactive",37],{"/blog/how-we-diagnosed-and-resolved-redis-latency-spikes":-1,"navigation-en-us":-1,"banner-en-us":-1,"footer-en-us":-1,"blog-post-authors-en-us-Matt Smiley":-1,"blog-related-posts-en-us-how-we-diagnosed-and-resolved-redis-latency-spikes":-1,"blog-promotions-en-us":-1,"next-steps-en-us":-1},"/blog/how-we-diagnosed-and-resolved-redis-latency-spikes/",1789154682070]</script></body></html> |