599 lines
413 KiB
HTML
599 lines
413 KiB
HTML
<!DOCTYPE html><!-- Minimal inline styling for MathJax --><html lang="en" data-tags="message queue" data-blog-type="engineering"> <head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><!-- Temporary: redirect raw Amplify domain to canonical. Remove once CF injects X-Canonical-Host. --><script>
|
||
if (window.location.hostname === 'main.d2372vaejiu5h9.amplifyapp.com') {
|
||
var u = new URL(window.location.href);
|
||
u.hostname = 'www.datadoghq.com';
|
||
window.location.replace(u.toString());
|
||
}
|
||
</script><!-- Preload required fonts --><link rel="preload" href="/fonts/web-fonts/NationalWeb-Book.woff2" as="font" type="font/woff2" crossorigin><link rel="preload" href="/fonts/web-fonts/NationalWeb-Medium.woff2" as="font" type="font/woff2" crossorigin><link rel="preload" href="/fonts/web-fonts/NationalWeb-Semibold.woff2" as="font" type="font/woff2" crossorigin><link rel="preload" href="/fonts/web-fonts/NationalWeb-Bold.woff2" as="font" type="font/woff2" crossorigin><link rel="preload" href="/fonts/web-fonts/RobotoMono-Regular.ttf" as="font" type="font/ttf" crossorigin><!-- Preconnect to required resources --><link rel="dns-prefetch preconnect" href="https://blog.dd-static.net"><link rel="dns-prefetch preconnect" href="https://www.googletagmanager.com"><link rel="dns-prefetch preconnect" href="https://consent.trustarc.com"><link rel="dns-prefetch preconnect" href="https://browser-intake-datadoghq.com"><link rel="dns-prefetch preconnect" href="https://gk6e3zbyuntvc5dap.a1.typesense.net"><title>How we use formal modeling, lightweight simulations, and chaos testing to design reliable distributed systems | Datadog</title><meta name="description" content="Learn how we used formal modeling and simulation to analyze a distributed, multi-tenant queueing system."><meta name="robots" content="index, follow"><link rel="canonical" href="https://www.datadoghq.com/blog/engineering/formal-modeling-and-simulation/"><meta name="date" content="2024-11-20"><!-- RSS Feed --><link rel="alternate" type="application/rss+xml" title="Datadog | The Monitor blog" href="https://www.datadoghq.com/blog/index.xml"><!-- Hreflang alternates --><link rel="alternate" hreflang="en" href="https://www.datadoghq.com/blog/engineering/formal-modeling-and-simulation/"><link rel="alternate" hreflang="x-default" href="https://www.datadoghq.com/blog/engineering/formal-modeling-and-simulation/"><!-- Additional head content from child layouts --><script type="application/ld+json">{"@context":"https://schema.org","@type":"Article","author":[{"@type":"Person","name":"Arun Parthiban"},{"@type":"Person","name":"Sesh Nalla"},{"@type":"Person","name":"Cecilia Wat-Kim"}],"datePublished":"2024-11-20T00:00:00.000Z","description":"Learn how we used formal modeling and simulation to analyze a distributed, multi-tenant queueing system.","headline":"How we use formal modeling, lightweight simulations, and chaos testing to design reliable distributed systems","image":"https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png","inLanguage":"en-US","mainEntityOfPage":{"@type":"WebPage","@id":"https://www.datadoghq.com/blog/engineering/formal-modeling-and-simulation/"},"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://blog.dd-static.net/img/dd_logo_n_70x75.png"},"name":"Datadog","sameAs":["https://twitter.com/datadoghq","https://www.instagram.com/datadoghq/","https://en.wikipedia.org/wiki/Datadog","https://www.facebook.com/datadoghq/","https://www.linkedin.com/company/datadog","https://github.com/DataDog","https://www.youtube.com/channel/UCPO2QgTCReBAThZca6MB9jg"],"url":"https://www.datadoghq.com/"},"keywords":"message queue"}</script><meta name="keywords" content="message queue"><meta name="author" content="Arun Parthiban, Sesh Nalla, Cecilia Wat-Kim"><!-- Open Graph Tags --><meta property="og:title" content="How we use formal modeling, lightweight simulations, and chaos testing to design reliable distributed systems | Datadog"><meta property="og:description" content="Learn how we used formal modeling and simulation to analyze a distributed, multi-tenant queueing system."><meta property="og:image" content="https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png"><meta property="og:type" content="article"><meta property="og:url" content="https://www.datadoghq.com/blog/engineering/formal-modeling-and-simulation/"><meta property="og:site_name" content="Datadog"><meta property="og:locale" content="en"><!-- Twitter Tags --><meta name="twitter:card" content="summary_large_image"><meta name="twitter:title" content="How we use formal modeling, lightweight simulations, and chaos testing to design reliable distributed systems | Datadog"><meta name="twitter:description" content="Learn how we used formal modeling and simulation to analyze a distributed, multi-tenant queueing system."><meta name="twitter:image" content="https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png"><meta name="twitter:url" content="https://www.datadoghq.com/blog/engineering/formal-modeling-and-simulation/"><meta name="twitter:site" content="@datadoghq"><meta name="twitter:creator" content="@datadoghq"><!-- Preconnect to required resources --><link rel="preconnect" href="https://www.googletagmanager.com" crossorigin><link rel="preconnect" href="https://webops-gtm.datadoghq.com" crossorigin><link rel="preconnect" href="https://browser-intake-datadoghq.com" crossorigin><!--- Google Tag Manager -->
|
||
<noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-WDC8G6" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||
<script>(function(){const gtmContainerId = "GTM-WDC8G6";
|
||
(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
|
||
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
|
||
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
|
||
'//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
|
||
})(window,document,'script','dataLayer',`${gtmContainerId}`);})();</script>
|
||
<!-- End Google Tag Manager --><link rel="stylesheet" href="/blog/_astro/Base.C9VITJZw.css"><link rel="stylesheet" href="/blog/_astro/index.DlKsH7qy.css"><script type="module" src="/blog/_astro/page.Bc5QPW76.js"></script></head> <body> <script type="module">document.querySelectorAll("dialog[id]").forEach(r=>{const e=r;if(e.__dialogInitialized)return;e.__dialogInitialized=!0;const n=e.id,l=e.querySelector(`#${n}-close`),t=document.querySelector("body"),o="overflow-hidden",d=()=>{e.close(),t.classList.remove(o),t.style.overflow=""},a=()=>{e.showModal(),t.classList.add(o)};e.addEventListener("click",i=>{const c=e.firstElementChild?.getBoundingClientRect()||e.getBoundingClientRect();(i.clientX<c.left||i.clientX>c.right||i.clientY<c.top||i.clientY>c.bottom)&&d()}),l&&l.addEventListener("click",()=>{d()}),e.addEventListener("keydown",i=>{i.key==="Escape"&&d()}),e.__openDialog=a,e.__closeDialog=d});document.querySelectorAll("[data-dialog-trigger]").forEach(r=>{const e=r;if(e.__triggerInitialized)return;e.__triggerInitialized=!0;const n=e.dataset.dialogTrigger;if(!n)return;const l=document.getElementById(n);if(!l)return;const t=l.__openDialog;e.addEventListener("click",()=>{t()}),e.addEventListener("keydown",o=>{(o.key==="Enter"||o.key===" ")&&(o.preventDefault(),t())})});</script> <div class="absolute top-0"><html><head></head><body><div class="announcement-banner text-center fixed top-0 z-[200] h-[30px] w-full flex items-center justify-center bg-[#632ca6] bg-gradient-to-r from-[#632ca6] to-[#2f80ed] text-center text-white font-semibold font-national text-20"><a class="fw-semibold text-nowrap" href="https://www.datadoghq.com/resources/gartner-magic-quadrant-observability-platforms-2026/?utm_source=inbound&utm_medium=corpsite-display&utm_campaign=dg-apm-ww-announcement-corpsite-announcement-gartnermq2026-obsplat"><span class="d-none d-md-block hidden banner-title-desktop tablet:block">Datadog named a Leader in the Gartner® Magic Quadrant™ for Observability Platforms
|
||
</span><span class="d-block d-md-none block tablet:hidden banner-title-mobile">Leader in the Gartner® Magic Quadrant™
|
||
</span></a><script>function getGeotarget(){const o=!1,t={webinars:[{codes:["US","CA"],desktop_title:null,end_date:null,link:null,mobile_title:null,start_date:null,supported_language:"en"},{codes:["UK","US","CA","GB","IE","BE","NL","PL","DK","SE",!1,"LU","IL","FI","GR","LI","CZ","ZA","IL","ES","DE","FR","IT","CH"],desktop_title:"Datadog Detect is back! Register for the virtual mini-conference on Oct 30, 12PM ET",end_date:"2025-10-30",link:"https://www.datadoghq.com/event/datadog-detect-q425/?utm_source=inbound&utm_medium=corpsite-display&utm_campaign=dg-security-ww-banner-corpsite",mobile_title:"Join us for Datadog Detect on Oct 30, 12PM ET",start_date:"2025-10-22",supported_language:"en"},{codes:["ES"],desktop_title:"Webinar de Datadog: Crea experiencias digitales impecables con monitorización End-to-End",end_date:"2025-05-06",link:"https://www.datadoghq.com/es/dg/webinars/uxdem-spain-2025/?utm_source=inbound&utm_medium=corpsite-display&utm_campaign=dg-dem-emea-livewebinar-202505uxdemsp",mobile_title:"Crea experiencias digitales impecables con monitorización End-to-End",start_date:"2025-04-22",supported_language:"es"},{codes:["JP"],desktop_title:"Datadog ウェビナー: エンドツーエンドの監視によって生み出されるシームレスな DX",end_date:"2025-05-14",link:"https://www.datadoghq.com/ja/dg/webinars/uxdem-japan-2025/?utm_source=inbound&utm_medium=corpsite-display&utm_campaign=dg-dem-japan-livewebinar-202505uxdemjp",mobile_title:"ウェビナー: エンドツーエンドの監視によって生み出されるシームレスな DX",start_date:"2025-05-01",supported_language:"ja"},{codes:["SG","NZ","AU","IN","PH","TW","VN","ID","TH","MY"],desktop_title:"Register Now: Create Seamless Digital Experiences with End-to-End Monitoring",end_date:"2025-04-24",link:"https://www.datadoghq.com/dg/webinars/uxdem-anz-2025/?utm_source=inbound&utm_medium=corpsite-display&utm_campaign=dg-dem-apac-livewebinar-202504uxdemanz",mobile_title:"Create Seamless Digital Experiences with End-to-End Monitoring",start_date:"2025-04-10",supported_language:"en"},{codes:["MX","AR","CO","PE","VE"],desktop_title:"Webinar de Datadog: Crea experiencias digitales fluidas con el monitoreo punto a punto",end_date:"2025-04-10",link:"https://www.datadoghq.com/es/dg/webinars/uxdem-es-2025/?utm_source=inbound&utm_medium=corpsite-display&utm_campaign=dg-dem-latam-livewebinar-202504uxdemes",mobile_title:"Crea experiencias digitales fluidas con el monitoreo punto a punto",start_date:"2025-04-01",supported_language:"es"},{codes:["FR","BE","CH","CA"],desktop_title:"Webinar Datadog: Créez des expériences numériques fluides grâce à la surveillance de bout en bout ",end_date:"2025-04-30",link:"https://www.datadoghq.com/fr/dg/webinars/uxdem-france-2025/?utm_source=inbound&utm_medium=corpsite-display&utm_campaign=dg-dem-emea-livewebinar-202504uxdemfr",mobile_title:"Créez des expériences numériques fluides grâce à la surveillance de bout en bout",start_date:"2025-04-16",supported_language:"fr"},{codes:["IT","CH"],desktop_title:"Webinar di Datadog: Creare digital experience impeccabili con il monitoraggio end-to-end",end_date:"2025-04-29",link:"https://www.datadoghq.com/it/dg/webinars/uxdem-italy-2025/?utm_source=inbound&utm_medium=corpsite-display&utm_campaign=dg-dem-emea-livewebinar-202504uxdemit",mobile_title:"Creare digital experience impeccabili con il monitoraggio end-to-end",start_date:"2025-04-15",supported_language:"it"},{codes:["KR"],desktop_title:"Datadog 웨비나 | 인프라 & 쿠버네티스 모니터링 쉽게 마스터하기! 2/21 (금) 2PM",end_date:"2025-02-20",link:"https://www.datadoghq.com/ko/dg/webinars/infra-korea-2025/?utm_source=inbound&utm_medium=corpsite-display&utm_campaign=dg-infra-korea-corpsite-banner-livewebinar-202502infrakr",mobile_title:"인프라 & 쿠버네티스 모니터링 쉽게 마스터하기! 2/21",start_date:"2025-01-30",supported_language:"ko"},{codes:["BR","PT"],desktop_title:"Webinar da Datadog: Crie experiências digitais perfeitas com monitoramento de ponta a ponta",end_date:"2025-04-09",link:"https://www.datadoghq.com/pt/dg/webinars/uxdem-pt-2025/?utm_source=inbound&utm_medium=corpsite-display&utm_campaign=dg-dem-latam-livewebinar-202504uxdempt",mobile_title:"Crie experiências digitais perfeitas com monitoramento de ponta a ponta",start_date:"2025-04-01",supported_language:"pt"},{codes:["DE","AT","CH"],desktop_title:"Datadog-Webinar: Schaffen Sie eine nahtlose digitale Experience mit End-to-End-Monitoring",end_date:"2025-04-09",link:"https://www.datadoghq.com/de/dg/webinars/uxdem-dach-2025/?utm_source=inbound&utm_medium=corpsite-display&utm_campaign=dg-dem-emea-livewebinar-202504uxdemdach",mobile_title:"Schaffen Sie eine nahtlose digitale Experience mit End-to-End-Monitoring",start_date:"2025-04-01",supported_language:"de"}]};let e="US";function n(e,t,n){const s=Date.now(),o={value:t,timestamp:s+n};localStorage.setItem(e,JSON.stringify(o))}function s(){const t=window.document.URL.match("(www|docs).datadoghq.com")?"https://www.datadoghq.com/locate":"https://corpsite-staging.datadoghq.com/locate";return fetch(t).then(e=>e.json()).then(t=>{e=t.country,n("country",e,6048e5)}).catch(n("country",e,0))}const i=()=>{try{const n=localStorage.getItem("country"),t=JSON.parse(n);if(Date.now()<t.timestamp)return e=t.value,e;s()}catch{s()}return e};if(t&&!o){i();let n=!1;t.webinars.forEach(t=>{t.codes.includes(e)&&(updateBanner(t),t.lp_link?n=!0:n=!1)})}}getGeotarget();function getLanguage(){const e=window.DD_RUM?.getUser?.().preferred_lang||navigator.language||"en",t=e.split("-");return t[0]}function populateBanner(e){document.querySelector(".banner-title-desktop").innerHTML=e.desktop_title,document.querySelector(".banner-title-mobile").innerHTML=e.mobile_title,document.querySelector(".announcement-banner > a").href=e.link}function updateBanner(e){const t=new Date,s=new Date(e.start_date),o=new Date(e.end_date),n=getLanguage(),{env:i}=document.documentElement.dataset;i!=="live"&&e.hasOwnProperty("test_message")&&e.test_message&&n===e.supported_language&&populateBanner(e),t>=s&&t<=o&&n===e.supported_language&&populateBanner(e)}</script></div><div id="main-nav-wrapper" x-data="{
|
||
mobileOpen: false,
|
||
dropdownOpen: 'none',
|
||
navScrolled: false,
|
||
hasAnnouncementBanner: true,
|
||
handleScroll() {
|
||
this.navScrolled = window.scrollY > 30;
|
||
},
|
||
handleResize() {
|
||
if (window.innerWidth >= 1024) {
|
||
this.mobileOpen = false;
|
||
this.dropdownOpen = 'none';
|
||
}
|
||
},
|
||
checkAnnouncementBanner() {
|
||
const announcementBanner = document.querySelector('.announcement-banner') || document.querySelector('.announcement-banner--large');
|
||
if (announcementBanner) {
|
||
this.hasAnnouncementBanner = true;
|
||
} else {
|
||
this.hasAnnouncementBanner = false;
|
||
}
|
||
}
|
||
}" x-init="checkAnnouncementBanner()" x-on:scroll.window="handleScroll" x-on:resize.window="handleResize"><nav class="main-nav h-[60px] desktop-sm:h-[130px] flex fixed right-0 left-0 items-center bg-white z-[200] font-normal mt-[30px]" :class="[navScrolled ? 'main-nav-scrolled desktop-sm:!h-[65px]' : '', hasAnnouncementBanner ? 'mt-[30px]' : '']" style="box-shadow:0 0 8px 8px rgba(0,0,0,7%)"><div class="container flex relative w-full justify-between desktop-sm:justify-start"><div class="hidden desktop-sm:flex justify-between w-full"><ul x-data="{ mouseOver: 'none' }" class="flex flex-row items-center list-none m-0 grow justify-start basis-0"><li x-on:mouseover="mouseOver = 'product'" x-on:mouseout="mouseOver = 'none'" class="dropdown relative product-dropdown"><a class="dropdown relative block pt-[8.5px] px-2 desktop-lg:px-3 pb-[9.5px] desktop-lg:px-3 text-18 font-semibold font-national uppercase text-[#555] hover:text-[#70f]" href="https://www.datadoghq.com/product/">Product</a><div x-show="mouseOver === 'product'" x-cloak="" class="dropdown-menu product-menu absolute z-40 bg-white min-h-[630px] w-[1300px] p-0 left-[13px] top-[42px] border-t-[3px] border-t-[#632ca6]" style="box-shadow:0 8px 10px rgba(0,0,0,7%)" x-data="{
|
||
openCategory: 'observability',
|
||
timeoutID: null,
|
||
setCategory(category) {
|
||
this.timeoutID = setTimeout(() => {
|
||
this.openCategory = category;
|
||
const productMenu = document.querySelector('.product-menu');
|
||
window.DD_RUM?.onReady(function() {
|
||
if (productMenu.classList.contains('show')) {
|
||
window.DD_RUM.addAction(`Product Category ${category} Hover`)
|
||
}
|
||
})
|
||
}, 160);
|
||
},
|
||
clearCategory() {
|
||
clearTimeout(this.timeoutID);
|
||
}
|
||
}" x-init="
|
||
const menu = document.querySelector('.product-menu');
|
||
var observer = new MutationObserver(function(mutations) {
|
||
mutations.forEach(function(mutation) {
|
||
if (mutation.attributeName === 'class' && !mutation.target.classList.contains('show')) {
|
||
openCategory = 'observability';
|
||
}
|
||
});
|
||
});
|
||
observer.observe(menu, { attributes: true });
|
||
"><div class="grid grid-cols-12"><div class="col-span-4 category-toggle-list-column border-r border-r-[#e2e2e2] p-10"><p class="product-hype px-3 py-7 font-bold font-national text-28 leading-32">The integrated platform for monitoring & security</p><ul class="category-toggle-list pr-5 mb-15"><li><button x-on:mouseover="setCategory('observability')" x-on:mouseout="clearCategory()" class="flex items-center gap-2 w-full font-bold font-national text-22 leading-24 text-left px-3 py-2.5 align-middle hover:text-[#70f] hover:[&_svg]:fill-[#70f] hover:bg-black/5 rounded h-12" :class="openCategory === 'observability' ? 'text-[#632ca6] [&_svg]:fill-[#632ca6]' : ''">
|
||
<svg class="w-[22px] h-auto" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M96,136a40,40,0,1,1,40-40A40.045,40.045,0,0,1,96,136Zm0-64a24,24,0,1,0,24,24A24.028,24.028,0,0,0,96,72Zm0,92C35.749,164,4.16,102.208,2.845,99.578L1.056,96l1.789-3.578C4.16,89.792,35.749,28,96,28s91.84,61.792,93.155,64.422L190.944,96l-1.789,3.578C187.84,102.208,156.252,164,96,164ZM19.165,96C26.457,108.305,53.3,148,96,148s69.544-39.7,76.836-52C165.551,83.708,138.706,44,96,44S26.457,83.7,19.165,96Z"></path>
|
||
</svg>
|
||
Observability</button></li><li><button x-on:mouseover="setCategory('security')" x-on:mouseout="clearCategory()" class="flex items-center gap-2 w-full font-bold font-national text-22 leading-24 text-left px-3 py-2.5 align-middle hover:text-[#70f] hover:[&_svg]:fill-[#70f] hover:bg-black/5 rounded h-12" :class="openCategory === 'security' ? 'text-[#632ca6] [&_svg]:fill-[#632ca6]' : ''">
|
||
<svg class="w-[22px] h-auto" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>security-platform</title>
|
||
<path d="M16 0.793l-12 6.938v9.604c0.007 4.42 2.392 8.281 5.944 10.372l0.056 0.031 6 3.468 6-3.469c3.608-2.122 5.993-5.983 6-10.402v-9.605zM6.667 10.811l8 4.625v4.42l-8-4.625zM7.997 8.5l6.669-3.856v7.711zM6.723 18.343l7.944 4.593v4.42l-3.333-1.926c-2.521-1.484-4.269-4.050-4.607-7.044l-0.004-0.043zM25.333 17.335c-0.005 3.439-1.86 6.443-4.623 8.070l-0.044 0.024-3.333 1.927v-22.712l8 4.625z"></path>
|
||
</svg>
|
||
Security</button></li><li><button x-on:mouseover="setCategory('digital-experience')" x-on:mouseout="clearCategory()" class="flex items-center gap-2 w-full font-bold font-national text-22 leading-24 text-left px-3 py-2.5 align-middle hover:text-[#70f] hover:[&_svg]:fill-[#70f] hover:bg-black/5 rounded h-12" :class="openCategory === 'digital-experience' ? 'text-[#632ca6] [&_svg]:fill-[#632ca6]' : ''">
|
||
<svg class="w-[22px] h-auto" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>rum</title>
|
||
<path d="M8.728 32h-8.728v-14.544h8.728v14.544zM2.912 29.088h2.912v-8.728h-2.912v8.728zM18.908 29.088h-7.269v-2.912h4.364v-14.544h-10.187v2.912h-2.905v-5.816h16.003v20.361zM32 20.361h-10.18v-2.912h7.276v-14.544h-20.367v2.912h-2.912v-5.816h26.184v20.361z"></path>
|
||
</svg>
|
||
Digital Experience</button></li><li><button x-on:mouseover="setCategory('software-delivery')" x-on:mouseout="clearCategory()" class="flex items-center gap-2 w-full font-bold font-national text-22 leading-24 text-left px-3 py-2.5 align-middle hover:text-[#70f] hover:[&_svg]:fill-[#70f] hover:bg-black/5 rounded h-12" :class="openCategory === 'software-delivery' ? 'text-[#632ca6] [&_svg]:fill-[#632ca6]' : ''">
|
||
<svg class="w-[22px] h-auto" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>ci</title>
|
||
<path d="M30.667 22.625v0.087c-0.006 4.392-3.565 7.95-7.956 7.955h-0.085c-3.937-0.005-7.205-2.864-7.845-6.62l-0.007-0.047h-3.107l4.333-4.333 4.333 4.333h-2.833c0.6 2.315 2.669 3.998 5.131 4h0.087c2.916-0.009 5.278-2.373 5.282-5.29v-0.085c-0.003-2.921-2.369-5.288-5.29-5.292h-13.42c-4.393-0.004-7.954-3.565-7.957-7.958v-0.087c0.006-4.392 3.565-7.95 7.956-7.955h0.085c4.393 0.004 7.954 3.564 7.958 7.956v2.044h-2.667v-2.043c-0.004-2.921-2.371-5.287-5.291-5.29h-0.087c-2.919 0.005-5.285 2.371-5.288 5.29v0.085c0.003 2.921 2.369 5.288 5.29 5.292h13.42c4.393 0.004 7.954 3.565 7.957 7.958v0z"></path>
|
||
</svg>
|
||
Software Delivery</button></li><li><button x-on:mouseover="setCategory('service-management')" x-on:mouseout="clearCategory()" class="flex items-center gap-2 w-full font-bold font-national text-22 leading-24 text-left px-3 py-2.5 align-middle hover:text-[#70f] hover:[&_svg]:fill-[#70f] hover:bg-black/5 rounded h-12" :class="openCategory === 'service-management' ? 'text-[#632ca6] [&_svg]:fill-[#632ca6]' : ''">
|
||
<svg class="w-[22px] h-auto" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M88,188.38V91.38l84-48.5v96.99l-84,48.51Zm16-87.76v60.04l52-30.02v-60.04l-52,30.02Zm-50-28.87v97l16-9.24V80.99l52-30.02,16-9.24V23.25L54,71.75ZM20,52.12v97l16-9.24V61.36l52-30.02,16-9.24V3.62L20,52.12Z"></path>
|
||
</svg>
|
||
Service Management</button></li><li><button x-on:mouseover="setCategory('ai')" x-on:mouseout="clearCategory()" class="flex items-center gap-2 w-full font-bold font-national text-22 leading-24 text-left px-3 py-2.5 align-middle hover:text-[#70f] hover:[&_svg]:fill-[#70f] hover:bg-black/5 rounded h-12" :class="openCategory === 'ai' ? 'text-[#632ca6] [&_svg]:fill-[#632ca6]' : ''">
|
||
<svg class="w-[22px] h-auto" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" fill="" viewBox="0 0 192 192" style="enable-background:new 0 0 192 192;" xml:space="preserve"><g id="Layer_7"><g><linearGradient id="SVGID_1_" gradientUnits="userSpaceOnUse" x1="89.0192" y1="122.732" x2="142.9808" y2="29.268"><stop offset="0" style="stop-color:#5500FF"></stop><stop offset="0.1715" style="stop-color:#4D09FF"></stop><stop offset="0.4511" style="stop-color:#3821FF"></stop><stop offset="0.802" style="stop-color:#1649FF"></stop><stop offset="1" style="stop-color:#0062FF"></stop></linearGradient><path class="st5" d="M120,16h-8c0,30.88-25.12,56-56,56v8c30.88,0,56,25.12,56,56h8c0-30.88,25.12-56,56-56v-8 C145.12,72,120,46.88,120,16z"></path><linearGradient id="SVGID_00000049207151058930698180000011839073544639225000_" gradientUnits="userSpaceOnUse" x1="35.4115" y1="168.7311" x2="68.5885" y2="111.267"><stop offset="0" style="stop-color:#5500FF"></stop><stop offset="0.1715" style="stop-color:#4D09FF"></stop><stop offset="0.4511" style="stop-color:#3821FF"></stop><stop offset="0.802" style="stop-color:#1649FF"></stop><stop offset="1" style="stop-color:#0062FF"></stop></linearGradient><path d="M56,104h-4c0,0,0,0,0,0c0,0,0,0,0,0h-4 c0,17.64-14.35,32-32,32v8c17.65,0,32,14.36,32,32h8c0-17.64,14.35-32,32-32v-8C70.35,136,56,121.64,56,104z"></path></g></g></svg>AI</button></li><li><button x-on:mouseover="setCategory('platform')" x-on:mouseout="clearCategory()" class="flex items-center gap-2 w-full font-bold font-national text-22 leading-24 text-left px-3 py-2.5 align-middle hover:text-[#70f] hover:[&_svg]:fill-[#70f] hover:bg-black/5 rounded h-12" :class="openCategory === 'platform' ? 'text-[#632ca6] [&_svg]:fill-[#632ca6]' : ''">
|
||
<svg class="w-[22px] h-auto" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>dashboard</title>
|
||
<path d="M32 24.876v1.79h-32v-21.333h1.988v19.543h30.012zM26.009 10.679l3.976 12.432h-25.982v-8.006l6.999-7.981 8.987 7.981 6.020-4.426z"></path>
|
||
</svg>
|
||
Platform Capabilities</button></li></ul><a href="https://www.datadoghq.com/pricing/" class="pricing-link group flex items-center gap-[5px] font-bold font-national text-[#632ca6] uppercase ml-3 text-16 absolute bottom-10 hover:text-[#70f]">View pricing<svg class="w-2 h-auto -tw-mb-px fill-[#632ca6] group-hover:fill-[#70f]" version="1.1" xmlns="http://www.w3.org/2000/svg" width="683" height="1024" viewBox="0 0 683 1024">
|
||
<title></title>
|
||
<g id="icomoon-ignore">
|
||
</g>
|
||
<path d="M391.327 512.001l-366.327-366.327 120.679-120.679 487.007 487.006-487.007 487.005-120.679-120.678 366.327-366.327z"></path>
|
||
</svg></a></div><div class="col-span-8"><div class="product-category h-full flex flex-col p-10 pl-[25px]" id="observability-detail" x-show="openCategory === 'observability'" x-cloak=""><div class="category-description p-7 rounded flex mb-10 text-white [background:var(--cta-bg)]" style="--cta-bg:linear-gradient(90deg, #8000FF 0%, #FF0080 100%)"><div class="category-description-main flex gap-5 flex-1 min-w-0 items-center"><svg class="w-12 h-auto fill-white shrink-0" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M96,136a40,40,0,1,1,40-40A40.045,40.045,0,0,1,96,136Zm0-64a24,24,0,1,0,24,24A24.028,24.028,0,0,0,96,72Zm0,92C35.749,164,4.16,102.208,2.845,99.578L1.056,96l1.789-3.578C4.16,89.792,35.749,28,96,28s91.84,61.792,93.155,64.422L190.944,96l-1.789,3.578C187.84,102.208,156.252,164,96,164ZM19.165,96C26.457,108.305,53.3,148,96,148s69.544-39.7,76.836-52C165.551,83.708,138.706,44,96,44S26.457,83.7,19.165,96Z"></path>
|
||
</svg><div class="info"><p class="category-header font-bold font-national text-28 leading-24 pb-[5px]">Observability</p><p class="category-tagline text-18 leading-22 m-0 font-medium">End-to-end, simplified visibility into your stack’s health & performance</p></div></div></div><div class="category-details flex justify-start gap-5"><div class="product-subcategory w-1/3"><p class="subcategory-header font-bold font-national uppercase text-16 leading-18 p-3">Infrastructure</p><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/infrastructure-monitoring/" data-dd-action-name="infrastructure_monitoring in observability">Infrastructure Monitoring</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/metrics/" data-dd-action-name="custom-metrics in observability">Custom Metrics</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/network-monitoring/" data-dd-action-name="network_monitoring in observability">Network Monitoring</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/container-monitoring/" data-dd-action-name="container_monitoring in observability">Container Monitoring</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/kubernetes-autoscaling/" data-dd-action-name="kubernetes_autoscaling in observability">Kubernetes Autoscaling</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/serverless-monitoring/" data-dd-action-name="serverless_monitoring in observability">Serverless Monitoring</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/cloud-cost-management/" data-dd-action-name="cloud_cost_management in observability">Cloud Cost Management</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/storage-management/" data-dd-action-name="storage_management in observability">Storage Management</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/gpu-monitoring/" data-dd-action-name="gpu_monitoring in observability">GPU Monitoring</a></li></ul></div><div class="product-subcategory w-1/3"><p class="subcategory-header font-bold font-national uppercase text-16 leading-18 p-3">Applications</p><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/apm/" data-dd-action-name="application_performance_monitoring in observability">Application Performance Monitoring</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/universal-service-monitoring/" data-dd-action-name="universal_service_monitoring in observability">Universal Service Monitoring</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/code-profiling/" data-dd-action-name="continuous_profiler in observability">Continuous Profiler</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/dynamic-instrumentation/" data-dd-action-name="dynamic_instrumentation in observability">Dynamic Instrumentation</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/products/ai/agent-observability/" data-dd-action-name="agent_observability in observability">Agent Observability</a></li></ul><p class="subcategory-header font-bold font-national uppercase text-16 leading-18 p-3">Data</p><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/database-monitoring/" data-dd-action-name="database_monitoring in observability">Database Monitoring</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/products/observability/data-observability/" data-dd-action-name="data_observability in observability">Data Observability</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/data-streams-monitoring/" data-dd-action-name="data_streams_monitoring in observability">Data Streams Monitoring</a></li></ul></div><div class="product-subcategory w-1/3"><p class="subcategory-header font-bold font-national uppercase text-16 leading-18 p-3">Logs</p><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/log-management/" data-dd-action-name="log_management in observability">Log Management</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/audit-trail/" data-dd-action-name="audit_trail in observability">Audit Trail</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/observability-pipelines/" data-dd-action-name="observability_pipelines in observability">Observability Pipelines</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/error-tracking/" data-dd-action-name="error_tracking in observability">Error Tracking</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/byoc-logs/" data-dd-action-name="byoc_logs in observability">Bring Your Own Cloud Log Management</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/sensitive-data-scanner/" data-dd-action-name="sensitive_data_scanner in observability">Sensitive Data Scanner</a></li></ul></div></div></div><div class="product-category h-full flex flex-col p-10 pl-[25px]" id="infra-detail" x-show="openCategory === 'infra'" x-cloak=""><div class="category-description p-7 rounded flex mb-10 text-white [background:var(--cta-bg)]" style="--cta-bg:linear-gradient(90deg, %!s(<nil>) 0%, %!s(<nil>) 100%)"><div class="category-description-main flex gap-5 flex-1 min-w-0 items-center"><svg class="w-12 h-auto fill-white shrink-0" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>host-map</title>
|
||
<path d="M14.214 13.103l-7.107 4.103-7.107-4.103v-8.206l7.107-4.103 7.107 4.103zM32 4.897v8.206l-7.107 4.103-7.107-4.103v-8.206l7.107-4.103zM29.333 6.436l-4.44-2.563-4.44 2.563v5.127l4.44 2.563 4.44-2.563zM9.037 19.98v8.040l6.963 4.020 6.962-4.020v-8.040l-6.963-4.020z"></path>
|
||
</svg><div class="info"><p class="category-header font-bold font-national text-28 leading-24 pb-[5px]">Infrastructure</p><p class="category-tagline text-18 leading-22 m-0 font-medium"></p></div></div></div><div class="category-details flex justify-start gap-5"></div></div><div class="product-category h-full flex flex-col p-10 pl-[25px]" id="apps-detail" x-show="openCategory === 'apps'" x-cloak=""><div class="category-description p-7 rounded flex mb-10 text-white [background:var(--cta-bg)]" style="--cta-bg:linear-gradient(90deg, %!s(<nil>) 0%, %!s(<nil>) 100%)"><div class="category-description-main flex gap-5 flex-1 min-w-0 items-center"><svg class="w-12 h-auto fill-white shrink-0" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>apm</title>
|
||
<path d="M0 2.667h32v3.81h-32v-3.81zM0 10.286h6v3.81h-6v-3.81zM10 10.286h22v3.81h-22v-3.81zM10 17.905h12v3.81h-12v-3.81zM26 17.905h6v3.81h-6v-3.81zM16 25.524h6v3.81h-6v-3.81z"></path>
|
||
</svg><div class="info"><p class="category-header font-bold font-national text-28 leading-24 pb-[5px]">Applications</p><p class="category-tagline text-18 leading-22 m-0 font-medium"></p></div></div></div><div class="category-details flex justify-start gap-5"></div></div><div class="product-category h-full flex flex-col p-10 pl-[25px]" id="data-detail" x-show="openCategory === 'data'" x-cloak=""><div class="category-description p-7 rounded flex mb-10 text-white [background:var(--cta-bg)]" style="--cta-bg:linear-gradient(90deg, %!s(<nil>) 0%, %!s(<nil>) 100%)"><div class="category-description-main flex gap-5 flex-1 min-w-0 items-center"><svg class="w-12 h-auto fill-white shrink-0" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M96,8C50.393,8,16,25.2,16,48v96c0,22.8,34.393,40,80,40s80-17.2,80-40V48C176,25.2,141.607,8,96,8ZM48.337,32.683C60.936,27.083,77.863,24,96,24c39.068,0,64,14.214,64,24,0,4.969-6.26,10.838-16.337,15.317C131.064,68.917,114.138,72,96,72,56.932,72,32,57.786,32,48,32,43.031,38.26,37.162,48.337,32.683Zm95.326,126.633C131.064,164.916,114.137,168,96,168c-39.068,0-64-14.215-64-24V120.577C46.356,130.075,69.242,136,96,136s49.644-5.925,64-15.423V144C160,148.969,153.74,154.838,143.663,159.316Zm0-48C131.064,116.916,114.137,120,96,120c-39.068,0-64-14.215-64-24V72.577C46.356,82.075,69.242,88,96,88s49.644-5.925,64-15.423V96C160,100.969,153.74,106.838,143.663,111.316ZM96,95.86a151.433,151.433,0,0,0,16-.83v16.1c-5.14.48-10.49.73-16,.73Zm47.66-8.69c.12-.05.23-.1.34-.15v17.27a113.137,113.137,0,0,1-16,4.47V92.34A90.144,90.144,0,0,0,143.66,87.17ZM96,144a151.433,151.433,0,0,0,16-.83v16.1c-5.14.48-10.49.73-16,.73Zm47.66-8.69c.12-.05.23-.1.34-.15v17.27a113.137,113.137,0,0,1-16,4.47v-16.42A90.144,90.144,0,0,0,143.66,135.312Z"></path>
|
||
</svg><div class="info"><p class="category-header font-bold font-national text-28 leading-24 pb-[5px]">Data</p><p class="category-tagline text-18 leading-22 m-0 font-medium"></p></div></div></div><div class="category-details flex justify-start gap-5"></div></div><div class="product-category h-full flex flex-col p-10 pl-[25px]" id="Logs-detail" x-show="openCategory === 'Logs'" x-cloak=""><div class="category-description p-7 rounded flex mb-10 text-white [background:var(--cta-bg)]" style="--cta-bg:linear-gradient(90deg, %!s(<nil>) 0%, %!s(<nil>) 100%)"><div class="category-description-main flex gap-5 flex-1 min-w-0 items-center"><svg class="w-12 h-auto fill-white shrink-0" width="48" height="48" viewBox="0 0 48 48" fill="" xmlns="http://www.w3.org/2000/svg">
|
||
<g id="Logs">
|
||
<path id="Vector" d="M28.1375 9.81131C21.6141 10.8028 16.3957 15.6567 14.7773 22.1801C13.4723 27.085 14.7773 31.678 17.0742 35.2269L9.87276 42.4283C8.56785 43.7333 8.56785 45.7163 9.87276 47.0212C11.1777 48.3262 13.1608 48.3262 14.4658 47.0212L21.6161 39.8198C25.2176 42.1167 29.809 43.4213 34.6629 42.1167C40.8728 40.4472 45.7792 34.9153 46.7707 28.3917C48.0737 17.6397 38.9403 8.50641 28.1375 9.81131ZM30.4344 37.5219C24.2245 37.5219 19.0063 32.3035 19.0063 26.0938C19.0063 19.9364 24.2247 14.7165 30.4344 14.7165C36.6441 14.7165 41.8625 19.9349 41.8625 26.0938C41.8638 32.3035 36.6439 37.5219 30.434 37.5219H30.4344ZM23.9106 6.52341H14.1518V0H23.9106V6.52341ZM36.9574 6.52341H27.1985V0H36.9574V6.52341ZM11.8553 32.617H1.10498V26.0936H10.8638C10.8638 28.39 11.1773 30.6865 11.8553 32.617ZM15.7702 13.0468C14.1518 15.0297 12.8483 17.3262 11.8568 19.5702H1.10498V13.0468H15.7702ZM10.8638 6.52341H1.10498V0H10.8638V6.52341ZM35.3404 29.3812H25.8951C25.8951 29.3812 25.5295 28.3897 25.5295 27.7627C25.3495 24.2183 28.5039 21.2393 32.0529 21.2393C33.3578 21.2393 34.3498 21.5528 35.3408 22.1783L35.3404 29.3812Z"></path>
|
||
</g>
|
||
</svg><div class="info"><p class="category-header font-bold font-national text-28 leading-24 pb-[5px]">Logs</p><p class="category-tagline text-18 leading-22 m-0 font-medium"></p></div></div></div><div class="category-details flex justify-start gap-5"></div></div><div class="product-category h-full flex flex-col p-10 pl-[25px]" id="security-detail" x-show="openCategory === 'security'" x-cloak=""><a class="category-description category-description-cta group p-7 rounded flex justify-between items-end mb-10 text-white no-underline [background:var(--cta-bg)] transition-[outline-color,box-shadow,background] duration-200 ease-in-out hover:[background:var(--cta-bg-hover,var(--cta-bg))] hover:outline hover:outline-2 hover:outline-[#8BD0FF] hover:outline-offset-[-1px] hover:shadow-[0_4px_16px_rgba(0,0,0,0.2)]" href="https://www.datadoghq.com/products/security/" data-dd-action-name="Category Banner security" style="--cta-bg:linear-gradient(90deg, #0060FF 0%, #0032A0 100%);--cta-bg-hover:linear-gradient(90deg, #0060FF 0%, #0032A0 50%, #001B58 100%)"><div class="category-description-main flex gap-5 flex-1 min-w-0 items-center"><svg class="w-12 h-auto fill-white shrink-0" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>security-platform</title>
|
||
<path d="M16 0.793l-12 6.938v9.604c0.007 4.42 2.392 8.281 5.944 10.372l0.056 0.031 6 3.468 6-3.469c3.608-2.122 5.993-5.983 6-10.402v-9.605zM6.667 10.811l8 4.625v4.42l-8-4.625zM7.997 8.5l6.669-3.856v7.711zM6.723 18.343l7.944 4.593v4.42l-3.333-1.926c-2.521-1.484-4.269-4.050-4.607-7.044l-0.004-0.043zM25.333 17.335c-0.005 3.439-1.86 6.443-4.623 8.070l-0.044 0.024-3.333 1.927v-22.712l8 4.625z"></path>
|
||
</svg><div class="info"><p class="category-header font-bold font-national text-28 leading-24 pb-[5px]">Security</p><p class="category-tagline text-18 leading-22 m-0 font-medium">Outpace AI-powered attacks with unified security and observability</p></div></div><span class="category-cta inline-flex items-center gap-2 transition-[gap] duration-200 ease-in-out group-hover:gap-3 pb-1.5 text-18 leading-18 font-bold font-national uppercase tracking-1 whitespace-nowrap text-white"><span class="category-cta-label">Learn more</span><svg class="w-2 h-auto fill-white" version="1.1" xmlns="http://www.w3.org/2000/svg" width="683" height="1024" viewBox="0 0 683 1024">
|
||
<title></title>
|
||
<g id="icomoon-ignore">
|
||
</g>
|
||
<path d="M391.327 512.001l-366.327-366.327 120.679-120.679 487.007 487.006-487.007 487.005-120.679-120.678 366.327-366.327z"></path>
|
||
</svg></span></a><div class="category-details flex justify-start gap-5"><div class="product-subcategory w-1/3"><p class="subcategory-header font-bold font-national uppercase text-16 leading-18 p-3">Code Security</p><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/code-security/" data-dd-action-name="code_security in security">Code Security</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/software-composition-analysis/" data-dd-action-name="software_composition_analysis in security">Software Composition Analysis</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/sast/" data-dd-action-name="sast in security">Static Code Analysis</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/iast/" data-dd-action-name="iast in security">Runtime Code Analysis</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/iac-security" data-dd-action-name="iac_security in security">IaC Security</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/secret-scanning/" data-dd-action-name="secret_scanning in security">Secret Scanning</a></li></ul></div><div class="product-subcategory w-1/3"><p class="subcategory-header font-bold font-national uppercase text-16 leading-18 p-3">Cloud Security</p><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/cloud-security/" data-dd-action-name="cloud_security in security">Cloud Security</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/sensitive-data-scanner/" data-dd-action-name="sensitive_data_scanner in security">Sensitive Data Scanner</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/cloud-security/#posture-management" data-dd-action-name="cloud_security_posture_management in security">Cloud Security Posture Management</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/cloud-security/#entitlement-management" data-dd-action-name="entitlement_management in security">Cloud Infrastructure Entitlement Management</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/cloud-security/#vulnerability-management" data-dd-action-name="vulnerability_management in security">Vulnerability Management</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/cloud-security/#compliance" data-dd-action-name="compliance in security">Compliance</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/workload-protection/" data-dd-action-name="workload_protection in security">Cloud Workload Protection</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/app-and-api-protection/" data-dd-action-name="app_and_api_protection in security">App & API Protection</a></li></ul></div><div class="product-subcategory w-1/3"><p class="subcategory-header font-bold font-national uppercase text-16 leading-18 p-3">SIEM</p><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/cloud-siem/" data-dd-action-name="cloud_siem in security">Cloud SIEM</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/ai/bits-security-analyst/" data-dd-action-name="bits_security_analyst in security">Bits Security Analyst</a></li></ul></div></div></div><div class="product-category h-full flex flex-col p-10 pl-[25px]" id="digital-experience-detail" x-show="openCategory === 'digital-experience'" x-cloak=""><div class="category-description p-7 rounded flex mb-10 text-white [background:var(--cta-bg)]" style="--cta-bg:linear-gradient(90deg, #00B765 0%, #00BBED 100%)"><div class="category-description-main flex gap-5 flex-1 min-w-0 items-center"><svg class="w-12 h-auto fill-white shrink-0" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>rum</title>
|
||
<path d="M8.728 32h-8.728v-14.544h8.728v14.544zM2.912 29.088h2.912v-8.728h-2.912v8.728zM18.908 29.088h-7.269v-2.912h4.364v-14.544h-10.187v2.912h-2.905v-5.816h16.003v20.361zM32 20.361h-10.18v-2.912h7.276v-14.544h-20.367v2.912h-2.912v-5.816h26.184v20.361z"></path>
|
||
</svg><div class="info"><p class="category-header font-bold font-national text-28 leading-24 pb-[5px]">Digital Experience</p><p class="category-tagline text-18 leading-22 m-0 font-medium">Optimize front-end performance and enhance user experiences</p></div></div></div><div class="category-details flex justify-start gap-5"><div class="product-subcategory w-1/3"><p class="subcategory-header font-bold font-national uppercase text-16 leading-18 p-3">Digital Experience</p><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/product-analytics/" data-dd-action-name="product_analytics in digital_experience">Product Analytics</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/experiments/" data-dd-action-name="experiments in digital_experience">Experiments</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/real-user-monitoring/" data-dd-action-name="browser_real_user_monitoring in digital_experience">Browser Real User Monitoring</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/real-user-monitoring/mobile-rum/" data-dd-action-name="mobile_real_user_monitoring in digital_experience">Mobile Real User Monitoring</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/real-user-monitoring/session-replay/" data-dd-action-name="session_replay in digital_experience">Session Replay</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/synthetic-monitoring/" data-dd-action-name="synthetic_monitoring in digital_experience">Synthetic Monitoring</a></li></ul></div><div class="product-subcategory w-1/3"><span class="subcategory-spacer block py-[21px]"></span><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/mobile-app-testing/" data-dd-action-name="mobile_app_testing in digital_experience">Mobile App Testing</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/continuous-testing/" data-dd-action-name="continuous_testing in digital_experience">Continuous Testing</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/error-tracking/" data-dd-action-name="error_tracking in digital_experience">Error Tracking</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/platform/dashboards/" data-dd-action-name="dashboards in digital_experience">Dashboards</a></li></ul></div></div></div><div class="product-category h-full flex flex-col p-10 pl-[25px]" id="software-delivery-detail" x-show="openCategory === 'software-delivery'" x-cloak=""><div class="category-description p-7 rounded flex mb-10 text-white [background:var(--cta-bg)]" style="--cta-bg:linear-gradient(90deg, #6300D4 0%, #9C43FE 100%)"><div class="category-description-main flex gap-5 flex-1 min-w-0 items-center"><svg class="w-12 h-auto fill-white shrink-0" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>ci</title>
|
||
<path d="M30.667 22.625v0.087c-0.006 4.392-3.565 7.95-7.956 7.955h-0.085c-3.937-0.005-7.205-2.864-7.845-6.62l-0.007-0.047h-3.107l4.333-4.333 4.333 4.333h-2.833c0.6 2.315 2.669 3.998 5.131 4h0.087c2.916-0.009 5.278-2.373 5.282-5.29v-0.085c-0.003-2.921-2.369-5.288-5.29-5.292h-13.42c-4.393-0.004-7.954-3.565-7.957-7.958v-0.087c0.006-4.392 3.565-7.95 7.956-7.955h0.085c4.393 0.004 7.954 3.564 7.958 7.956v2.044h-2.667v-2.043c-0.004-2.921-2.371-5.287-5.291-5.29h-0.087c-2.919 0.005-5.285 2.371-5.288 5.29v0.085c0.003 2.921 2.369 5.288 5.29 5.292h13.42c4.393 0.004 7.954 3.565 7.957 7.958v0z"></path>
|
||
</svg><div class="info"><p class="category-header font-bold font-national text-28 leading-24 pb-[5px]">Software Delivery</p><p class="category-tagline text-18 leading-22 m-0 font-medium">Build, test, secure and ship quality code faster</p></div></div></div><div class="category-details flex justify-start gap-5"><div class="product-subcategory w-1/3"><p class="subcategory-header font-bold font-national uppercase text-16 leading-18 p-3">Software Delivery</p><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/continuous-testing/" data-dd-action-name="continuous_testing in software_delivery">Continuous Testing</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/internal-developer-portal/" data-dd-action-name="internal_developer_portal in software_delivery">Internal Developer Portal (IDP)</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/ci-cd-monitoring/" data-dd-action-name="ci_visibility in software_delivery">CI Visibility</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/test-optimization/" data-dd-action-name="test_optimization in software_delivery">Test Optimization</a></li></ul></div><div class="product-subcategory w-1/3"><span class="subcategory-spacer block py-[21px]"></span><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/feature-flags/" data-dd-action-name="feature_flags in software_delivery">Feature Flags</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/code-coverage/" data-dd-action-name="code_coverage in software_delivery">Code Coverage</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/software-composition-analysis/" data-dd-action-name="software_composition_analysis in software_delivery">Software Composition Analysis</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/platform/dora-metrics/" data-dd-action-name="dora_metrics in software_delivery">DORA Metrics + AI Impact</a></li></ul></div></div></div><div class="product-category h-full flex flex-col p-10 pl-[25px]" id="service-management-detail" x-show="openCategory === 'service-management'" x-cloak=""><div class="category-description p-7 rounded flex mb-10 text-white [background:var(--cta-bg)]" style="--cta-bg:linear-gradient(90deg, #FF0080 0%, #FF5E00 100%)"><div class="category-description-main flex gap-5 flex-1 min-w-0 items-center"><svg class="w-12 h-auto fill-white shrink-0" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M88,188.38V91.38l84-48.5v96.99l-84,48.51Zm16-87.76v60.04l52-30.02v-60.04l-52,30.02Zm-50-28.87v97l16-9.24V80.99l52-30.02,16-9.24V23.25L54,71.75ZM20,52.12v97l16-9.24V61.36l52-30.02,16-9.24V3.62L20,52.12Z"></path>
|
||
</svg><div class="info"><p class="category-header font-bold font-national text-28 leading-24 pb-[5px]">Service Management</p><p class="category-tagline text-18 leading-22 m-0 font-medium">Integrated, streamlined workflows for faster time-to-resolution</p></div></div></div><div class="category-details flex justify-start gap-5"><div class="product-subcategory w-1/3"><p class="subcategory-header font-bold font-national uppercase text-16 leading-18 p-3">Incident Response</p><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/incident-response/" data-dd-action-name="incident_management in service_management">Incident Management</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/case-management/" data-dd-action-name="work_management in service_management">Work Management</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/service-level-objectives/" data-dd-action-name="service_level_objectives in service_management">Service Level Objectives</a></li></ul></div><div class="product-subcategory w-1/3"><p class="subcategory-header font-bold font-national uppercase text-16 leading-18 p-3">Actions</p><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/workflow-automation/" data-dd-action-name="workflow_automation in service_management">Workflow Automation</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/app-builder/" data-dd-action-name="dd_apps in service_management">Datadog Apps</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/event-management/" data-dd-action-name="event_management in service_management">Event Management</a></li></ul></div><div class="product-subcategory w-1/3"><p class="subcategory-header font-bold font-national uppercase text-16 leading-18 p-3">Context Graph</p><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/internal-developer-portal/" data-dd-action-name="internal_developer_portal in service_management">Internal Developer Portal (IDP)</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/software-catalog/" data-dd-action-name="software_catalog in service_management">Software Catalog</a></li></ul></div></div></div><div class="product-category h-full flex flex-col p-10 pl-[25px]" id="ai-detail" x-show="openCategory === 'ai'" x-cloak=""><div class="category-description p-7 rounded flex mb-10 text-white [background:var(--cta-bg)]" style="--cta-bg:linear-gradient(90deg, #0060FF 0%, #8000FF 100%)"><div class="category-description-main flex gap-5 flex-1 min-w-0 items-center"><svg class="w-12 h-auto fill-white shrink-0" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" fill="" viewBox="0 0 192 192" style="enable-background:new 0 0 192 192;" xml:space="preserve"><g id="Layer_7"><g><linearGradient id="SVGID_1_" gradientUnits="userSpaceOnUse" x1="89.0192" y1="122.732" x2="142.9808" y2="29.268"><stop offset="0" style="stop-color:#5500FF"></stop><stop offset="0.1715" style="stop-color:#4D09FF"></stop><stop offset="0.4511" style="stop-color:#3821FF"></stop><stop offset="0.802" style="stop-color:#1649FF"></stop><stop offset="1" style="stop-color:#0062FF"></stop></linearGradient><path class="st5" d="M120,16h-8c0,30.88-25.12,56-56,56v8c30.88,0,56,25.12,56,56h8c0-30.88,25.12-56,56-56v-8 C145.12,72,120,46.88,120,16z"></path><linearGradient id="SVGID_00000049207151058930698180000011839073544639225000_" gradientUnits="userSpaceOnUse" x1="35.4115" y1="168.7311" x2="68.5885" y2="111.267"><stop offset="0" style="stop-color:#5500FF"></stop><stop offset="0.1715" style="stop-color:#4D09FF"></stop><stop offset="0.4511" style="stop-color:#3821FF"></stop><stop offset="0.802" style="stop-color:#1649FF"></stop><stop offset="1" style="stop-color:#0062FF"></stop></linearGradient><path d="M56,104h-4c0,0,0,0,0,0c0,0,0,0,0,0h-4 c0,17.64-14.35,32-32,32v8c17.65,0,32,14.36,32,32h8c0-17.64,14.35-32,32-32v-8C70.35,136,56,121.64,56,104z"></path></g></g></svg><div class="info"><p class="category-header font-bold font-national text-28 leading-24 pb-[5px]">AI</p><p class="category-tagline text-18 leading-22 m-0 font-medium">Monitor and improve model performance. Pinpoint root causes and detect anomalies</p></div></div></div><div class="category-details flex justify-start gap-5"><div class="product-subcategory w-1/3"><p class="subcategory-header font-bold font-national uppercase text-16 leading-18 p-3">AI for Datadog</p><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/ai/bits-investigation/" data-dd-action-name="bits_investigation in ai">Bits Investigation</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/ai/bits-ai-agents/" data-dd-action-name="bits_ai in ai">Bits AI</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/ai/bits-chat/" data-dd-action-name="bits_chat in ai">Bits Chat</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/ai/bits-agent-builder/" data-dd-action-name="bits_agent_builder in ai">Bits Agent Builder</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/ai/bits-code/" data-dd-action-name="bits_code in ai">Bits Code</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/ai/bits-security-analyst/" data-dd-action-name="bits_security_analyst in ai">Bits Security Analyst</a></li></ul></div><div class="product-subcategory w-1/3"><p class="subcategory-header font-bold font-national uppercase text-16 leading-18 p-3">Datadog for AI</p><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/products/ai/agent-observability/" data-dd-action-name="agent_observability in ai">Agent Observability</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/ai/mcp-server/" data-dd-action-name="mcp_server in ai">MCP Server</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/ai/mcp-server/" data-dd-action-name="pup_cli in ai">Pup CLI</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/gpu-monitoring/" data-dd-action-name="gpu_monitoring in ai">GPU Monitoring</a></li></ul></div></div></div><div class="product-category h-full flex flex-col p-10 pl-[25px]" id="platform-detail" x-show="openCategory === 'platform'" x-cloak=""><div class="category-description p-7 rounded flex mb-10 text-white [background:var(--cta-bg)]" style="--cta-bg:linear-gradient(90deg, #8000FF 0%, #4300B0 100%)"><div class="category-description-main flex gap-5 flex-1 min-w-0 items-center"><svg class="w-12 h-auto fill-white shrink-0" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>dashboard</title>
|
||
<path d="M32 24.876v1.79h-32v-21.333h1.988v19.543h30.012zM26.009 10.679l3.976 12.432h-25.982v-8.006l6.999-7.981 8.987 7.981 6.020-4.426z"></path>
|
||
</svg><div class="info"><p class="category-header font-bold font-national text-28 leading-24 pb-[5px]">Platform Capabilities</p><p class="category-tagline text-18 leading-22 m-0 font-medium">Built-in features & integrations that power the Datadog platform</p></div></div></div><div class="category-details flex justify-start gap-5"><div class="product-subcategory w-1/3"><p class="subcategory-header font-bold font-national uppercase text-16 leading-18 p-3">Built-in Features</p><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/ai/bits-ai-agents/" data-dd-action-name="bits_ai in platform_capabilities">Bits AI</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/ai/bits-chat/" data-dd-action-name="bits_chat in platform_capabilities">Bits Chat</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/ai/bits-agent-builder/" data-dd-action-name="bits_agent_builder in platform_capabilities">Bits Agent Builder</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/platform/alerts/" data-dd-action-name="alerts in platform_capabilities">Alerts</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/metrics/" data-dd-action-name="custom-metrics in platform_capabilities">Custom Metrics</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/platform/dashboards/" data-dd-action-name="dashboards in platform_capabilities">Dashboards</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://docs.datadoghq.com/notebooks/" data-dd-action-name="notebooks in platform_capabilities">Notebooks</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://docs.datadoghq.com/service_management/mobile/?tab=ios" data-dd-action-name="mobile_app in platform_capabilities">Mobile App</a></li></ul></div><div class="product-subcategory w-1/3"><span class="subcategory-spacer block py-[21px]"></span><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/fleet-automation/" data-dd-action-name="fleet_automation in platform_capabilities">Fleet Automation</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://docs.datadoghq.com/account_management/rbac/?tab=datadogapplication" data-dd-action-name="access_control in platform_capabilities">Access Control</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/governance-console/" data-dd-action-name="governance_console in platform_capabilities">Governance Console</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/platform/dora-metrics/" data-dd-action-name="dora_metrics in platform_capabilities">DORA Metrics + AI Impact</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/incident-response/" data-dd-action-name="incident_management in platform_capabilities">Incident Management</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/case-management/" data-dd-action-name="work_management in platform_capabilities">Work Management</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/workflow-automation/" data-dd-action-name="workflow_automation in platform_capabilities">Workflow Automation</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://docs.datadoghq.com/account_management/teams/" data-dd-action-name="teams in platform_capabilities">Teams</a></li></ul></div><div class="product-subcategory w-1/3"><p class="subcategory-header font-bold font-national uppercase text-16 leading-18 p-3">Extensibility</p><ul class="product-list"><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/solutions/opentelemetry/" data-dd-action-name="opentelemetry in platform_capabilities">OpenTelemetry</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/platform/integrations/" data-dd-action-name="integrations in platform_capabilities">Integrations</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/platform/ides/" data-dd-action-name="ide_plugins in platform_capabilities">IDE Plugins</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/ai/mcp-server/" data-dd-action-name="mcp_server in platform_capabilities">MCP Server</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/ai/agent-directory/" data-dd-action-name="agent_directory in platform_capabilities">Agent Directory</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/app-builder/" data-dd-action-name="dd_apps in platform_capabilities">Datadog Apps</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://docs.datadoghq.com/api/" data-dd-action-name="api in platform_capabilities">API</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/marketplacepartners/" data-dd-action-name="marketplace in platform_capabilities">Marketplace</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/ai/mcp-server/" data-dd-action-name="pup_cli in platform_capabilities">Pup CLI</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/software-catalog/" data-dd-action-name="software_catalog in platform_capabilities">Software Catalog</a></li><li><a class="block p-3 text-18 leading-18 text-wrap capitalize hover:text-[#70f] hover:bg-black/5 rounded" href="https://www.datadoghq.com/product/service-level-objectives/" data-dd-action-name="service_level_objectives in platform_capabilities">Service Level Objectives</a></li></ul></div></div></div></div></div></div></li><li><a class="customers-menu block pt-[8.5px] px-2 desktop-lg:px-3 pb-[9.5px] desktop-lg:px-3 text-18 font-semibold font-national uppercase text-[#555] hover:text-[#70f]" href="https://www.datadoghq.com/customers/"><span>Customers</span></a></li><li><a class="pricing-menu block pt-[8.5px] px-2 desktop-lg:px-3 pb-[9.5px] desktop-lg:px-3 text-18 font-semibold font-national uppercase text-[#555] hover:text-[#70f]" href="https://www.datadoghq.com/pricing/"><span>Pricing</span></a></li><li x-on:mouseover="mouseOver = 'solutions'" x-on:mouseout="mouseOver = 'none'" class="dropdown relative solutions-dropdown"><a class="dropdown relative block pt-[8.5px] px-2 desktop-lg:px-3 pb-[9.5px] desktop-lg:px-3 text-18 font-semibold font-national uppercase text-[#555] hover:text-[#70f]" href="https://www.datadoghq.com/">Solutions</a><ul x-show="mouseOver === 'solutions'" x-cloak="" class="dropdown-menu solutions-menu absolute z-40 bg-white w-[820px] p-5 left-[9px] top-[42px] border-t-[3px] border-t-[#632ca6]" style="box-shadow:0 8px 10px rgba(0,0,0,7%)"><div class="grid grid-cols-3 gap-[30px]"><div><p class="font-semibold font-national text-black text-14 leading-18 tablet:text-16 tablet:leading-20 uppercase mb-0 py-2 px-[11px]">Industry</p><ul class="list-none mt-0"><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/financial-services/" data-dd-action-name="financial_services in Solutions"><span class="inline-flex">Financial Services</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/manufacturing-logistics/" data-dd-action-name="manufacturing_and_logistics in Solutions"><span class="inline-flex">Manufacturing & Logistics</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/healthcare/" data-dd-action-name="healthcare_and_life_sciences in Solutions"><span class="inline-flex">Healthcare/Life Sciences</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/retail-ecommerce/" data-dd-action-name="retail_and_ecommerce in Solutions"><span class="inline-flex">Retail/E-Commerce</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/government/" data-dd-action-name="government in Solutions"><span class="inline-flex">Government</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/education/" data-dd-action-name="education in Solutions"><span class="inline-flex">Education</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/media-entertainment/" data-dd-action-name="media_and_entertainment in Solutions"><span class="inline-flex">Media & Entertainment</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/technology/" data-dd-action-name="technology in Solutions"><span class="inline-flex">Technology</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/gaming/" data-dd-action-name="gaming in Solutions"><span class="inline-flex">Gaming</span></a></li></ul></div><div><p class="font-semibold font-national text-black text-14 leading-18 tablet:text-16 tablet:leading-20 uppercase mb-0 py-2 px-[11px]">Technology</p><ul class="list-none mt-0"><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/aws/" data-dd-action-name="amazon_web_services_monitoring in Solutions"><span class="inline-flex">Amazon Web Services Monitoring</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/azure/" data-dd-action-name="azure_monitoring in Solutions"><span class="inline-flex">Azure Monitoring</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/googlecloud/" data-dd-action-name="google_cloud_monitoring in Solutions"><span class="inline-flex">Google Cloud Monitoring</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/oci-monitoring/" data-dd-action-name="oci_monitoring in Solutions"><span class="inline-flex">Oracle Cloud Monitoring</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/kubernetes/" data-dd-action-name="kubernetes_monitoring in Solutions"><span class="inline-flex">Kubernetes Monitoring</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/openshift/" data-dd-action-name="red_hat_openshift in Solutions"><span class="inline-flex">Red Hat OpenShift</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/pivotal-platform/" data-dd-action-name="pivotal_platform in Solutions"><span class="inline-flex">Pivotal Platform</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/openai/" data-dd-action-name="openai in Solutions"><span class="inline-flex">OpenAI</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/sap-monitoring/" data-dd-action-name="sap_monitoring in Solutions"><span class="inline-flex">SAP Monitoring</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/opentelemetry/" data-dd-action-name="opentelemetry in Solutions"><span class="inline-flex">OpenTelemetry</span></a></li></ul></div><div><p class="font-semibold font-national text-black text-14 leading-18 tablet:text-16 tablet:leading-20 uppercase mb-0 py-2 px-[11px]">Use Case</p><ul class="list-none mt-0"><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/application-security/" data-dd-action-name="application_security in Solutions"><span class="inline-flex">Application Security</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/cloud-migration/" data-dd-action-name="cloud_migration in Solutions"><span class="inline-flex">Cloud Migration</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/monitoring-consolidation/" data-dd-action-name="monitoring_consolidation in Solutions"><span class="inline-flex">Monitoring Consolidation</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/unified-commerce-monitoring/" data-dd-action-name="unified_commerce_monitoring in Solutions"><span class="inline-flex">Unified Commerce Monitoring</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/soar/" data-dd-action-name="soar in Solutions"><span class="inline-flex">SOAR</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/devops/" data-dd-action-name="devops in Solutions"><span class="inline-flex">DevOps</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/finops/" data-dd-action-name="finops in Solutions"><span class="inline-flex">FinOps</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/shift-left-testing/" data-dd-action-name="shift_left_testing in Solutions"><span class="inline-flex">Shift-Left Testing</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/digital-experience-monitoring/" data-dd-action-name="digital_experience_monitoring in Solutions"><span class="inline-flex">Digital Experience Monitoring</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/security-analytics/" data-dd-action-name="security_analytics in Solutions"><span class="inline-flex">Security Analytics</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/security/cis-benchmarks/aws/" data-dd-action-name="compliance_for_cis_benchmarks in Solutions"><span class="inline-flex">Compliance for CIS Benchmarks</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/hybrid-cloud-monitoring/" data-dd-action-name="hybrid_cloud_monitoring in Solutions"><span class="inline-flex">Hybrid Cloud Monitoring</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/iot-monitoring/" data-dd-action-name="iot_monitoring in Solutions"><span class="inline-flex">Edge Device Monitoring</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/real-time-business-intelligence/" data-dd-action-name="real_time_bi in Solutions"><span class="inline-flex">Real-Time BI</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/on-premises-monitoring/" data-dd-action-name="on_premises_monitoring in Solutions"><span class="inline-flex">On-Premises Monitoring</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/log-analysis-and-correlation/" data-dd-action-name="log_analysis_and_correlation in Solutions"><span class="inline-flex">Log Analysis & Correlation</span></a></li><li><a class="block whitespace-nowrap capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18" href="https://www.datadoghq.com/solutions/cnapp/" data-dd-action-name="cnapp in Solutions"><span class="inline-flex">CNAPP</span></a></li></ul></div></div></ul></li><li><a class="docs-menu block pt-[8.5px] px-2 desktop-lg:px-3 pb-[9.5px] desktop-lg:px-3 text-18 font-semibold font-national uppercase text-[#555] hover:text-[#70f]" href="https://docs.datadoghq.com/"><span>Docs</span></a></li></ul><div><a href="https://www.datadoghq.com/" class="js-logo-download hidden desktop-sm:flex flex-col items-center justify-center relative"><img height="75" class="bits-logo-lg max-w-full transition-all h-[75px]" :class="navScrolled ? '!h-[50px]' : ''" src="https://corp.dd-static.net/img/dd_logo_n_70x75.png?ch=Width,DPR&fit=max&auto=format&w=70&h=75" srcset="https://corp.dd-static.net/img/dd_logo_n_70x75.png?ch=Width,DPR&fit=max&auto=format&w=70&h=75 1x, https://corp.dd-static.net/img/dd_logo_n_70x75.png?ch=Width,DPR&fit=max&auto=format&w=70&h=75&dpr=2 2x" alt="DataDog">
|
||
<img class="datadog-text-logo-lg w-[93px] mt-[5px] h-auto max-w-full block" :class="navScrolled ? 'hidden' : ''" height="14" src="https://corp.dd-static.net/img/dd-logo-n-200.png?ch=Width,DPR&fit=max&auto=format&h=14&auto=format&w=807" srcset="https://corp.dd-static.net/img/dd-logo-n-200.png?ch=Width,DPR&fit=max&auto=format&h=14&auto=format&w=807 1x, https://corp.dd-static.net/img/dd-logo-n-200.png?ch=Width,DPR&fit=max&auto=format&h=14&auto=format&w=807&dpr=2 2x" alt="DataDog"></a></div><ul x-data="{ mouseOver: 'none' }" class="flex flex-row items-center list-none m-0 justify-end grow basis-0"><li x-on:mouseover="mouseOver = 'about'" x-on:mouseout="mouseOver = 'none'" class="dropdown relative"><a href="https://www.datadoghq.com/about/leadership/" class="dropdown block pt-[8.5px] px-2 desktop-lg:px-3 pb-[9.5px] desktop-lg:px-3 font-semibold font-national text-18 uppercase text-[#555] hover:text-[#70f]">About</a><ul x-show="mouseOver === 'about'" x-cloak="" class="dropdown-menu absolute z-40 bg-white p-5 left-[9px] top-[42px] border-t-[3px] border-t-[#632ca6]" style="box-shadow:0 8px 10px rgba(0,0,0,7%)" aria-labelledby="dropdownMenuButton"><li><a class="block capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18 text-nowrap" href="https://www.datadoghq.com/about/contact/"><span>Contact</span></a></li><li><a class="block capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18 text-nowrap" href="https://www.datadoghq.com/partner/network/"><span>Partners</span></a></li><li><a class="block capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18 text-nowrap" href="https://www.datadoghq.com/about/latest-news/press-releases/"><span>Newsroom</span></a></li><li><a class="block capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18 text-nowrap" href="https://www.datadoghq.com/events-webinars/"><span>Events & Webinars</span></a></li><li><a class="block capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18 text-nowrap" href="https://www.datadoghq.com/about/leadership/"><span>Leadership</span></a></li><li><a class="block capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18 text-nowrap" href="https://careers.datadoghq.com/"><span>Careers</span></a></li><li><a class="block capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18 text-nowrap" href="https://www.datadoghq.com/about/analyst/"><span>Analyst Reports</span></a></li><li><a class="block capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18 text-nowrap" href="https://investors.datadoghq.com/"><span>Investor Relations</span></a></li><li><a class="block capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18 text-nowrap" href="https://www.datadoghq.com/esg-report/"><span>ESG Report</span></a></li><li><a class="block capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18 text-nowrap" href="https://www.datadoghq.com/trust/"><span>Trust Hub</span></a></li></ul></li><li x-on:mouseover="mouseOver = 'blog'" x-on:mouseout="mouseOver = 'none'" class="dropdown relative"><a href="https://www.datadoghq.com/blog/" class="dropdown block pt-[8.5px] px-2 desktop-lg:px-3 pb-[9.5px] desktop-lg:px-3 font-semibold font-national text-18 uppercase text-[#555] hover:text-[#70f]">Blog</a><ul x-show="mouseOver === 'blog'" x-cloak="" class="dropdown-menu absolute z-40 bg-white p-5 left-[9px] top-[42px] border-t-[3px] border-t-[#632ca6]" style="box-shadow:0 8px 10px rgba(0,0,0,7%)" aria-labelledby="dropdownMenuButton"><li><a class="block capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18 text-nowrap" href="https://www.datadoghq.com/blog/"><span>The Monitor</span></a></li><li><a class="block capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18 text-nowrap" href="https://www.datadoghq.com/blog/engineering/"><span>Engineering</span></a></li><li><a class="block capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18 text-nowrap" href="https://www.datadoghq.com/blog/ai/"><span>AI</span></a></li><li><a class="block capitalize px-[11px] py-[7px] hover:text-[#70f] hover:bg-black/5 rounded text-18 text-nowrap" href="https://securitylabs.datadoghq.com/"><span>Security Labs</span></a></li></ul></li><li class="final"><a class="font-semibold font-national block pt-[8.5px] px-2 desktop-lg:px-3 pb-[9.5px] desktop-lg:px-3 text-18 uppercase text-[#555] hover:text-[#70f] transition-all" href="https://app.datadoghq.com/" data-event-category="Signup" data-event-action="Step 1 Top Right Initiated" data-event-label=" {{url path}}" style="position:relative"><span>Login</span></a></li><li class="final"><a class="font-semibold font-national sign-up-trigger btn btn-gradient js-cta-topRight pt-[8.5px] px-7 pb-[9.5px] block text-18 text-[#555] uppercase border-none border-0 border-[transparent] bg-white rounded min-w-[120px] ml-2.5 hover:text-white transition-all" href="https://www.datadoghq.com/" data-event-category="Signup" data-event-action="Step 1 Top Right Initiated" data-event-label=" {{url path}}" style="position:relative"><span class="hidden desktop-sm:block">GET STARTED FREE</span>
|
||
<span class="block desktop-sm:hidden">Free Trial</span></a></li></ul></div><div class="flex desktop-sm:hidden items-center justify-center"><a href="https://www.datadoghq.com/"><img src="https://corp.dd-static.net/img/datadog_rbg_n_2x.png?format=png" class="bits-logo-mobile d-inline-block" alt="Datadog Logo" width="157" height="40"></a></div><button x-on:click="mobileOpen = !mobileOpen" type="button" class="navbar-toggler flex flex-col gap-1 desktop-sm:hidden relative my-3 px-2.5 py-[9px] border border-[transparent]" data-bs-toggle="" data-bs-target="#collapseExample" aria-expanded="false" aria-controls="collapseExample">
|
||
<span class="visually-hidden sr-only">Toggle navigation</span>
|
||
<span class="icon-bar transition-all block w-[22px] h-0.5 bg-[#555]" :class="mobileOpen ? 'rotate-45 translate-x-1.5 translate-y-1.5' : ''"></span>
|
||
<span class="icon-bar transition-all block w-[22px] h-0.5 bg-[#555]" :class="mobileOpen ? 'opacity-0' : 'opacity-100'"></span>
|
||
<span class="icon-bar transition-all block w-[22px] h-0.5 bg-[#555]" :class="mobileOpen ? '-rotate-45 translate-x-1.5 -translate-y-1.5' : ''"></span></button></div></nav><div x-show="mobileOpen" x-cloak="" @click="mobileOpen = false" id="mobile-nav-bg" class="desktop-sm:hidden transition-all fixed top-0 left-0 w-full h-full bg-black z-50" :class="mobileOpen ? 'opacity-25' : 'opacity-0'"></div><div x-cloak="" id="mobile-nav" class="desktop-sm:hidden transition-all w-[400px] pt-[90px] bg-white fixed top-0 overflow-y-auto h-full z-60 pb-5 font-normal" :class="mobileOpen ? 'right-0' : 'right-[-500px]'"><ul><li x-on:click="dropdownOpen === 'product' ? dropdownOpen = 'none' : dropdownOpen = 'product'" class="border-b border-b-[#c7c7c7] text-18 dropdown"><a href="#" data-astro-reload="" :class="dropdownOpen === 'product' ? 'text-[#632ca6]' : 'text-[#555]'" class="dropdown-toggle block font-semibold font-national uppercase text-[#555] hover:text-[#632ca6] whitespace-nowrap" data-bs-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="true" data-dd-action-name="Product Dropdown on Mobile"><div class="nav-menu-item ml-5 py-3 pl-[14px]">Product
|
||
<span class="caret absolute right-[22px] inline-flex h-[25px]"><svg viewBox="0 0 17.78 10.51" width="18px" class="inline-block transition-all" :class="dropdownOpen === 'product' ? 'rotate-180' : 'rotate-0'">
|
||
<polyline class="dropdown-arrow fill-none stroke-[#555]" style="stroke-width: 2px;" points="16.92 1.14 8.92 9.14 0.92 1.14"></polyline>
|
||
</svg></span></div></a><ul x-cloak="" x-show="dropdownOpen === 'product'" class="dropdown-menu" id="product-mobile-menu"><li><p class="mobile-product-category sticky top-0 bg-white flex items-center gap-2 pt-[13px] pb-[5px] pl-[19px] font-bold font-national text-22 leading-24 ml-5"><svg class="w-[22px] h-auto" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>host-map</title>
|
||
<path d="M14.214 13.103l-7.107 4.103-7.107-4.103v-8.206l7.107-4.103 7.107 4.103zM32 4.897v8.206l-7.107 4.103-7.107-4.103v-8.206l7.107-4.103zM29.333 6.436l-4.44-2.563-4.44 2.563v5.127l4.44 2.563 4.44-2.563zM9.037 19.98v8.040l6.963 4.020 6.962-4.020v-8.040l-6.963-4.020z"></path>
|
||
</svg>
|
||
Infrastructure</p><ul class="mb-[5px] pb-2.5"><li><a href="https://www.datadoghq.com/product/infrastructure-monitoring/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Infrastructure Monitoring</a></li><li><a href="https://www.datadoghq.com/product/metrics/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Custom Metrics</a></li><li><a href="https://www.datadoghq.com/product/network-monitoring/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Network Monitoring</a></li><li><a href="https://www.datadoghq.com/product/container-monitoring/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Container Monitoring</a></li><li><a href="https://www.datadoghq.com/product/kubernetes-autoscaling/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Kubernetes Autoscaling</a></li><li><a href="https://www.datadoghq.com/product/serverless-monitoring/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Serverless Monitoring</a></li><li><a href="https://www.datadoghq.com/product/cloud-cost-management/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Cloud Cost Management</a></li><li><a href="https://www.datadoghq.com/product/storage-management/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Storage Management</a></li><li><a href="https://www.datadoghq.com/product/gpu-monitoring/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">GPU Monitoring</a></li></ul></li><li><p class="mobile-product-category sticky top-0 bg-white flex items-center gap-2 pt-[13px] pb-[5px] pl-[19px] font-bold font-national text-22 leading-24 ml-5"><svg class="w-[22px] h-auto" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>apm</title>
|
||
<path d="M0 2.667h32v3.81h-32v-3.81zM0 10.286h6v3.81h-6v-3.81zM10 10.286h22v3.81h-22v-3.81zM10 17.905h12v3.81h-12v-3.81zM26 17.905h6v3.81h-6v-3.81zM16 25.524h6v3.81h-6v-3.81z"></path>
|
||
</svg>
|
||
Applications</p><ul class="mb-[5px] pb-2.5"><li><a href="https://www.datadoghq.com/product/apm/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Application Performance Monitoring</a></li><li><a href="https://www.datadoghq.com/product/universal-service-monitoring/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Universal Service Monitoring</a></li><li><a href="https://www.datadoghq.com/product/code-profiling/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Continuous Profiler</a></li><li><a href="https://www.datadoghq.com/product/dynamic-instrumentation/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Dynamic Instrumentation</a></li><li><a href="https://www.datadoghq.com/products/ai/agent-observability/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Agent Observability</a></li></ul></li><li><p class="mobile-product-category sticky top-0 bg-white flex items-center gap-2 pt-[13px] pb-[5px] pl-[19px] font-bold font-national text-22 leading-24 ml-5"><svg class="w-[22px] h-auto" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M96,8C50.393,8,16,25.2,16,48v96c0,22.8,34.393,40,80,40s80-17.2,80-40V48C176,25.2,141.607,8,96,8ZM48.337,32.683C60.936,27.083,77.863,24,96,24c39.068,0,64,14.214,64,24,0,4.969-6.26,10.838-16.337,15.317C131.064,68.917,114.138,72,96,72,56.932,72,32,57.786,32,48,32,43.031,38.26,37.162,48.337,32.683Zm95.326,126.633C131.064,164.916,114.137,168,96,168c-39.068,0-64-14.215-64-24V120.577C46.356,130.075,69.242,136,96,136s49.644-5.925,64-15.423V144C160,148.969,153.74,154.838,143.663,159.316Zm0-48C131.064,116.916,114.137,120,96,120c-39.068,0-64-14.215-64-24V72.577C46.356,82.075,69.242,88,96,88s49.644-5.925,64-15.423V96C160,100.969,153.74,106.838,143.663,111.316ZM96,95.86a151.433,151.433,0,0,0,16-.83v16.1c-5.14.48-10.49.73-16,.73Zm47.66-8.69c.12-.05.23-.1.34-.15v17.27a113.137,113.137,0,0,1-16,4.47V92.34A90.144,90.144,0,0,0,143.66,87.17ZM96,144a151.433,151.433,0,0,0,16-.83v16.1c-5.14.48-10.49.73-16,.73Zm47.66-8.69c.12-.05.23-.1.34-.15v17.27a113.137,113.137,0,0,1-16,4.47v-16.42A90.144,90.144,0,0,0,143.66,135.312Z"></path>
|
||
</svg>
|
||
Data</p><ul class="mb-[5px] pb-2.5"><li><a href="https://www.datadoghq.com/product/database-monitoring/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Database Monitoring</a></li><li><a href="https://www.datadoghq.com/products/observability/data-observability/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Data Observability</a></li><li><a href="https://www.datadoghq.com/product/data-streams-monitoring/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Data Streams Monitoring</a></li></ul></li><li><p class="mobile-product-category sticky top-0 bg-white flex items-center gap-2 pt-[13px] pb-[5px] pl-[19px] font-bold font-national text-22 leading-24 ml-5"><svg class="w-[22px] h-auto" width="48" height="48" viewBox="0 0 48 48" fill="" xmlns="http://www.w3.org/2000/svg">
|
||
<g id="Logs">
|
||
<path id="Vector" d="M28.1375 9.81131C21.6141 10.8028 16.3957 15.6567 14.7773 22.1801C13.4723 27.085 14.7773 31.678 17.0742 35.2269L9.87276 42.4283C8.56785 43.7333 8.56785 45.7163 9.87276 47.0212C11.1777 48.3262 13.1608 48.3262 14.4658 47.0212L21.6161 39.8198C25.2176 42.1167 29.809 43.4213 34.6629 42.1167C40.8728 40.4472 45.7792 34.9153 46.7707 28.3917C48.0737 17.6397 38.9403 8.50641 28.1375 9.81131ZM30.4344 37.5219C24.2245 37.5219 19.0063 32.3035 19.0063 26.0938C19.0063 19.9364 24.2247 14.7165 30.4344 14.7165C36.6441 14.7165 41.8625 19.9349 41.8625 26.0938C41.8638 32.3035 36.6439 37.5219 30.434 37.5219H30.4344ZM23.9106 6.52341H14.1518V0H23.9106V6.52341ZM36.9574 6.52341H27.1985V0H36.9574V6.52341ZM11.8553 32.617H1.10498V26.0936H10.8638C10.8638 28.39 11.1773 30.6865 11.8553 32.617ZM15.7702 13.0468C14.1518 15.0297 12.8483 17.3262 11.8568 19.5702H1.10498V13.0468H15.7702ZM10.8638 6.52341H1.10498V0H10.8638V6.52341ZM35.3404 29.3812H25.8951C25.8951 29.3812 25.5295 28.3897 25.5295 27.7627C25.3495 24.2183 28.5039 21.2393 32.0529 21.2393C33.3578 21.2393 34.3498 21.5528 35.3408 22.1783L35.3404 29.3812Z"></path>
|
||
</g>
|
||
</svg>
|
||
Logs</p><ul class="mb-[5px] pb-2.5"><li><a href="https://www.datadoghq.com/product/log-management/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Log Management</a></li><li><a href="https://www.datadoghq.com/product/audit-trail/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Audit Trail</a></li><li><a href="https://www.datadoghq.com/product/observability-pipelines/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Observability Pipelines</a></li><li><a href="https://www.datadoghq.com/product/error-tracking/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Error Tracking</a></li><li><a href="https://www.datadoghq.com/product/byoc-logs/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Bring Your Own Cloud Log Management</a></li><li><a href="https://www.datadoghq.com/product/sensitive-data-scanner/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Sensitive Data Scanner</a></li></ul></li><li><p class="mobile-product-category sticky top-0 bg-white flex items-center gap-2 pt-[13px] pb-[5px] pl-[19px] font-bold font-national text-22 leading-24 ml-5"><svg class="w-[22px] h-auto" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>security-platform</title>
|
||
<path d="M16 0.793l-12 6.938v9.604c0.007 4.42 2.392 8.281 5.944 10.372l0.056 0.031 6 3.468 6-3.469c3.608-2.122 5.993-5.983 6-10.402v-9.605zM6.667 10.811l8 4.625v4.42l-8-4.625zM7.997 8.5l6.669-3.856v7.711zM6.723 18.343l7.944 4.593v4.42l-3.333-1.926c-2.521-1.484-4.269-4.050-4.607-7.044l-0.004-0.043zM25.333 17.335c-0.005 3.439-1.86 6.443-4.623 8.070l-0.044 0.024-3.333 1.927v-22.712l8 4.625z"></path>
|
||
</svg>
|
||
Security</p><ul class="mb-[5px] pb-2.5"><li><a href="https://www.datadoghq.com/products/security/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Overview</a></li><li><a href="https://www.datadoghq.com/product/code-security/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Code Security</a></li><li><a href="https://www.datadoghq.com/product/software-composition-analysis/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Software Composition Analysis</a></li><li><a href="https://www.datadoghq.com/product/sast/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Static Code Analysis</a></li><li><a href="https://www.datadoghq.com/product/iast/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Runtime Code Analysis</a></li><li><a href="https://www.datadoghq.com/product/iac-security" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">IaC Security</a></li><li><a href="https://www.datadoghq.com/product/secret-scanning/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Secret Scanning</a></li><li><a href="https://www.datadoghq.com/product/cloud-security/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Cloud Security</a></li><li><a href="https://www.datadoghq.com/product/sensitive-data-scanner/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Sensitive Data Scanner</a></li><li><a href="https://www.datadoghq.com/product/cloud-security/#posture-management" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Cloud Security Posture Management</a></li><li><a href="https://www.datadoghq.com/product/cloud-security/#entitlement-management" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Cloud Infrastructure Entitlement Management</a></li><li><a href="https://www.datadoghq.com/product/cloud-security/#vulnerability-management" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Vulnerability Management</a></li><li><a href="https://www.datadoghq.com/product/cloud-security/#compliance" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Compliance</a></li><li><a href="https://www.datadoghq.com/product/workload-protection/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Cloud Workload Protection</a></li><li><a href="https://www.datadoghq.com/product/app-and-api-protection/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">App & API Protection</a></li><li><a href="https://www.datadoghq.com/product/cloud-siem/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Cloud SIEM</a></li><li><a href="https://www.datadoghq.com/product/ai/bits-security-analyst/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Bits Security Analyst</a></li></ul></li><li><p class="mobile-product-category sticky top-0 bg-white flex items-center gap-2 pt-[13px] pb-[5px] pl-[19px] font-bold font-national text-22 leading-24 ml-5"><svg class="w-[22px] h-auto" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>rum</title>
|
||
<path d="M8.728 32h-8.728v-14.544h8.728v14.544zM2.912 29.088h2.912v-8.728h-2.912v8.728zM18.908 29.088h-7.269v-2.912h4.364v-14.544h-10.187v2.912h-2.905v-5.816h16.003v20.361zM32 20.361h-10.18v-2.912h7.276v-14.544h-20.367v2.912h-2.912v-5.816h26.184v20.361z"></path>
|
||
</svg>
|
||
Digital Experience</p><ul class="mb-[5px] pb-2.5"><li><a href="https://www.datadoghq.com/product/product-analytics/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Product Analytics</a></li><li><a href="https://www.datadoghq.com/product/experiments/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Experiments</a></li><li><a href="https://www.datadoghq.com/product/real-user-monitoring/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Browser Real User Monitoring</a></li><li><a href="https://www.datadoghq.com/product/real-user-monitoring/mobile-rum/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Mobile Real User Monitoring</a></li><li><a href="https://www.datadoghq.com/product/real-user-monitoring/session-replay/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Session Replay</a></li><li><a href="https://www.datadoghq.com/product/synthetic-monitoring/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Synthetic Monitoring</a></li><li><a href="https://www.datadoghq.com/product/mobile-app-testing/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Mobile App Testing</a></li><li><a href="https://www.datadoghq.com/product/continuous-testing/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Continuous Testing</a></li><li><a href="https://www.datadoghq.com/product/error-tracking/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Error Tracking</a></li><li><a href="https://www.datadoghq.com/product/platform/dashboards/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Dashboards</a></li></ul></li><li><p class="mobile-product-category sticky top-0 bg-white flex items-center gap-2 pt-[13px] pb-[5px] pl-[19px] font-bold font-national text-22 leading-24 ml-5"><svg class="w-[22px] h-auto" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>ci</title>
|
||
<path d="M30.667 22.625v0.087c-0.006 4.392-3.565 7.95-7.956 7.955h-0.085c-3.937-0.005-7.205-2.864-7.845-6.62l-0.007-0.047h-3.107l4.333-4.333 4.333 4.333h-2.833c0.6 2.315 2.669 3.998 5.131 4h0.087c2.916-0.009 5.278-2.373 5.282-5.29v-0.085c-0.003-2.921-2.369-5.288-5.29-5.292h-13.42c-4.393-0.004-7.954-3.565-7.957-7.958v-0.087c0.006-4.392 3.565-7.95 7.956-7.955h0.085c4.393 0.004 7.954 3.564 7.958 7.956v2.044h-2.667v-2.043c-0.004-2.921-2.371-5.287-5.291-5.29h-0.087c-2.919 0.005-5.285 2.371-5.288 5.29v0.085c0.003 2.921 2.369 5.288 5.29 5.292h13.42c4.393 0.004 7.954 3.565 7.957 7.958v0z"></path>
|
||
</svg>
|
||
Software Delivery</p><ul class="mb-[5px] pb-2.5"><li><a href="https://www.datadoghq.com/product/continuous-testing/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Continuous Testing</a></li><li><a href="https://www.datadoghq.com/product/internal-developer-portal/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Internal Developer Portal (IDP)</a></li><li><a href="https://www.datadoghq.com/product/ci-cd-monitoring/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">CI Visibility</a></li><li><a href="https://www.datadoghq.com/product/test-optimization/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Test Optimization</a></li><li><a href="https://www.datadoghq.com/product/feature-flags/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Feature Flags</a></li><li><a href="https://www.datadoghq.com/product/code-coverage/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Code Coverage</a></li><li><a href="https://www.datadoghq.com/product/software-composition-analysis/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Software Composition Analysis</a></li><li><a href="https://www.datadoghq.com/product/platform/dora-metrics/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">DORA Metrics + AI Impact</a></li></ul></li><li><p class="mobile-product-category sticky top-0 bg-white flex items-center gap-2 pt-[13px] pb-[5px] pl-[19px] font-bold font-national text-22 leading-24 ml-5"><svg class="w-[22px] h-auto" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M88,188.38V91.38l84-48.5v96.99l-84,48.51Zm16-87.76v60.04l52-30.02v-60.04l-52,30.02Zm-50-28.87v97l16-9.24V80.99l52-30.02,16-9.24V23.25L54,71.75ZM20,52.12v97l16-9.24V61.36l52-30.02,16-9.24V3.62L20,52.12Z"></path>
|
||
</svg>
|
||
Service Management</p><ul class="mb-[5px] pb-2.5"><li><a href="https://www.datadoghq.com/product/incident-response/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Incident Management</a></li><li><a href="https://www.datadoghq.com/product/case-management/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Work Management</a></li><li><a href="https://www.datadoghq.com/product/service-level-objectives/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Service Level Objectives</a></li><li><a href="https://www.datadoghq.com/product/workflow-automation/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Workflow Automation</a></li><li><a href="https://www.datadoghq.com/product/app-builder/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Datadog Apps</a></li><li><a href="https://www.datadoghq.com/product/event-management/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Event Management</a></li><li><a href="https://www.datadoghq.com/product/internal-developer-portal/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Internal Developer Portal (IDP)</a></li><li><a href="https://www.datadoghq.com/product/software-catalog/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Software Catalog</a></li></ul></li><li><p class="mobile-product-category sticky top-0 bg-white flex items-center gap-2 pt-[13px] pb-[5px] pl-[19px] font-bold font-national text-22 leading-24 ml-5"><svg class="w-[22px] h-auto" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" fill="" viewBox="0 0 192 192" style="enable-background:new 0 0 192 192;" xml:space="preserve"><g id="Layer_7"><g><linearGradient id="SVGID_1_" gradientUnits="userSpaceOnUse" x1="89.0192" y1="122.732" x2="142.9808" y2="29.268"><stop offset="0" style="stop-color:#5500FF"></stop><stop offset="0.1715" style="stop-color:#4D09FF"></stop><stop offset="0.4511" style="stop-color:#3821FF"></stop><stop offset="0.802" style="stop-color:#1649FF"></stop><stop offset="1" style="stop-color:#0062FF"></stop></linearGradient><path class="st5" d="M120,16h-8c0,30.88-25.12,56-56,56v8c30.88,0,56,25.12,56,56h8c0-30.88,25.12-56,56-56v-8 C145.12,72,120,46.88,120,16z"></path><linearGradient id="SVGID_00000049207151058930698180000011839073544639225000_" gradientUnits="userSpaceOnUse" x1="35.4115" y1="168.7311" x2="68.5885" y2="111.267"><stop offset="0" style="stop-color:#5500FF"></stop><stop offset="0.1715" style="stop-color:#4D09FF"></stop><stop offset="0.4511" style="stop-color:#3821FF"></stop><stop offset="0.802" style="stop-color:#1649FF"></stop><stop offset="1" style="stop-color:#0062FF"></stop></linearGradient><path d="M56,104h-4c0,0,0,0,0,0c0,0,0,0,0,0h-4 c0,17.64-14.35,32-32,32v8c17.65,0,32,14.36,32,32h8c0-17.64,14.35-32,32-32v-8C70.35,136,56,121.64,56,104z"></path></g></g></svg>AI</p><ul class="mb-[5px] pb-2.5"><li><a href="https://www.datadoghq.com/product/ai/bits-investigation/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Bits Investigation</a></li><li><a href="https://www.datadoghq.com/product/ai/bits-ai-agents/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Bits AI</a></li><li><a href="https://www.datadoghq.com/product/ai/bits-chat/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Bits Chat</a></li><li><a href="https://www.datadoghq.com/product/ai/bits-agent-builder/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Bits Agent Builder</a></li><li><a href="https://www.datadoghq.com/product/ai/bits-code/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Bits Code</a></li><li><a href="https://www.datadoghq.com/product/ai/bits-security-analyst/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Bits Security Analyst</a></li><li><a href="https://www.datadoghq.com/products/ai/agent-observability/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Agent Observability</a></li><li><a href="https://www.datadoghq.com/product/ai/mcp-server/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">MCP Server</a></li><li><a href="https://www.datadoghq.com/product/ai/mcp-server/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Pup CLI</a></li><li><a href="https://www.datadoghq.com/product/gpu-monitoring/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">GPU Monitoring</a></li></ul></li><li><p class="mobile-product-category sticky top-0 bg-white flex items-center gap-2 pt-[13px] pb-[5px] pl-[19px] font-bold font-national text-22 leading-24 ml-5"><svg class="w-[22px] h-auto" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>dashboard</title>
|
||
<path d="M32 24.876v1.79h-32v-21.333h1.988v19.543h30.012zM26.009 10.679l3.976 12.432h-25.982v-8.006l6.999-7.981 8.987 7.981 6.020-4.426z"></path>
|
||
</svg>
|
||
Platform Capabilities</p><ul class="mb-[5px] pb-2.5"><li><a href="https://www.datadoghq.com/product/ai/bits-ai-agents/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Bits AI</a></li><li><a href="https://www.datadoghq.com/product/ai/bits-chat/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Bits Chat</a></li><li><a href="https://www.datadoghq.com/product/ai/bits-agent-builder/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Bits Agent Builder</a></li><li><a href="https://www.datadoghq.com/product/platform/alerts/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Alerts</a></li><li><a href="https://www.datadoghq.com/product/metrics/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Custom Metrics</a></li><li><a href="https://www.datadoghq.com/product/platform/dashboards/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Dashboards</a></li><li><a href="https://docs.datadoghq.com/notebooks/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Notebooks</a></li><li><a href="https://docs.datadoghq.com/service_management/mobile/?tab=ios" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Mobile App</a></li><li><a href="https://www.datadoghq.com/product/fleet-automation/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Fleet Automation</a></li><li><a href="https://docs.datadoghq.com/account_management/rbac/?tab=datadogapplication" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Access Control</a></li><li><a href="https://www.datadoghq.com/product/governance-console/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Governance Console</a></li><li><a href="https://www.datadoghq.com/product/platform/dora-metrics/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">DORA Metrics + AI Impact</a></li><li><a href="https://www.datadoghq.com/product/incident-response/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Incident Management</a></li><li><a href="https://www.datadoghq.com/product/case-management/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Work Management</a></li><li><a href="https://www.datadoghq.com/product/workflow-automation/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Workflow Automation</a></li><li><a href="https://docs.datadoghq.com/account_management/teams/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Teams</a></li><li><a href="https://www.datadoghq.com/solutions/opentelemetry/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">OpenTelemetry</a></li><li><a href="https://www.datadoghq.com/product/platform/integrations/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Integrations</a></li><li><a href="https://www.datadoghq.com/product/platform/ides/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">IDE Plugins</a></li><li><a href="https://www.datadoghq.com/product/ai/mcp-server/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">MCP Server</a></li><li><a href="https://www.datadoghq.com/product/ai/agent-directory/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Agent Directory</a></li><li><a href="https://www.datadoghq.com/product/app-builder/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Datadog Apps</a></li><li><a href="https://docs.datadoghq.com/api/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">API</a></li><li><a href="https://www.datadoghq.com/marketplacepartners/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Marketplace</a></li><li><a href="https://www.datadoghq.com/product/ai/mcp-server/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Pup CLI</a></li><li><a href="https://www.datadoghq.com/product/software-catalog/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Software Catalog</a></li><li><a href="https://www.datadoghq.com/product/service-level-objectives/" class="block pt-[9px] pb-2 pl-[70px] hover:text-[#632ca6]">Service Level Objectives</a></li></ul></li></ul></li><li class="border-b border-b-[#c7c7c7] text-18"><a href="https://www.datadoghq.com/customers/" class="block font-semibold font-national uppercase text-[#555] hover:text-[#632ca6]"><div class="nav-menu-item ml-5 py-3 pl-[14px]">Customers</div></a></li><li class="border-b border-b-[#c7c7c7] text-18"><a href="https://www.datadoghq.com/pricing/" class="block font-semibold font-national uppercase text-[#555] hover:text-[#632ca6]"><div class="nav-menu-item ml-5 py-3 pl-[14px]">Pricing</div></a></li><li x-on:click="dropdownOpen === 'solutions' ? dropdownOpen = 'none' : dropdownOpen = 'solutions'" class="border-b border-b-[#c7c7c7] text-18 dropdown"><a href="#" data-astro-reload="" :class="dropdownOpen === 'solutions' ? 'text-[#632ca6]' : 'text-[#555]'" class="dropdown-toggle block font-semibold font-national uppercase text-[#555] hover:text-[#632ca6] whitespace-nowrap" data-bs-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="true" data-dd-action-name="Solutions Dropdown on Mobile"><div class="nav-menu-item ml-5 py-3 pl-[14px]">Solutions
|
||
<span class="caret absolute right-[22px] inline-flex h-[25px]"><svg viewBox="0 0 17.78 10.51" width="18px" class="inline-block transition-all" :class="dropdownOpen === 'solutions' ? 'rotate-180' : 'rotate-0'">
|
||
<polyline class="dropdown-arrow fill-none stroke-[#555]" style="stroke-width: 2px;" points="16.92 1.14 8.92 9.14 0.92 1.14"></polyline>
|
||
</svg></span></div></a><ul x-cloak="" x-show="dropdownOpen === 'solutions'" class="dropdown-menu mb-[5px] pb-2.5" id="solutions-mobile-menu"><p class="pl-15 py-2.5 font-semibold font-national text-16 leading-18 uppercase sticky top-0 bg-white">Industry</p><li><a href="https://www.datadoghq.com/solutions/financial-services/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Financial Services</div></a></li><li><a href="https://www.datadoghq.com/solutions/manufacturing-logistics/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Manufacturing & Logistics</div></a></li><li><a href="https://www.datadoghq.com/solutions/healthcare/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Healthcare/Life Sciences</div></a></li><li><a href="https://www.datadoghq.com/solutions/retail-ecommerce/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Retail/E-Commerce</div></a></li><li><a href="https://www.datadoghq.com/solutions/government/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Government</div></a></li><li><a href="https://www.datadoghq.com/solutions/education/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Education</div></a></li><li><a href="https://www.datadoghq.com/solutions/media-entertainment/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Media & Entertainment</div></a></li><li><a href="https://www.datadoghq.com/solutions/technology/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Technology</div></a></li><li><a href="https://www.datadoghq.com/solutions/gaming/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Gaming</div></a></li><p class="pl-15 py-2.5 font-semibold font-national text-16 leading-18 uppercase sticky top-0 bg-white mt-3">Technology</p><li><a href="https://www.datadoghq.com/solutions/aws/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Amazon Web Services Monitoring</div></a></li><li><a href="https://www.datadoghq.com/solutions/azure/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Azure Monitoring</div></a></li><li><a href="https://www.datadoghq.com/solutions/googlecloud/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Google Cloud Monitoring</div></a></li><li><a href="https://www.datadoghq.com/solutions/oci-monitoring/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Oracle Cloud Monitoring</div></a></li><li><a href="https://www.datadoghq.com/solutions/kubernetes/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Kubernetes Monitoring</div></a></li><li><a href="https://www.datadoghq.com/solutions/openshift/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Red Hat OpenShift</div></a></li><li><a href="https://www.datadoghq.com/solutions/pivotal-platform/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Pivotal Platform</div></a></li><li><a href="https://www.datadoghq.com/solutions/openai/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">OpenAI</div></a></li><li><a href="https://www.datadoghq.com/solutions/sap-monitoring/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">SAP Monitoring</div></a></li><li><a href="https://www.datadoghq.com/solutions/opentelemetry/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">OpenTelemetry</div></a></li><p class="pl-15 py-2.5 font-semibold font-national text-16 leading-18 uppercase sticky top-0 bg-white mt-3">Use Case</p><li><a href="https://www.datadoghq.com/solutions/application-security/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Application Security</div></a></li><li><a href="https://www.datadoghq.com/solutions/cloud-migration/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Cloud Migration</div></a></li><li><a href="https://www.datadoghq.com/solutions/monitoring-consolidation/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Monitoring Consolidation</div></a></li><li><a href="https://www.datadoghq.com/solutions/unified-commerce-monitoring/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Unified Commerce Monitoring</div></a></li><li><a href="https://www.datadoghq.com/solutions/soar/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">SOAR</div></a></li><li><a href="https://www.datadoghq.com/solutions/devops/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">DevOps</div></a></li><li><a href="https://www.datadoghq.com/solutions/finops/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">FinOps</div></a></li><li><a href="https://www.datadoghq.com/solutions/shift-left-testing/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Shift-Left Testing</div></a></li><li><a href="https://www.datadoghq.com/solutions/digital-experience-monitoring/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Digital Experience Monitoring</div></a></li><li><a href="https://www.datadoghq.com/solutions/security-analytics/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Security Analytics</div></a></li><li><a href="https://www.datadoghq.com/solutions/security/cis-benchmarks/aws/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Compliance for CIS Benchmarks</div></a></li><li><a href="https://www.datadoghq.com/solutions/hybrid-cloud-monitoring/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Hybrid Cloud Monitoring</div></a></li><li><a href="https://www.datadoghq.com/solutions/iot-monitoring/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Edge Device Monitoring</div></a></li><li><a href="https://www.datadoghq.com/solutions/real-time-business-intelligence/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Real-Time BI</div></a></li><li><a href="https://www.datadoghq.com/solutions/on-premises-monitoring/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">On-Premises Monitoring</div></a></li><li><a href="https://www.datadoghq.com/solutions/log-analysis-and-correlation/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Log Analysis & Correlation</div></a></li><li><a href="https://www.datadoghq.com/solutions/cnapp/"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">CNAPP</div></a></li></ul></li><li x-on:click="dropdownOpen === 'about' ? dropdownOpen = 'none' : dropdownOpen = 'about'" class="border-b border-b-[#c7c7c7] text-18 dropdown"><a href="#" data-astro-reload="" :class="dropdownOpen === 'about' ? 'text-[#632ca6]' : 'text-[#555]'" class="dropdown-toggle block font-semibold font-national uppercase text-[#555] hover:text-[#632ca6] whitespace-nowrap" data-bs-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="true" data-dd-action-name="About Dropdown on Mobile"><div class="nav-menu-item ml-5 py-3 pl-[14px]">About
|
||
<span class="caret absolute right-[22px] inline-flex h-[25px]"><svg viewBox="0 0 17.78 10.51" width="18px" class="inline-block transition-all" :class="dropdownOpen === 'about' ? 'rotate-180' : 'rotate-0'">
|
||
<polyline class="dropdown-arrow fill-none stroke-[#555]" style="stroke-width: 2px;" points="16.92 1.14 8.92 9.14 0.92 1.14"></polyline>
|
||
</svg></span></div></a><ul x-cloak="" x-show="dropdownOpen === 'about'" class="dropdown-menu mb-[5px] pb-2.5"><li><a href="https://www.datadoghq.com/about/contact/" class="block"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Contact</div></a></li><li><a href="https://www.datadoghq.com/partner/network/" class="block"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Partners</div></a></li><li><a href="https://www.datadoghq.com/about/latest-news/press-releases/" class="block"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Newsroom</div></a></li><li><a href="https://www.datadoghq.com/events-webinars/" class="block"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Events & Webinars</div></a></li><li><a href="https://www.datadoghq.com/about/leadership/" class="block"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Leadership</div></a></li><li><a href="https://careers.datadoghq.com/" class="block"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Careers</div></a></li><li><a href="https://www.datadoghq.com/about/analyst/" class="block"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Analyst Reports</div></a></li><li><a href="https://investors.datadoghq.com/" class="block"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Investor Relations</div></a></li><li><a href="https://www.datadoghq.com/esg-report/" class="block"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">ESG Report</div></a></li><li><a href="https://www.datadoghq.com/trust/" class="block"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Trust Hub</div></a></li></ul></li><li x-on:click="dropdownOpen === 'blog' ? dropdownOpen = 'none' : dropdownOpen = 'blog'" class="border-b border-b-[#c7c7c7] text-18 dropdown"><a href="#" data-astro-reload="" :class="dropdownOpen === 'blog' ? 'text-[#632ca6]' : 'text-[#555]'" class="dropdown-toggle block font-semibold font-national uppercase text-[#555] hover:text-[#632ca6] whitespace-nowrap" data-bs-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="true" data-dd-action-name="Blog Dropdown on Mobile"><div class="nav-menu-item ml-5 py-3 pl-[14px]">Blog
|
||
<span class="caret absolute right-[22px] inline-flex h-[25px]"><svg viewBox="0 0 17.78 10.51" width="18px" class="inline-block transition-all" :class="dropdownOpen === 'blog' ? 'rotate-180' : 'rotate-0'">
|
||
<polyline class="dropdown-arrow fill-none stroke-[#555]" style="stroke-width: 2px;" points="16.92 1.14 8.92 9.14 0.92 1.14"></polyline>
|
||
</svg></span></div></a><ul x-cloak="" x-show="dropdownOpen === 'blog'" class="dropdown-menu mb-[5px] pb-2.5"><li><a href="https://www.datadoghq.com/blog/" class="block"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">The Monitor</div></a></li><li><a href="https://www.datadoghq.com/blog/engineering/" class="block"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Engineering</div></a></li><li><a href="https://www.datadoghq.com/blog/ai/" class="block"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">AI</div></a></li><li><a href="https://securitylabs.datadoghq.com/" class="block"><div class="nav-menu-item-children block pt-[9px] pb-2 pl-15 hover:text-[#632ca6]">Security Labs</div></a></li></ul></li><li class="border-b border-b-[#c7c7c7] text-18 final"><a href="https://docs.datadoghq.com/" class="block font-semibold font-national uppercase text-[#555] hover:text-[#632ca6]" aria-label="Docs link"><div class="nav-menu-item ml-5 py-3 pl-[14px]">Docs</div></a></li><li class="border-b border-b-[#c7c7c7] text-18 final"><a href="https://app.datadoghq.com/" class="block font-semibold font-national uppercase text-[#555] hover:text-[#632ca6]" aria-label="Login link"><div class="nav-menu-item ml-5 py-3 pl-[14px]">Login</div></a></li><li class="flex justify-center text-center final"><a href="https://www.datadoghq.com/" class="block font-semibold font-national uppercase btn btn-default btn-info btn-outline outline knockout-around sign-up sign-up sign-up-trigger rounded text-white border-0 w-[89%] mt-5 bg-[transparent] hover:bg-opacity-70 px-5 pt-[11px] pb-3" data-event-category="Signup" data-event-action="Step 1 Top Right Initiated" data-event-label=" {{url path}}" aria-label="Get Started link" style="position:relative"><div class="nav-menu-item py-[5px] pl-0">Get Started</div></a></li></ul><style>.knockout-around:before{top:-1px;left:-1px;right:-1px;bottom:-1px;border-radius:5px;background-image:-webkit-gradient(linear,left top,right top,from(#3023ae),to(#c024b6));background-image:linear-gradient(to right,#3023ae,#c024b6);z-index:-2;content:'';position:absolute}.knockout-around:after{top:0;left:0;bottom:0;right:0;border-radius:3px;z-index:-1;content:'';position:absolute}</style></div><style>.btn-gradient:before{content:'';position:absolute;top:-1px;left:-1px;right:-1px;bottom:-1px;border-radius:5px;background-image:-webkit-gradient(linear,left top,right top,from(#3023ae),to(#c024b6));background-image:linear-gradient(to right,#3023ae,#c024b6);z-index:-2}.btn-gradient:hover{background:linear-gradient(45deg,#c024b6 0%,#3023ae 100%)}.main-nav-scrolled .btn-gradient{background:linear-gradient(45deg,#3023ae 0%,#c024b6 100%);color:#fff}</style></div></body></html></div> <dialog id="signUpModal" class="container my-auto border-0 bg-transparent flex w-full justify-center backdrop:bg-black/70 backdrop:backdrop-blur-[2px]" data-astro-cid-hfxa2sdw> <div class="relative w-full h-auto overflow-y-auto bg-white rounded-8 overflow-hidden max-w-[542px]" data-astro-cid-hfxa2sdw> <div class="flex items-center py-3 px-5" data-astro-cid-hfxa2sdw> <h3 class="h3 font-normal w-full text-center text-purple-600" data-astro-cid-hfxa2sdw>Get Started with Datadog</h3> <button id="signUpModal-close" class=" transition-default group flex cursor-pointer justify-center items-center w-max" title="Close modal" data-dd-action-name="dialog: button: close" aria-label="Close modal" data-astro-cid-hckxi45f><span class="sr-only" data-astro-cid-hckxi45f>Close modal</span><svg aria-hidden="true" focusable="false" width="32" height="32" viewBox="0 0 32 32" fill xmlns="http://www.w3.org/2000/svg" data-astro-cid-hckxi45f="true" class="transition-default h-5 w-auto fill-black order-2">
|
||
<path d="M7.6029 5.72327L5.71729 7.60889L24.3849 26.2765L26.2705 24.3909L7.6029 5.72327Z"></path>
|
||
<path d="M24.3849 5.7235L5.71729 24.3911L7.6029 26.2767L26.2705 7.60911L24.3849 5.7235Z"></path>
|
||
</svg></button> </div> <iframe title="DD Signup" width="500px" height="auto" frameborder="0" loading="lazy" class="js-signup-frame h-[815px] w-full max-w-[542px]" style="border: none;" sandbox="allow-scripts allow-forms allow-same-origin allow-top-navigation"></iframe> </div> </dialog> <main class="mt-[90px] desktop-sm:mt-[160px]"> <header class="container desktop:py-15 tablet:py-12 py-9"><div class="flex tablet:flex-row flex-col flex-nowrap justify-between desktop:gap-x-10 tablet:gap-6 tablet:gap-y-0 gap-y-9 tablet:grid tablet:grid-cols-12"><div class="flex flex-col flex-nowrap tablet:col-span-9 desktop-lg:col-span-10"><a href="/blog/engineering/" aria-label="Go to Engineering page" class="eyebrow-h1 bg-gradient-to-r hover:text-purple-600 from-purple-600 from-0% to-cyan-400 to-100% text-transparent bg-clip-text block w-max">Engineering</a><div class="mb-2 tablet:mb-3 desktop:mb-4"></div><h1 class="h1">How we use formal modeling, lightweight simulations, and chaos testing to design reliable distributed systems</h1><div class="desktop:mb-9 tablet:mb-7 mb-5"></div><div class="flex flex-wrap gap-2"><a title="message queue" aria-label="message queue" data-value="message queue" href="/blog/search/?tag=message%20queue" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
hover:text-white bg-grey-200 shadow-100 hover:bg-black active:bg-black rounded py-1 px-2 w-max button-roboto-mono"><span class="block false">message queue</span></a></div></div><div id="listen-rail-column" class="tablet:col-span-3 desktop-lg:col-span-2"><div class="flex flex-col border-t border-t-grey-400 pt-1 tablet:pt-1.5 desktop:pt-2"><!-- Published --><div class="flex flex-row justify-between items-center"><p class="p-roboto-mono-small text-grey-700 uppercase">Published</p><time class="p-small font-bold font-national" datetime="2024-11-20"> Nov 20, 2024 </time></div><div class="flex flex-row justify-between items-center border-t border-t-grey-400 pt-1 tablet:pt-1.5 desktop:pt-2 mt-1 tablet:mt-1.5 desktop:mt-2"><p class="p-roboto-mono-small text-grey-700 uppercase">Read time</p><p class="p-small font-bold font-national lining-nums">22m</p></div></div></div></div><div class="desktop:mb-12 tablet:mb-9 mb-7"></div><div class="content-image-component"> <!-- Render an image with optional popup functionality and caption --> <figure class> <picture><source media="(max-width: 767px)" srcset="https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=677&height=344&dpr=1 1x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=677&height=344&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=677&height=344&dpr=2 2x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=677&height=344&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=677&height=344&dpr=3 3x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=677&height=344&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=677&height=344&dpr=4 4x"><source media="(min-width: 768px) and (max-width: 1024px)" srcset="https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=934&height=474&dpr=1 1x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=934&height=474&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=934&height=474&dpr=2 2x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=934&height=474&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=934&height=474&dpr=3 3x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=934&height=474&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=934&height=474&dpr=4 4x"><source media="(min-width: 1025px)" srcset="https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=1400&height=711&dpr=1 1x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=1400&height=711&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=1400&height=711&dpr=2 2x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=1400&height=711&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=1400&height=711&dpr=3 3x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=1400&height=711&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=1400&height=711&dpr=4 4x"><img loading="eager" fetchpriority="high" class="h-auto rounded tablet:rounded-md desktop:rounded-lg w-full aspect-[380/193] object-cover" id="1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1" src="https://web-assets.dd-static.net/42588/1776291097-activemq-architecture-and-metrics-active-mq_longform_series_181009_part_1.png?format=auto&fit=crop&quality=75&disable=upscale&width=1400&height=711&dpr=1" alt="How we use formal modeling, lightweight simulations, and chaos testing to design reliable distributed systems" title="How we use formal modeling, lightweight simulations, and chaos testing to design reliable distributed systems" width="1400" height="711"></picture><script type="module">window.matchMedia("(prefers-reduced-motion: reduce)").matches&&document.querySelectorAll("video").forEach(e=>{e.autoplay=!1,e.controls=!0});</script> </figure> <!-- TODO: Update src with widths related to dialog size constraints + make img src dynamic (to avoid load unless requested via click) --> </div></header><div class="container desktop-sm:grid desktop-sm:grid-cols-12 desktop-sm:gap-10"><!-- Main Content — first in DOM so it streams before the sidebar --><section class="article-content desktop-sm:col-span-9 desktop-sm:col-start-4 desktop-sm:grid desktop-sm:grid-cols-9 desktop-sm:h-fit desktop:text-20 desktop:leading-28 tablet:text-18 tablet:leading-26"><div class="desktop-sm:col-span-6 grid grid-cols-1 mobile-lg:grid-cols-2 desktop-sm:gap-x-10 tablet:gap-x-6 tablet:gap-y-4 gap-y-2"><div class="flex tablet:h-12 gap-4 items-center"><div class="content-image-component"> <!-- Render an image with optional popup functionality and caption --> <figure class> <picture><source media="(max-width: 767px)" srcset="https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=4 4x"><source media="(min-width: 768px) and (max-width: 1024px)" srcset="https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=4 4x"><source media="(min-width: 1025px)" srcset="https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=4 4x"><img loading="eager" fetchpriority="high" class="h-auto rounded-full w-12 h-12" id="1776351371-arun-parthiban" src="https://web-assets.dd-static.net/42588/1776351371-arun-parthiban.png?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=1" alt="Arun Parthiban" title="Arun Parthiban" width="48" height></picture> </figure> <!-- TODO: Update src with widths related to dialog size constraints + make img src dynamic (to avoid load unless requested via click) --> </div><div class="flex flex-col"><p class="p-small font-bold font-national">Arun Parthiban</p></div></div><div class="flex tablet:h-12 gap-4 items-center"><div class="content-image-component"> <!-- Render an image with optional popup functionality and caption --> <figure class> <picture><source media="(max-width: 767px)" srcset="https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=4 4x"><source media="(min-width: 768px) and (max-width: 1024px)" srcset="https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=4 4x"><source media="(min-width: 1025px)" srcset="https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=4 4x"><img loading="eager" fetchpriority="high" class="h-auto rounded-full w-12 h-12" id="1776351376-sesh-nalla" src="https://web-assets.dd-static.net/42588/1776351376-sesh-nalla.png?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=1" alt="Sesh Nalla" title="Sesh Nalla" width="48" height></picture> </figure> <!-- TODO: Update src with widths related to dialog size constraints + make img src dynamic (to avoid load unless requested via click) --> </div><div class="flex flex-col"><p class="p-small font-bold font-national">Sesh Nalla</p></div></div><div class="flex tablet:h-12 gap-4 items-center"><div class="content-image-component"> <!-- Render an image with optional popup functionality and caption --> <figure class> <picture><source media="(max-width: 767px)" srcset="https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=4 4x"><source media="(min-width: 768px) and (max-width: 1024px)" srcset="https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=4 4x"><source media="(min-width: 1025px)" srcset="https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=4 4x"><img loading="eager" fetchpriority="high" class="h-auto rounded-full w-12 h-12" id="1776351257-ceciliawatt" src="https://web-assets.dd-static.net/42588/1776351257-ceciliawatt.jpg?format=auto&fit=bounds&quality=75&disable=upscale&width=48&dpr=1" alt="Cecilia Wat-Kim" title="Cecilia Wat-Kim" width="48" height></picture> </figure> <!-- TODO: Update src with widths related to dialog size constraints + make img src dynamic (to avoid load unless requested via click) --> </div><div class="flex flex-col"><p class="p-small font-bold font-national">Cecilia Wat-Kim</p></div></div></div><div class="desktop-sm:col-span-full desktop-sm:mb-9 tablet:mb-7 mb-5"></div><div class="tablet:col-span-full tablet:grid tablet:grid-cols-12 tablet:gap-x-6 desktop-sm:grid-cols-9 desktop-sm:gap-x-10 markdown-content [&>*:not(.content-image-wrapper)]:tablet:col-span-12 [&>*:not(.content-image-wrapper)]:desktop-sm:col-span-7 [&>.interactive-diagram-component]:tablet:col-span-12 [&>.interactive-diagram-component]:desktop-sm:col-span-8 [&>form-wrapper]:tablet:col-span-12 [&>form-wrapper]:desktop-sm:col-span-8 [&>.expressive-code]:tablet:col-span-12 [&>.expressive-code]:desktop-sm:col-span-8 [&>.table-component]:tablet:col-span-12 [&>.table-component]:desktop-sm:col-span-8 [&>.dato-code-block-wrapper]:tablet:col-span-12 [&>.dato-code-block-wrapper]:desktop-sm:col-span-8 [&>.dast-content]:col-span-full"><div class="dast-content tablet:grid tablet:grid-cols-subgrid [&>*:not(.content-image-wrapper)]:tablet:col-span-12 [&>*:not(.content-image-wrapper)]:desktop-sm:col-span-7 [&>.dato-code-block-wrapper]:desktop-sm:col-span-8 [&>.table-component]:desktop-sm:col-span-8 [&>.iframe-block-wrapper]:desktop-sm:col-span-8" data-astro-cid-twmmywjp> <p>It is a trusted premise of software engineering that we build large systems from smaller components, each of which can be designed and tested with a high degree of confidence. Some design problems, though, only become evident at the system level, and the absence of reliable methods for testing system-level issues can sometimes take us by surprise. Decisions that are legitimate at the component level can have unexpected and sometimes dramatic consequences at the system level. Our <a href="https://www.datadoghq.com/blog/2023-03-08-multiregion-infrastructure-connectivity-issue/">global outage</a> on March 8, 2023, was a humbling example of this, and it set forth new goals for us, in both the short and long term.</p>
|
||
<p>One of these long-term goals is tackling the challenge of analyzing distributed systems. Distributed systems unlock orders of magnitude of higher scale and service availability, even in the face of zonal and regional failures—but they also introduce more complexity and new failure modes. Experience has taught us that it is nearly impossible to intuitively reason about the behavior of a complex distributed system of concurrently executing processes, even if it is composed of simple parts. Dealing with this additional complexity requires thorough testing of all forms—unit, integration, etc.—as well as testing failure modes, which is often done through chaos tests. These traditional methods for testing software systems focus on system description at the lowest level of detail. Still, higher-level design errors are the hardest to find and the most costly to fix.</p>
|
||
<p>To analyze a distributed system during its design phase, we can use additional techniques such as formal modeling and lightweight simulations to describe and ascertain complex system behavior at any desired level of abstraction, tracking it from concept to sometimes even code. They help verify the behavior of a system with multiple well-defined, interacting parts by using model checking to evaluate its properties in all possible design-permitted states.</p>
|
||
<p>We recently had the opportunity to use these approaches while working on a new message queuing service named Courier, which allows us to quickly and reliably deliver data across our platforms. In this blog post, we’ll be sharing what insights we gained into our Courier service by applying formal modeling, lightweight simulations, and chaos testing.</p>
|
||
<h3 id="formal-modeling-and-simulations"><a href="#formal-modeling-and-simulations">Formal modeling and simulations</a></h3>
|
||
<p>In <a href="https://brooker.co.za/blog/2022/06/02/formal.html" target="_blank" rel="noopener noreferrer">formal modeling</a>, we use a high-level specification language to describe a software application and the properties it must satisfy. We then run these descriptions through a model checker that performs an exhaustive search of the states of the application and verifies that the properties are fulfilled.</p>
|
||
<p>However, there are aspects of a system that extend beyond whether properties are fulfilled—such as latency, cost, and scalability—that formal modeling cannot describe and verify. To account for this, we can additionally use <a href="https://brooker.co.za/blog/2022/04/11/simulation.html" target="_blank" rel="noopener noreferrer">lightweight simulation</a>: we can build a replica of a system and execute it under controlled conditions to describe statistical properties. This method provides insights into performance characteristics under various real-world conditions and workloads.</p>
|
||
<p>There are some caveats to this approach. While modeling and lightweight simulation can validate the design of a system, they do not validate its implementation. These techniques also introduce overhead, as we must keep models and simulations up-to-date with the design of a system. Despite these tradeoffs, we felt that the value we’d gain by employing these techniques outweighed the cost while building Courier. For a foundational system like Courier, which was being designed for several Datadog products, including <a href="https://www.datadoghq.com/product/workflow-automation/">Workflow Automation</a> and <a href="https://www.datadoghq.com/product/synthetic-monitoring/">Synthetic Monitoring</a>, reliability and correctness were essential. Further, Courier incorporated several lessons learned from the March 8 outage into its design, and we wanted a quick mechanism to validate and iterate upon these new ideas. And lastly, while unit testing and chaos testing are nonnegotiable parts of our development process, modeling and lightweight simulations provide an additional layer of validation.</p>
|
||
<h2 id="courier-a-mission-critical-multi-tenant-message-queuing-service"><a href="#courier-a-mission-critical-multi-tenant-message-queuing-service">Courier: a mission-critical, multi-tenant message queuing service</a></h2>
|
||
<p>Before we discuss analyzing Courier, we’ll explain what it is. Many Datadog products rely on a queuing system that handles asynchronous task processing. We had used a Redis-backed queuing system for over a decade, but we began to encounter issues with throughput, scaling, and durability. After much discussion, we decided to implement a new queuing system, named Courier, with the following requirements:</p>
|
||
<ol><li><p><strong>Multi-tenancy</strong>: Courier would be used by numerous teams or products within Datadog, so it needs to guarantee sufficient isolation between different tenants in the system.</p></li><li><p><strong>At-least-once delivery</strong>: Once a message is sent to Courier, the message must not be lost. The message must be delivered and acknowledged, or moved to a dead letter queue. This ensures that our users do not miss any notifications.</p></li><li><p><strong>Graceful degradation and high availability</strong>: Courier’s throughput should degrade linearly with available compute capacity. This requirement is a direct response to the outage on March 8, 2023, where a sudden loss of compute capacity caused a severe drop in throughput. By ensuring linear degradation, we aim to maintain partial service availability even when resources are constrained, preventing a complete service failure as experienced during the outage.</p></li><li><p><strong>Horizontal scalability</strong>: Courier’s throughput should increase linearly with compute capacity. This ensures that we can handle a proportionally larger workload, meeting growing demands without needing to redesign the system architecture.</p></li></ol>
|
||
<h3 id="solving-for-multi-tenancy"><a href="#solving-for-multi-tenancy">Solving for multi-tenancy</a></h3>
|
||
<p>To solve for multi-tenancy, we decided to have Courier backed by multiple FoundationDB clusters and have each tenant in the system sharded into a subset of these clusters such that no two tenants shared the exact set of FoundationDB clusters. Our plan was to start with eight clusters and shard each tenant into four clusters. With this configuration, Courier can support up to 70 tenants (<sup>8</sup>C<sub>4</sub> = 70) tenants. With this approach, even if one tenant somehow managed to saturate or cause unavailability in its four clusters, all other tenants in the system would have at least 25 percent capacity available. This was sufficient for our use cases.</p>
|
||
<div class="content-image-wrapper tablet:col-span-12 desktop-sm:col-span-8"> <div class="content-image-component"> <!-- Render an image with optional popup functionality and caption --> <figure class> <picture><source media="(max-width: 767px)" srcset="https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=4 4x"><source media="(min-width: 768px) and (max-width: 1024px)" srcset="https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=4 4x"><source media="(min-width: 1025px)" srcset="https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=4 4x"><img loading="lazy" class="h-auto max-w-full tablet:cursor-zoom-in" id="open-popup--1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev" src="https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Initial design for Courier." title="Initial design for Courier." width="3530" height="1761"></picture> </figure> <!-- TODO: Update src with widths related to dialog size constraints + make img src dynamic (to avoid load unless requested via click) --> <dialog-wrapper data-id="dialog-popup--1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev" data-trigger-id="open-popup--1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev" data-disable-mobile-popup="true" data-astro-cid-d4yttbaw="true"> <dialog id="dialog-popup--1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev" class="container h-full content-center bg-transparent backdrop:bg-black/70 backdrop:backdrop-blur-[2px]" data-astro-cid-d4yttbaw> <span class="relative mx-auto [&>*:first-child]:bg-white block tablet:p-9 desktop-sm:p-12 rounded-3xl bg-white max-h-[calc(100vh-6rem)] overflow-y-auto" data-astro-cid-d4yttbaw> <figure class="content-image-component overflow-scroll max-h-[calc(100vh-12rem)]"> <img loading="lazy" class="w-full h-auto" srcset="https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=4 4x" src="https://web-assets.dd-static.net/42588/1776351383-formal-modeling-and-simulation-courier-first-arch-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Initial design for Courier." width="1026"> </figure> <div class="desktop-sm:p-6 absolute top-0 right-0 z-10 p-3" data-astro-cid-d4yttbaw> <button title="Close dialog" aria-label="Close dialog" data-dd-action-name="dialog: icon: close dialog" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
|
||
"><span class="sr-only">Close dialog</span><svg aria-hidden="true" focusable="false" width="32" height="32" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg" class="h-6 w-6 fill-grey-400 hover:fill-purple-300 order-2 transition-default">
|
||
<path d="M7.6029 5.72327L5.71729 7.60889L24.3849 26.2765L26.2705 24.3909L7.6029 5.72327Z"></path>
|
||
<path d="M24.3849 5.7235L5.71729 24.3911L7.6029 26.2767L26.2705 7.60911L24.3849 5.7235Z"></path>
|
||
</svg></button> </div> </span> </dialog> </dialog-wrapper> <script type="module">class c extends HTMLElement{constructor(){super();const s=this.dataset.id,r=this.dataset.triggerId,e=document.getElementById(s),n=e?.querySelector("button"),o=document.getElementById(r),d=document.querySelector("body");if(!e||!n||!o)return;e.addEventListener("close",()=>{d.style.overflow=""}),e.addEventListener("click",t=>{const i=e.firstElementChild?.getBoundingClientRect()||e.getBoundingClientRect();(t.clientX<i.left||t.clientX>i.right||t.clientY<i.top||t.clientY>i.bottom)&&e.close()}),n.addEventListener("click",()=>{e.close()});let l=this.dataset.disableMobilePopup==="true"&&window.innerWidth<1025;window.addEventListener("resize",()=>{l=this.dataset.disableMobilePopup==="true"&&window.innerWidth<1025}),o.addEventListener("click",()=>{l||(e.showModal(),d.style.overflow="hidden")})}}customElements.define("dialog-wrapper",c);</script> </div> </div>
|
||
<p>We also planned to have a broker layer that would expose gRPC APIs for sending, receiving, and deleting messages. All clients would connect to this API layer, where the sharding logic described above would happen. The brokers would also perform health checks on FoundationDB and exclude any clusters that were unhealthy. Both FoundationDB and the brokers are deployed across three availability zones for high availability.</p>
|
||
<p>The sequence diagram below shows what an example interaction would look like in Courier:</p>
|
||
<div class="content-image-wrapper tablet:col-span-12 desktop-sm:col-span-8"> <div class="content-image-component"> <!-- Render an image with optional popup functionality and caption --> <figure class> <picture><source media="(max-width: 767px)" srcset="https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=4 4x"><source media="(min-width: 768px) and (max-width: 1024px)" srcset="https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=4 4x"><source media="(min-width: 1025px)" srcset="https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=4 4x"><img loading="lazy" class="h-auto max-w-full tablet:cursor-zoom-in" id="open-popup--1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2" src="https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Sequence diagram for the initial Courier design." title="Sequence diagram for the initial Courier design." width="3530" height="2150"></picture> </figure> <!-- TODO: Update src with widths related to dialog size constraints + make img src dynamic (to avoid load unless requested via click) --> <dialog-wrapper data-id="dialog-popup--1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2" data-trigger-id="open-popup--1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2" data-disable-mobile-popup="true" data-astro-cid-d4yttbaw="true"> <dialog id="dialog-popup--1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2" class="container h-full content-center bg-transparent backdrop:bg-black/70 backdrop:backdrop-blur-[2px]" data-astro-cid-d4yttbaw> <span class="relative mx-auto [&>*:first-child]:bg-white block tablet:p-9 desktop-sm:p-12 rounded-3xl bg-white max-h-[calc(100vh-6rem)] overflow-y-auto" data-astro-cid-d4yttbaw> <figure class="content-image-component overflow-scroll max-h-[calc(100vh-12rem)]"> <img loading="lazy" class="w-full h-auto" srcset="https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=4 4x" src="https://web-assets.dd-static.net/42588/1776351387-formal-modeling-and-simulation-courier-first-seq-diagram-rev2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Sequence diagram for the initial Courier design." width="1026"> </figure> <div class="desktop-sm:p-6 absolute top-0 right-0 z-10 p-3" data-astro-cid-d4yttbaw> <button title="Close dialog" aria-label="Close dialog" data-dd-action-name="dialog: icon: close dialog" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
|
||
"><span class="sr-only">Close dialog</span><svg aria-hidden="true" focusable="false" width="32" height="32" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg" class="h-6 w-6 fill-grey-400 hover:fill-purple-300 order-2 transition-default">
|
||
<path d="M7.6029 5.72327L5.71729 7.60889L24.3849 26.2765L26.2705 24.3909L7.6029 5.72327Z"></path>
|
||
<path d="M24.3849 5.7235L5.71729 24.3911L7.6029 26.2767L26.2705 7.60911L24.3849 5.7235Z"></path>
|
||
</svg></button> </div> </span> </dialog> </dialog-wrapper> </div> </div>
|
||
<p>At this point, we had a detailed design document that described how the system worked, and we had used our collective experience to reason through the system’s failure modes, but we still didn’t have a definitive way to say if the system met all requirements. The use of various design elements, including multiple FoundationDB clusters and health checks, led to too many scenarios for us to consider. This was our impetus to formally model Courier.</p>
|
||
<h2 id="tla"><a href="#tla">TLA+</a></h2>
|
||
<p>There are many ways to formally model a system—including <a href="https://lamport.azurewebsites.net/tla/tla.html" target="_blank" rel="noopener noreferrer">TLA+</a>, a formal specification language developed by Leslie Lamport. TLA+ provides a high-level language for expressing system specifications, capturing both the structure and behavior of a system. It allows us to precisely define system properties, explore different execution paths, and verify the correctness of the system against those properties. The power of TLA+ lies in its ability to express complex systems concisely and precisely, abstracting away unnecessary implementation details.</p>
|
||
<p>TLA+ comes bundled with a model checker called TLC, which takes a specification written in TLA+, performs an exhaustive search of the state space of the specification, and validates all states against any properties that we provide. If model checking passes, it implies that the properties we provided to TLC hold in every state that the system can exist in.</p>
|
||
<p>We’d previously used TLA+ to verify idempotency for <a href="https://www.datadoghq.com/blog/engineering/husky-deep-dive/">Husky</a> and to model a replication bug in one of Datadog’s distributed cron schedulers. Since we already had experience with TLA+, we decided to use it to model Courier as well.</p>
|
||
<h3 id="tla-model-overview"><a href="#tla-model-overview">TLA+ model overview</a></h3>
|
||
<p>We modeled Courier as three processes:</p>
|
||
<ul><li><p><strong>Broker</strong>: Handles SendMessage, ReceiveMessage, and DeleteMessage requests as well as performs health checks on FoundationDB.</p></li><li><p><strong>Sender</strong>: Dispatches SendMessage requests to the Courier Broker to submit messages to a queue.</p></li><li><p><strong>Receiver</strong>: Interacts with the Broker to receive and delete messages from the queue.</p></li></ul>
|
||
<p>As mentioned earlier, modeling a system formally often requires specifying the system at a higher level of abstraction than the implementation. For Courier, we omitted a number of implementation details, such as the exact data structures that store messages in FoundationDB, and extended message leases. We even abstracted FoundationDB by representing it as a <code>clusters</code> variable in the model and updating it atomically:</p>
|
||
<div class="dato-code-block-wrapper" data-astro-cid-twmmywjp> <div class="expressive-code"><link rel="stylesheet" href="/blog/_astro/ec.cd55k.css"><script type="module" src="/blog/_astro/ec.0vx5m.js"></script><figure class="frame"><figcaption class="header"></figcaption><pre data-language="text"><code><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">1</div></div><div class="code"><span style="--0:#000000">clusters = [c \in 1..NumClusters |-> [t \in 1..NumTenants |-> [q \in 1..NumQueuesPerTenant |-> <<>>]]]</span></div></div></code></pre><div class="copy"><div aria-live="polite"></div><button title="Copy to clipboard" data-copied="Copied!" data-code="clusters = [c \in 1..NumClusters |-> [t \in 1..NumTenants |-> [q \in 1..NumQueuesPerTenant |-> <<>>]]]"><div></div></button></div></figure></div> </div>
|
||
<p>Abstractions allowed us to create a model with a relatively smaller state space, which in turn allowed us to test the system more broadly—and to use the model as a building block to explore interactions with external components. We can extend the model to describe and test specific aspects in more detail in the future if necessary.</p>
|
||
<h4 id="modeling-state-transitions"><a href="#modeling-state-transitions">Modeling state transitions </a></h4>
|
||
<p>Models in TLA+ are described discretely; the system is described as a state machine where an event causes the system to transition from one state to another.</p>
|
||
<p>This is how we described the possible state transitions of Courier in TLA+:</p>
|
||
<div class="dato-code-block-wrapper" data-astro-cid-twmmywjp> <div class="expressive-code"><figure class="frame"><figcaption class="header"></figcaption><pre data-language="text"><code><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">1</div></div><div class="code"><span style="--0:#000000">Next == \/ (\E self \in Sender!ProcSet: SendMsg(self) \/ HandleSendMsgResponse(self))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">2</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">\/ (\E self \in Receiver!ProcSet: ReceiveMsg(self) \/ HandleReceiveMsgResponse(self) \/ DeleteMsg(self) \/ HandleDeleteMsgResponse(self))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">3</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">\/ (\E self \in Broker!ProcSet: ClusterHealthcheck(self))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">4</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">\/ RestartBroker</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">5</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">\/ ClusterUnavailable</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">6</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">\/ ClusterAvailable</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">7</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">\/ Terminating</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">8</div></div><div class="code">
|
||
</div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">9</div></div><div class="code"><span style="--0:#000000">Spec == Init /\ [][Next]_vars</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">10</div></div><div class="code"><span style="--0:#000000">/\ WF_vars(RestartBroker)</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">11</div></div><div class="code"><span style="--0:#000000">/\ WF_vars(ClusterUnavailable)</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">12</div></div><div class="code"><span style="--0:#000000">/\ WF_vars(ClusterAvailable)</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">13</div></div><div class="code"><span style="--0:#000000">/\ WF_vars(Terminating)</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">14</div></div><div class="code"><span style="--0:#000000">/\ \A s \in Sender!ProcSet : /\ WF_vars(SendMsg(s))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">15</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ WF_vars(HandleSendMsgResponse(s))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">16</div></div><div class="code"><span style="--0:#000000">\* only one receiver must always proceed, the rest can fail and stop at any stage</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">17</div></div><div class="code"><span style="--0:#000000">/\ WF_vars(ReceiveMsg(1))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">18</div></div><div class="code"><span style="--0:#000000">/\ WF_vars(HandleReceiveMsgResponse(1))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">19</div></div><div class="code"><span style="--0:#000000">/\ WF_vars(DeleteMsg(1))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">20</div></div><div class="code"><span style="--0:#000000">/\ WF_vars(HandleDeleteMsgResponse(1))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">21</div></div><div class="code"><span style="--0:#000000">/\ \A b \in Broker!ProcSet : /\ WF_vars(ClusterHealthcheck(b))</span></div></div></code></pre><div class="copy"><div aria-live="polite"></div><button title="Copy to clipboard" data-copied="Copied!" data-code="Next == \/ (\E self \in Sender!ProcSet: SendMsg(self) \/ HandleSendMsgResponse(self)) \/ (\E self \in Receiver!ProcSet: ReceiveMsg(self) \/ HandleReceiveMsgResponse(self) \/ DeleteMsg(self) \/ HandleDeleteMsgResponse(self)) \/ (\E self \in Broker!ProcSet: ClusterHealthcheck(self)) \/ RestartBroker \/ ClusterUnavailable \/ ClusterAvailable \/ TerminatingSpec == Init /\ [][Next]_vars/\ WF_vars(RestartBroker)/\ WF_vars(ClusterUnavailable)/\ WF_vars(ClusterAvailable)/\ WF_vars(Terminating)/\ \A s \in Sender!ProcSet : /\ WF_vars(SendMsg(s)) /\ WF_vars(HandleSendMsgResponse(s))\* only one receiver must always proceed, the rest can fail and stop at any stage/\ WF_vars(ReceiveMsg(1))/\ WF_vars(HandleReceiveMsgResponse(1))/\ WF_vars(DeleteMsg(1))/\ WF_vars(HandleDeleteMsgResponse(1))/\ \A b \in Broker!ProcSet : /\ WF_vars(ClusterHealthcheck(b))"><div></div></button></div></figure></div> </div>
|
||
<p><code>Next ==</code> specifies the possible next states of the system based on the current state and the enabled actions. This means that in the next state, the system can transition by performing any of these actions. In our case, it means that any of the following actions can be the next in the state transition:</p>
|
||
<ul><li><p>Any of the Senders sends a message or handles a Broker response</p></li><li><p>Any of the Receivers receives or deletes a message, or handles a Broker response</p></li><li><p>Any Broker instance performs a FoundationDB cluster health check</p></li><li><p>Any Broker instance is restarted</p></li><li><p>A FoundationDB cluster becomes unavailable</p></li><li><p>A FoundationDB cluster becomes available</p></li></ul>
|
||
<p><code>Spec ==</code> is used to define the overall specification or behavior of the system being modeled. It allows us to combine various properties, constraints, and initialization conditions into a single specification. For example, here we use <code>WF_vars</code> to specify <em>weak fairness</em> for all actions individually: weak fairness ensures that if an action is enabled infinitely often in the behavior of the system, then it must eventually occur. This allows for occasional skipping or delaying of the action, but the action cannot be indefinitely neglected or ignored. If we do not specify an action (or process) as fair, this indicates that the action can stop occurring at any time.</p>
|
||
<h4 id="modeling-sendmessage"><a href="#modeling-sendmessage">Modeling SendMessage</a></h4>
|
||
<p>By default, Courier runs eight FoundationDB clusters and allows each tenant to be on four out of eight clusters. Writing tenant messages to multiple clusters ensures high availability and helps spread the load.</p>
|
||
<p>So how do we model handling a SendMessage request?</p>
|
||
<p>First, the Sender dispatches a request:</p>
|
||
<div class="dato-code-block-wrapper" data-astro-cid-twmmywjp> <div class="expressive-code"><figure class="frame"><figcaption class="header"></figcaption><pre data-language="text"><code><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">1</div></div><div class="code"><span style="--0:#000000">SendMsg(self) == /\ pcSender[self] = "SendMsg"</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">2</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ IF Cardinality(senderMsgs) = 0 THEN</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">3</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ pcSender' = [pcSender EXCEPT ![self] = "Done"]</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">4</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ UNCHANGED <<**unchanged_vars**>></span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">5</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">ELSE</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">6</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ LET msg == CHOOSE m \in senderMsgs: TRUE</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">7</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">IN \E b \in 1..NumBrokerInstances: Broker!SendMsg(b, msg, self)</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">8</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ pcSender' = [pcSender EXCEPT ![self] = "HandleSendMsgResponse"]</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">9</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ UNCHANGED <<**unchanged_vars**>></span></div></div></code></pre><div class="copy"><div aria-live="polite"></div><button title="Copy to clipboard" data-copied="Copied!" data-code="SendMsg(self) == /\ pcSender[self] = "SendMsg" /\ IF Cardinality(senderMsgs) = 0 THEN /\ pcSender' = [pcSender EXCEPT ![self] = "Done"] /\ UNCHANGED <<**unchanged_vars**>> ELSE /\ LET msg == CHOOSE m \in senderMsgs: TRUE IN \E b \in 1..NumBrokerInstances: Broker!SendMsg(b, msg, self) /\ pcSender' = [pcSender EXCEPT ![self] = "HandleSendMsgResponse"] /\ UNCHANGED <<**unchanged_vars**>>"><div></div></button></div></figure></div> </div>
|
||
<p>The Broker processes the request and responds to it:</p>
|
||
<div class="dato-code-block-wrapper" data-astro-cid-twmmywjp> <div class="expressive-code"><figure class="frame"><figcaption class="header"></figcaption><pre data-language="text"><code><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">1</div></div><div class="code"><span style="--0:#000000">SendMsg(self, msg, sender) ==</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">2</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">LET cs == SelectSeq(tenantToCluster[self][msg.tenant], LAMBDA x: ~(x \in knownUnavailableClusters[self]) /\ ~(x \in unavailableClusters))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">3</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">IN</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">4</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">IF Len(cs) > 0 THEN</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">5</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ uniqueId' = uniqueId + 1</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">6</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ clusters' = [clusters EXCEPT ![cs[1]][msg.tenant][msg.queue] = Append(clusters[cs[1]][msg.tenant][msg.queue], [msg EXCEPT !.msgID = uniqueId, !.clusterId = cs[1]])]</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">7</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ SendMsgOK' = [SendMsgOK EXCEPT ![sender] = Append(@, msg)]</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">8</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ stats' = [stats EXCEPT !.sent = @ + 1, !.sentIds = @ \union {uniqueId}]</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">9</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ UNCHANGED <<**unchanged_vars**>></span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">10</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">ELSE</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">11</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ SendMsgError' = [SendMsgError EXCEPT ![sender] = Append(@, msg)]</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">12</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ UNCHANGED <<**unchanged_vars**>></span></div></div></code></pre><div class="copy"><div aria-live="polite"></div><button title="Copy to clipboard" data-copied="Copied!" data-code="SendMsg(self, msg, sender) == LET cs == SelectSeq(tenantToCluster[self][msg.tenant], LAMBDA x: ~(x \in knownUnavailableClusters[self]) /\ ~(x \in unavailableClusters)) IN IF Len(cs) > 0 THEN /\ uniqueId' = uniqueId + 1 /\ clusters' = [clusters EXCEPT ![cs[1]][msg.tenant][msg.queue] = Append(clusters[cs[1]][msg.tenant][msg.queue], [msg EXCEPT !.msgID = uniqueId, !.clusterId = cs[1]])] /\ SendMsgOK' = [SendMsgOK EXCEPT ![sender] = Append(@, msg)] /\ stats' = [stats EXCEPT !.sent = @ + 1, !.sentIds = @ \union {uniqueId}] /\ UNCHANGED <<**unchanged_vars**>> ELSE /\ SendMsgError' = [SendMsgError EXCEPT ![sender] = Append(@, msg)] /\ UNCHANGED <<**unchanged_vars**>>"><div></div></button></div></figure></div> </div>
|
||
<p>Courier maps each tenant to a set of FoundationDB clusters and maintains a list of healthy clusters (via active and passive health checking). On every SendMessage request, Courier routes the message to an available cluster mapped to the tenant. The request fails only if all clusters mapped to the tenant are unavailable.</p>
|
||
<h4 id="modeling-receivemessage-and-deletemessage"><a href="#modeling-receivemessage-and-deletemessage">Modeling ReceiveMessage and DeleteMessage</a></h4>
|
||
<p>Here is how we model the Broker processing a ReceiveMessage request:</p>
|
||
<div class="dato-code-block-wrapper" data-astro-cid-twmmywjp> <div class="expressive-code"><figure class="frame"><figcaption class="header"></figcaption><pre data-language="text"><code><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">1</div></div><div class="code"><span style="--0:#000000">ReceiveMsg(self, req, receiver) ==</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">2</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">LET cs == SelectSeq(tenantToCluster[self][req.tenant], LAMBDA x: ~(x \in knownUnavailableClusters[self]) /\ ~(x \in unavailableClusters) /\ Len(clusters[x][req.tenant][req.queue]) > 0 )</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">3</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">IN</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">4</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">IF Len(cs) = 0 THEN</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">5</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ ReceiveMsgError' = [ReceiveMsgError EXCEPT ![receiver] = Append(@, req)]</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">6</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ UNCHANGED <<**unchanged_vars**>></span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">7</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">ELSE</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">8</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ clusterTime' = clusterTime + 1</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">9</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ ReceiveMsgOK' = [ReceiveMsgOK EXCEPT ![receiver] = Append(@, req)]</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">10</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ clusters' = [clusters EXCEPT ![cs[1]][req.tenant][req.queue] = UpdateSeq(clusters[cs[1]][req.tenant][req.queue], 1, uniqueId, cs[1], uniqueId, leaseRecords[self], clusterTime)]</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">11</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ uniqueId' = uniqueId + Len(SelectSeq(clusters'[cs[1]][req.tenant][req.queue], LAMBDA x: x.receiveReqId = uniqueId))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">12</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ deadLetterQueue' = [deadLetterQueue EXCEPT ![cs[1]][req.tenant][req.queue] = @ \o SelectSeq(clusters[cs[1]][req.tenant][req.queue], LAMBDA x: ~(\E y \in 1..Len(clusters'[cs[1]][req.tenant][req.queue]): clusters'[cs[1]][req.tenant][req.queue][y].msgID = x.msgID))]</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">13</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ ReceiveMsgResult' = [ReceiveMsgResult EXCEPT ![receiver] = Append(@, SelectSeq(clusters'[cs[1]][req.tenant][req.queue], LAMBDA x: x.receiveReqId = uniqueId))]</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">14</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ UNCHANGED <<**unchanged_vars**>></span></div></div></code></pre><div class="copy"><div aria-live="polite"></div><button title="Copy to clipboard" data-copied="Copied!" data-code="ReceiveMsg(self, req, receiver) == LET cs == SelectSeq(tenantToCluster[self][req.tenant], LAMBDA x: ~(x \in knownUnavailableClusters[self]) /\ ~(x \in unavailableClusters) /\ Len(clusters[x][req.tenant][req.queue]) > 0 ) IN IF Len(cs) = 0 THEN /\ ReceiveMsgError' = [ReceiveMsgError EXCEPT ![receiver] = Append(@, req)] /\ UNCHANGED <<**unchanged_vars**>> ELSE /\ clusterTime' = clusterTime + 1 /\ ReceiveMsgOK' = [ReceiveMsgOK EXCEPT ![receiver] = Append(@, req)] /\ clusters' = [clusters EXCEPT ![cs[1]][req.tenant][req.queue] = UpdateSeq(clusters[cs[1]][req.tenant][req.queue], 1, uniqueId, cs[1], uniqueId, leaseRecords[self], clusterTime)] /\ uniqueId' = uniqueId + Len(SelectSeq(clusters'[cs[1]][req.tenant][req.queue], LAMBDA x: x.receiveReqId = uniqueId)) /\ deadLetterQueue' = [deadLetterQueue EXCEPT ![cs[1]][req.tenant][req.queue] = @ \o SelectSeq(clusters[cs[1]][req.tenant][req.queue], LAMBDA x: ~(\E y \in 1..Len(clusters'[cs[1]][req.tenant][req.queue]): clusters'[cs[1]][req.tenant][req.queue][y].msgID = x.msgID))] /\ ReceiveMsgResult' = [ReceiveMsgResult EXCEPT ![receiver] = Append(@, SelectSeq(clusters'[cs[1]][req.tenant][req.queue], LAMBDA x: x.receiveReqId = uniqueId))] /\ UNCHANGED <<**unchanged_vars**>>"><div></div></button></div></figure></div> </div>
|
||
<p>Under normal circumstances, messages are read and deleted by Receivers after processing or moved to dead letter queues. A <code>deliveryCount</code> variable (in another part of the code) tracks how many times a message was received and informs the service when to move it to a dead letter queue if a Receiver does not delete it. Once a message reaches a designated maximum delivery count, it is moved to the dead letter queue and will no longer be available to any Receivers. For the purposes of this model, we set the maximum delivery count to 2.</p>
|
||
<p>If the tenant has no available clusters or no active queues for the requested tenant-queue combination, the Broker returns an error for the request. Otherwise, the Broker checks if there are any available messages (messages that do not have an active lease and have not reached their delivery maximum), and returns such messages to the Receiver. The Broker also moves any messages that belong to the given tenant and queue and have reached their delivery maximum to the dead letter queue.</p>
|
||
<p>Here is the model code for a Broker processing a DeleteMessage request:</p>
|
||
<div class="dato-code-block-wrapper" data-astro-cid-twmmywjp> <div class="expressive-code"><figure class="frame"><figcaption class="header"></figcaption><pre data-language="text"><code><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">1</div></div><div class="code"><span style="--0:#000000">DeleteMsg(self, msg, receiver) ==</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">2</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ IF ~(msg.clusterId \in unavailableClusters) /\ ~(msg.clusterId \in knownUnavailableClusters[self]) THEN</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">3</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ clusters' = [clusters EXCEPT ![msg.clusterId][msg.tenant][msg.queue] = IF msg.leaseExpiration >= clusterTime THEN SelectSeq(clusters[msg.clusterId][msg.tenant][msg.queue], LAMBDA x: x.msgID /= msg.msgID) ELSE @]</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">4</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ IF Len(clusters[msg.clusterId][msg.tenant][msg.queue]) > Len(clusters'[msg.clusterId][msg.tenant][msg.queue]) THEN</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">5</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ DeleteMsgOK' = [DeleteMsgOK EXCEPT ![receiver] = Append(@, msg)]</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">6</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ UNCHANGED <<**unchanged_vars**>></span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">7</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">ELSE</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">8</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ DeleteMsgError' = [DeleteMsgError EXCEPT ![receiver] = Append(@, [msg EXCEPT !.error = "lease_expired"])]</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">9</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ UNCHANGED <<**unchanged_vars**>></span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">10</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">ELSE</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">11</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ DeleteMsgError' = [DeleteMsgError EXCEPT ![receiver] = Append(@, msg)]</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">12</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/\ UNCHANGED <<**unchanged_vars**>></span></div></div></code></pre><div class="copy"><div aria-live="polite"></div><button title="Copy to clipboard" data-copied="Copied!" data-code="DeleteMsg(self, msg, receiver) == /\ IF ~(msg.clusterId \in unavailableClusters) /\ ~(msg.clusterId \in knownUnavailableClusters[self]) THEN /\ clusters' = [clusters EXCEPT ![msg.clusterId][msg.tenant][msg.queue] = IF msg.leaseExpiration >= clusterTime THEN SelectSeq(clusters[msg.clusterId][msg.tenant][msg.queue], LAMBDA x: x.msgID /= msg.msgID) ELSE @] /\ IF Len(clusters[msg.clusterId][msg.tenant][msg.queue]) > Len(clusters'[msg.clusterId][msg.tenant][msg.queue]) THEN /\ DeleteMsgOK' = [DeleteMsgOK EXCEPT ![receiver] = Append(@, msg)] /\ UNCHANGED <<**unchanged_vars**>> ELSE /\ DeleteMsgError' = [DeleteMsgError EXCEPT ![receiver] = Append(@, [msg EXCEPT !.error = "lease_expired"])] /\ UNCHANGED <<**unchanged_vars**>> ELSE /\ DeleteMsgError' = [DeleteMsgError EXCEPT ![receiver] = Append(@, msg)] /\ UNCHANGED <<**unchanged_vars**>>"><div></div></button></div></figure></div> </div>
|
||
<p>If the message to be deleted is stored on an available cluster, and the message lease expiration time is greater than or equal to cluster time, the message is deleted from the queue. If the lease the Receiver submitted with the DeleteMessage request is expired (lease expiration < cluster time) or the cluster the message is on is not available, the Broker returns an error.</p>
|
||
<h4 id="specifying-properties"><a href="#specifying-properties">Specifying properties </a></h4>
|
||
<p>As mentioned earlier in this blog post, we’re interested in verifying that Courier never loses a message. This means once a message is sent to Courier, it is either received by a receiver and deleted, or it is moved to the dead letter queue. Here’s how we specified this in TLA+:</p>
|
||
<div class="dato-code-block-wrapper" data-astro-cid-twmmywjp> <div class="expressive-code"><figure class="frame"><figcaption class="header"></figcaption><pre data-language="text"><code><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">1</div></div><div class="code"><span style="--0:#000000">NoLostMsgs == <>[](Cardinality(Sender!Messages) * Cardinality(Sender!ProcSet) = stats.deleted + stats.deadLetterQueue)</span></div></div></code></pre><div class="copy"><div aria-live="polite"></div><button title="Copy to clipboard" data-copied="Copied!" data-code="NoLostMsgs == <>[](Cardinality(Sender!Messages) * Cardinality(Sender!ProcSet) = stats.deleted + stats.deadLetterQueue)"><div></div></button></div></figure></div> </div>
|
||
<h3 id="results"><a href="#results">Results</a></h3>
|
||
<p>Running the model described above through the TLC model checker passed.</p>
|
||
<div class="content-image-wrapper tablet:col-span-12 desktop-sm:col-span-8"> <div class="content-image-component"> <!-- Render an image with optional popup functionality and caption --> <figure class> <picture><source media="(max-width: 767px)" srcset="https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=4 4x"><source media="(min-width: 768px) and (max-width: 1024px)" srcset="https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=4 4x"><source media="(min-width: 1025px)" srcset="https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=4 4x"><img loading="lazy" class="h-auto max-w-full tablet:cursor-zoom-in" id="open-popup--1776351391-formal-modeling-and-simulation-model_result" src="https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Screenshot of TLC model checker, showing a successful result." title="Screenshot of TLC model checker, showing a successful result." width="912" height="406"></picture> <figcaption class="mt-4 px-15 p-small italic text-grey-600">A successful result in TLC.</figcaption> </figure> <!-- TODO: Update src with widths related to dialog size constraints + make img src dynamic (to avoid load unless requested via click) --> <dialog-wrapper data-id="dialog-popup--1776351391-formal-modeling-and-simulation-model_result" data-trigger-id="open-popup--1776351391-formal-modeling-and-simulation-model_result" data-disable-mobile-popup="true" data-astro-cid-d4yttbaw="true"> <dialog id="dialog-popup--1776351391-formal-modeling-and-simulation-model_result" class="container h-full content-center bg-transparent backdrop:bg-black/70 backdrop:backdrop-blur-[2px]" data-astro-cid-d4yttbaw> <span class="relative mx-auto [&>*:first-child]:bg-white block tablet:p-9 desktop-sm:p-12 rounded-3xl bg-white max-h-[calc(100vh-6rem)] overflow-y-auto" data-astro-cid-d4yttbaw> <figure class="content-image-component overflow-scroll max-h-[calc(100vh-12rem)]"> <img loading="lazy" class="w-full h-auto" srcset="https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=4 4x" src="https://web-assets.dd-static.net/42588/1776351391-formal-modeling-and-simulation-model_result.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Screenshot of TLC model checker, showing a successful result." width="1026"> <figcaption class="mt-4 px-9 p-small italic text-grey-600">A successful result in TLC.</figcaption> </figure> <div class="desktop-sm:p-6 absolute top-0 right-0 z-10 p-3" data-astro-cid-d4yttbaw> <button title="Close dialog" aria-label="Close dialog" data-dd-action-name="dialog: icon: close dialog" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
|
||
"><span class="sr-only">Close dialog</span><svg aria-hidden="true" focusable="false" width="32" height="32" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg" class="h-6 w-6 fill-grey-400 hover:fill-purple-300 order-2 transition-default">
|
||
<path d="M7.6029 5.72327L5.71729 7.60889L24.3849 26.2765L26.2705 24.3909L7.6029 5.72327Z"></path>
|
||
<path d="M24.3849 5.7235L5.71729 24.3911L7.6029 26.2767L26.2705 7.60911L24.3849 5.7235Z"></path>
|
||
</svg></button> </div> </span> </dialog> </dialog-wrapper> </div> </div>
|
||
<p>In this case, TLC scanned 5,515,710 distinct states and did not find any cases where our <code>NoLostMsgs</code> property was violated.</p>
|
||
<p>While the successful model checking result gave us more confidence in the design for Courier, we found the modeling process valuable in a couple of other ways as well:</p>
|
||
<ul><li><p>It provided us with a shared understanding of the system and a language through which we could communicate precisely about the system.</p></li><li><p>It forced us to be more precise while implementing the system; we found several bugs by simply thinking through if our implementation was consistent with what was modeled. For example, we found a class of bugs related to FoundationDB’s optimistic concurrency control that we abstracted away in the model. (We solved these bugs by ensuring that our transactions were all idempotent).</p></li></ul>
|
||
<h2 id="simulations"><a href="#simulations">Simulations </a></h2>
|
||
<p>Formally modeling Courier allowed us to verify its correctness properties, but we found it insufficient to explain the performance characteristics of the systems. How much latency can we expect? How does throughput of the system vary under outages and overload?</p>
|
||
<p>We were specifically interested in how Courier would perform if it experienced an outage similar to Datadog’s March 8 incident. One of the lessons we learned from this incident pertains to quorum-based distribution systems, such as FoundationDB. These systems are available only if a quorum of nodes are alive; likewise, on recovery, a quorum of nodes must be alive before the system can serve even a single request.</p>
|
||
<p>We mentioned earlier that we chose to back Courier with multiple FoundationDB clusters for multi-tenancy, but another reason we made this decision was to hedge our bets in an outage scenario. Having multiple smaller FoundationDB clusters increases the likelihood that at least some quorums would survive if we had a loss in compute capacity. For example, if we only had one 100-node FoundationDB cluster and lost 51 nodes, then our service would be completely unavailable. But instead, if we had ten ten-node FoundationDB clusters and lost 51 nodes, up to eight clusters could survive, depending on where the node losses happened.</p>
|
||
<p>We implemented a discrete event simulation using <a href="https://simpy.readthedocs.io/en/latest/" target="_blank" rel="noopener noreferrer">SimPy</a> to quantify more precisely how Courier behaves. In the simulation, we modeled Senders, Receivers, Brokers, and FoundationDB, and we measured throughput against node loss. We used <a href="https://apple.github.io/foundationdb/benchmarking.html" target="_blank" rel="noopener noreferrer">benchmark data</a> from FoundationDB to model CPU usage as well. We simulated two scenarios:</p>
|
||
<ul><li><p>Optimistic: Failed nodes are distributed across FoundationDB clusters, preserving as many functioning clusters as possible</p></li><li><p>Pessimistic: Failed nodes are distributed in the way that results in the maximum number of failed clusters</p></li></ul>
|
||
<p>The chart below shows how Courier behaves in both these scenarios, compared to both a single-cluster system and a hypothetical best-case system with linear degradation:</p>
|
||
<div class="content-image-wrapper tablet:col-span-12 desktop-sm:col-span-8"> <div class="content-image-component"> <!-- Render an image with optional popup functionality and caption --> <figure class> <picture><source media="(max-width: 767px)" srcset="https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=4 4x"><source media="(min-width: 768px) and (max-width: 1024px)" srcset="https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=4 4x"><source media="(min-width: 1025px)" srcset="https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=4 4x"><img loading="lazy" class="h-auto max-w-full tablet:cursor-zoom-in" id="open-popup--1776351395-formal-modeling-and-simulation-chart1" src="https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Line graph showing number of successful requests plotted against failed node percentage." title="Line graph showing number of successful requests plotted against failed node percentage." width="1600" height="957"></picture> </figure> <!-- TODO: Update src with widths related to dialog size constraints + make img src dynamic (to avoid load unless requested via click) --> <dialog-wrapper data-id="dialog-popup--1776351395-formal-modeling-and-simulation-chart1" data-trigger-id="open-popup--1776351395-formal-modeling-and-simulation-chart1" data-disable-mobile-popup="true" data-astro-cid-d4yttbaw="true"> <dialog id="dialog-popup--1776351395-formal-modeling-and-simulation-chart1" class="container h-full content-center bg-transparent backdrop:bg-black/70 backdrop:backdrop-blur-[2px]" data-astro-cid-d4yttbaw> <span class="relative mx-auto [&>*:first-child]:bg-white block tablet:p-9 desktop-sm:p-12 rounded-3xl bg-white max-h-[calc(100vh-6rem)] overflow-y-auto" data-astro-cid-d4yttbaw> <figure class="content-image-component overflow-scroll max-h-[calc(100vh-12rem)]"> <img loading="lazy" class="w-full h-auto" srcset="https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=4 4x" src="https://web-assets.dd-static.net/42588/1776351395-formal-modeling-and-simulation-chart1.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Line graph showing number of successful requests plotted against failed node percentage." width="1026"> </figure> <div class="desktop-sm:p-6 absolute top-0 right-0 z-10 p-3" data-astro-cid-d4yttbaw> <button title="Close dialog" aria-label="Close dialog" data-dd-action-name="dialog: icon: close dialog" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
|
||
"><span class="sr-only">Close dialog</span><svg aria-hidden="true" focusable="false" width="32" height="32" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg" class="h-6 w-6 fill-grey-400 hover:fill-purple-300 order-2 transition-default">
|
||
<path d="M7.6029 5.72327L5.71729 7.60889L24.3849 26.2765L26.2705 24.3909L7.6029 5.72327Z"></path>
|
||
<path d="M24.3849 5.7235L5.71729 24.3911L7.6029 26.2767L26.2705 7.60911L24.3849 5.7235Z"></path>
|
||
</svg></button> </div> </span> </dialog> </dialog-wrapper> </div> </div>
|
||
<p>The chart shows that in the optimistic scenario, Courier’s throughput matches that of the best-case system, and in the pessimistic scenario, Courier still behaves better than a system with a single cluster.</p>
|
||
<p>At first, we did not understand the oscillations in throughput, but we found an explanation by examining the CPU usage data from our simulation:</p>
|
||
<div class="content-image-wrapper tablet:col-span-12 desktop-sm:col-span-8"> <div class="content-image-component"> <!-- Render an image with optional popup functionality and caption --> <figure class> <picture><source media="(max-width: 767px)" srcset="https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=4 4x"><source media="(min-width: 768px) and (max-width: 1024px)" srcset="https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=4 4x"><source media="(min-width: 1025px)" srcset="https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=4 4x"><img loading="lazy" class="h-auto max-w-full tablet:cursor-zoom-in" id="open-popup--1776351399-formal-modeling-and-simulation-chart2" src="https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Line graph showing cluster usage plotted against failed node percentage." title="Line graph showing cluster usage plotted against failed node percentage." width="1000" height="600"></picture> </figure> <!-- TODO: Update src with widths related to dialog size constraints + make img src dynamic (to avoid load unless requested via click) --> <dialog-wrapper data-id="dialog-popup--1776351399-formal-modeling-and-simulation-chart2" data-trigger-id="open-popup--1776351399-formal-modeling-and-simulation-chart2" data-disable-mobile-popup="true" data-astro-cid-d4yttbaw="true"> <dialog id="dialog-popup--1776351399-formal-modeling-and-simulation-chart2" class="container h-full content-center bg-transparent backdrop:bg-black/70 backdrop:backdrop-blur-[2px]" data-astro-cid-d4yttbaw> <span class="relative mx-auto [&>*:first-child]:bg-white block tablet:p-9 desktop-sm:p-12 rounded-3xl bg-white max-h-[calc(100vh-6rem)] overflow-y-auto" data-astro-cid-d4yttbaw> <figure class="content-image-component overflow-scroll max-h-[calc(100vh-12rem)]"> <img loading="lazy" class="w-full h-auto" srcset="https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=4 4x" src="https://web-assets.dd-static.net/42588/1776351399-formal-modeling-and-simulation-chart2.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Line graph showing cluster usage plotted against failed node percentage." width="1026"> </figure> <div class="desktop-sm:p-6 absolute top-0 right-0 z-10 p-3" data-astro-cid-d4yttbaw> <button title="Close dialog" aria-label="Close dialog" data-dd-action-name="dialog: icon: close dialog" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
|
||
"><span class="sr-only">Close dialog</span><svg aria-hidden="true" focusable="false" width="32" height="32" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg" class="h-6 w-6 fill-grey-400 hover:fill-purple-300 order-2 transition-default">
|
||
<path d="M7.6029 5.72327L5.71729 7.60889L24.3849 26.2765L26.2705 24.3909L7.6029 5.72327Z"></path>
|
||
<path d="M24.3849 5.7235L5.71729 24.3911L7.6029 26.2767L26.2705 7.60911L24.3849 5.7235Z"></path>
|
||
</svg></button> </div> </span> </dialog> </dialog-wrapper> </div> </div>
|
||
<p>We observed that as a FoundationDB cluster loses nodes, its CPU usage increases, which reduces throughput—but when Courier’s health check detects that a cluster has failed, traffic moves to healthy FoundationDB clusters, causing throughput to recover until the healthy clusters lose nodes. This repeats for each FoundationDB cluster, causing oscillations. To work around this, we changed the inputs to the simulation so that each transaction required less processing power, thereby reducing the CPU load on the remaining nodes. This adjustment diminished the oscillations, as the system could handle the transactions more efficiently. We then re-ran the simulation.</p>
|
||
<div class="content-image-wrapper tablet:col-span-12 desktop-sm:col-span-8"> <div class="content-image-component"> <!-- Render an image with optional popup functionality and caption --> <figure class> <picture><source media="(max-width: 767px)" srcset="https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=4 4x"><source media="(min-width: 768px) and (max-width: 1024px)" srcset="https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=4 4x"><source media="(min-width: 1025px)" srcset="https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=4 4x"><img loading="lazy" class="h-auto max-w-full tablet:cursor-zoom-in" id="open-popup--1776351404-formal-modeling-and-simulation-chart3" src="https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Line graph showing number of successful requests plotted against failed node percentage." title="Line graph showing number of successful requests plotted against failed node percentage." width="1000" height="600"></picture> </figure> <!-- TODO: Update src with widths related to dialog size constraints + make img src dynamic (to avoid load unless requested via click) --> <dialog-wrapper data-id="dialog-popup--1776351404-formal-modeling-and-simulation-chart3" data-trigger-id="open-popup--1776351404-formal-modeling-and-simulation-chart3" data-disable-mobile-popup="true" data-astro-cid-d4yttbaw="true"> <dialog id="dialog-popup--1776351404-formal-modeling-and-simulation-chart3" class="container h-full content-center bg-transparent backdrop:bg-black/70 backdrop:backdrop-blur-[2px]" data-astro-cid-d4yttbaw> <span class="relative mx-auto [&>*:first-child]:bg-white block tablet:p-9 desktop-sm:p-12 rounded-3xl bg-white max-h-[calc(100vh-6rem)] overflow-y-auto" data-astro-cid-d4yttbaw> <figure class="content-image-component overflow-scroll max-h-[calc(100vh-12rem)]"> <img loading="lazy" class="w-full h-auto" srcset="https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=4 4x" src="https://web-assets.dd-static.net/42588/1776351404-formal-modeling-and-simulation-chart3.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Line graph showing number of successful requests plotted against failed node percentage." width="1026"> </figure> <div class="desktop-sm:p-6 absolute top-0 right-0 z-10 p-3" data-astro-cid-d4yttbaw> <button title="Close dialog" aria-label="Close dialog" data-dd-action-name="dialog: icon: close dialog" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
|
||
"><span class="sr-only">Close dialog</span><svg aria-hidden="true" focusable="false" width="32" height="32" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg" class="h-6 w-6 fill-grey-400 hover:fill-purple-300 order-2 transition-default">
|
||
<path d="M7.6029 5.72327L5.71729 7.60889L24.3849 26.2765L26.2705 24.3909L7.6029 5.72327Z"></path>
|
||
<path d="M24.3849 5.7235L5.71729 24.3911L7.6029 26.2767L26.2705 7.60911L24.3849 5.7235Z"></path>
|
||
</svg></button> </div> </span> </dialog> </dialog-wrapper> </div> </div>
|
||
<p>The chart above shows that oscillations in throughput are greatly diminished with this new configuration. This demonstrates how different configurations can yield significantly different results, underscoring the importance of testing various setups to understand their impact on system performance.</p>
|
||
<h3 id="closing-the-loop"><a href="#closing-the-loop">Closing the loop</a></h3>
|
||
<p>After Courier was implemented, we performed a chaos test that recreated the pessimistic scenario described above. We deployed our actual implementation of Courier, along with four FoundationDB clusters, to a staging environment and used Datadog’s <a href="https://github.com/DataDog/chaos-controller" target="_blank" rel="noopener noreferrer">chaos controller</a> to induce node loss on the FoundationDB clusters. The following chart shows the throughput of a single tenant in the system against node loss:</p>
|
||
<div class="content-image-wrapper tablet:col-span-12 desktop-sm:col-span-8"> <div class="content-image-component"> <!-- Render an image with optional popup functionality and caption --> <figure class> <picture><source media="(max-width: 767px)" srcset="https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=4 4x"><source media="(min-width: 768px) and (max-width: 1024px)" srcset="https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=4 4x"><source media="(min-width: 1025px)" srcset="https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=4 4x"><img loading="lazy" class="h-auto max-w-full tablet:cursor-zoom-in" id="open-popup--1776351408-formal-modeling-and-simulation-chart4" src="https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Linegraph showing throughput and compute loss percent over time." title="Linegraph showing throughput and compute loss percent over time." width="1600" height="676"></picture> </figure> <!-- TODO: Update src with widths related to dialog size constraints + make img src dynamic (to avoid load unless requested via click) --> <dialog-wrapper data-id="dialog-popup--1776351408-formal-modeling-and-simulation-chart4" data-trigger-id="open-popup--1776351408-formal-modeling-and-simulation-chart4" data-disable-mobile-popup="true" data-astro-cid-d4yttbaw="true"> <dialog id="dialog-popup--1776351408-formal-modeling-and-simulation-chart4" class="container h-full content-center bg-transparent backdrop:bg-black/70 backdrop:backdrop-blur-[2px]" data-astro-cid-d4yttbaw> <span class="relative mx-auto [&>*:first-child]:bg-white block tablet:p-9 desktop-sm:p-12 rounded-3xl bg-white max-h-[calc(100vh-6rem)] overflow-y-auto" data-astro-cid-d4yttbaw> <figure class="content-image-component overflow-scroll max-h-[calc(100vh-12rem)]"> <img loading="lazy" class="w-full h-auto" srcset="https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=4 4x" src="https://web-assets.dd-static.net/42588/1776351408-formal-modeling-and-simulation-chart4.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Linegraph showing throughput and compute loss percent over time." width="1026"> </figure> <div class="desktop-sm:p-6 absolute top-0 right-0 z-10 p-3" data-astro-cid-d4yttbaw> <button title="Close dialog" aria-label="Close dialog" data-dd-action-name="dialog: icon: close dialog" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
|
||
"><span class="sr-only">Close dialog</span><svg aria-hidden="true" focusable="false" width="32" height="32" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg" class="h-6 w-6 fill-grey-400 hover:fill-purple-300 order-2 transition-default">
|
||
<path d="M7.6029 5.72327L5.71729 7.60889L24.3849 26.2765L26.2705 24.3909L7.6029 5.72327Z"></path>
|
||
<path d="M24.3849 5.7235L5.71729 24.3911L7.6029 26.2767L26.2705 7.60911L24.3849 5.7235Z"></path>
|
||
</svg></button> </div> </span> </dialog> </dialog-wrapper> </div> </div>
|
||
<p>This is very similar to our findings from the simulations.</p>
|
||
<p>We found that the simulations were very useful for Courier when it came to determining how availability scaled with the number of replicas. Simulations gave us a much more descriptive picture of what a failure’s range of impact would look like in real life. They also surfaced potential cascading failures: each FoundationDB failure would result in a higher load on remaining clusters and further negatively impact availability. Additionally, after Courier was built, simulations made it easier for us to perform chaos tests, as we knew exactly what to test for and approximately what to expect.</p>
|
||
<h2 id="performance-testing"><a href="#performance-testing">Performance testing </a></h2>
|
||
<p>Next, we moved onto performance testing our implementation. We created a very basic set up that had Sender, Receiver, and Broker nodes with a single FoundationDB cluster. We controlled throughput to Courier by varying the number of Sender and Receiver nodes, and we heavily overprovisioned the Brokers so we could understand FoundationDB’s performance characteristics more directly for our use cases. Unfortunately, we couldn’t get enough throughput out of FoundationDB for our implementation to be cost effective. We observed hotspots in CPU usage that cycled through different storage nodes in the FoundationDB cluster:</p>
|
||
<div class="content-image-wrapper tablet:col-span-12 desktop-sm:col-span-8"> <div class="content-image-component"> <!-- Render an image with optional popup functionality and caption --> <figure class> <picture><source media="(max-width: 767px)" srcset="https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=4 4x"><source media="(min-width: 768px) and (max-width: 1024px)" srcset="https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=4 4x"><source media="(min-width: 1025px)" srcset="https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=4 4x"><img loading="lazy" class="h-auto max-w-full tablet:cursor-zoom-in" id="open-popup--1776351412-formal-modeling-and-simulation-chart5" src="https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Graph showing storage CPU usage over time." title="Graph showing storage CPU usage over time." width="868" height="352"></picture> </figure> <!-- TODO: Update src with widths related to dialog size constraints + make img src dynamic (to avoid load unless requested via click) --> <dialog-wrapper data-id="dialog-popup--1776351412-formal-modeling-and-simulation-chart5" data-trigger-id="open-popup--1776351412-formal-modeling-and-simulation-chart5" data-disable-mobile-popup="true" data-astro-cid-d4yttbaw="true"> <dialog id="dialog-popup--1776351412-formal-modeling-and-simulation-chart5" class="container h-full content-center bg-transparent backdrop:bg-black/70 backdrop:backdrop-blur-[2px]" data-astro-cid-d4yttbaw> <span class="relative mx-auto [&>*:first-child]:bg-white block tablet:p-9 desktop-sm:p-12 rounded-3xl bg-white max-h-[calc(100vh-6rem)] overflow-y-auto" data-astro-cid-d4yttbaw> <figure class="content-image-component overflow-scroll max-h-[calc(100vh-12rem)]"> <img loading="lazy" class="w-full h-auto" srcset="https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=4 4x" src="https://web-assets.dd-static.net/42588/1776351412-formal-modeling-and-simulation-chart5.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Graph showing storage CPU usage over time." width="1026"> </figure> <div class="desktop-sm:p-6 absolute top-0 right-0 z-10 p-3" data-astro-cid-d4yttbaw> <button title="Close dialog" aria-label="Close dialog" data-dd-action-name="dialog: icon: close dialog" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
|
||
"><span class="sr-only">Close dialog</span><svg aria-hidden="true" focusable="false" width="32" height="32" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg" class="h-6 w-6 fill-grey-400 hover:fill-purple-300 order-2 transition-default">
|
||
<path d="M7.6029 5.72327L5.71729 7.60889L24.3849 26.2765L26.2705 24.3909L7.6029 5.72327Z"></path>
|
||
<path d="M24.3849 5.7235L5.71729 24.3911L7.6029 26.2767L26.2705 7.60911L24.3849 5.7235Z"></path>
|
||
</svg></button> </div> </span> </dialog> </dialog-wrapper> </div> </div>
|
||
<p>After some investigation, we understood that these hotspots were caused by Courier’s write pattern and how FoundationDB distributed keys across storage nodes.</p>
|
||
<p>FoundationDB employs range-based sharding, where data is sharded into contiguous key ranges, and each key range is assigned to one of the storage servers in the cluster. If one of the key ranges runs hot (that is, if it begins consuming high amounts of CPU), FoundationDB breaks that key range into multiple subranges and moves the subranges to different nodes. In Courier, we were using the message receipt timestamp in the keys used to store messages in queues. So the keys that represented the tail of the queue would always run hot, since that’s where all new messages were added. When FoundationDB split the key range, the tail of the queue moved to a different FoundationDB node, causing a CPU usage hotspot on that node. When the key range was split again, the tail simply moved to a different node, causing another hotspot.</p>
|
||
<p>To work around this, we added a shard number to the keys and wrote new messages with a randomly selected shard number. This allowed FoundationDB to partition the data in such a way that multiple storage servers can process incoming messages. Similarly, we deliver messages from a random shard number. The downside of this approach is that it made our queue completely unordered, making it very difficult to troubleshoot. With multiple FoundationDB clusters and multiple shard numbers within those clusters, it became difficult to provide even basic metrics like queue lag. And since we were delivering messages in random order, end-to-end tail latencies for message delivery were high.</p>
|
||
<p>One potential solution was to scan through all the shards on delivery and pick the oldest message. But this would not work well for highly concurrent or high-throughput queues, since it would cause too many conflicts with FoundationDB’s optimistic concurrency control.</p>
|
||
<p>Instead, we decided to add a sequencer process to Courier that orders messages for delivery. The sequencer runs periodically, scans all FoundationDB clusters for undelivered messages, and stores them in-memory in the order in which they should be delivered. When a ReceiveMessage request is received, Courier pops a message from the sequencer and delivers that message to the client. We brought back the Redis sequencer from our initial implementation, since Redis is efficient and well-suited for this role. The sequencer needs only to be fast and not durable, as we can rehydrate it from the FoundationDB clusters if necessary. The interactions between the processes now look like:</p>
|
||
<div class="content-image-wrapper tablet:col-span-12 desktop-sm:col-span-8"> <div class="content-image-component"> <!-- Render an image with optional popup functionality and caption --> <figure class> <picture><source media="(max-width: 767px)" srcset="https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=4 4x"><source media="(min-width: 768px) and (max-width: 1024px)" srcset="https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=4 4x"><source media="(min-width: 1025px)" srcset="https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=4 4x"><img loading="lazy" class="h-auto max-w-full tablet:cursor-zoom-in" id="open-popup--1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev" src="https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Sequence diagram for the updated Courier design." title="Sequence diagram for the updated Courier design." width="3530" height="2535"></picture> </figure> <!-- TODO: Update src with widths related to dialog size constraints + make img src dynamic (to avoid load unless requested via click) --> <dialog-wrapper data-id="dialog-popup--1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev" data-trigger-id="open-popup--1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev" data-disable-mobile-popup="true" data-astro-cid-d4yttbaw="true"> <dialog id="dialog-popup--1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev" class="container h-full content-center bg-transparent backdrop:bg-black/70 backdrop:backdrop-blur-[2px]" data-astro-cid-d4yttbaw> <span class="relative mx-auto [&>*:first-child]:bg-white block tablet:p-9 desktop-sm:p-12 rounded-3xl bg-white max-h-[calc(100vh-6rem)] overflow-y-auto" data-astro-cid-d4yttbaw> <figure class="content-image-component overflow-scroll max-h-[calc(100vh-12rem)]"> <img loading="lazy" class="w-full h-auto" srcset="https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=4 4x" src="https://web-assets.dd-static.net/42588/1776351416-formal-modeling-and-simulation-courier-updated-seq-diagram-rev.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Sequence diagram for the updated Courier design." width="1026"> </figure> <div class="desktop-sm:p-6 absolute top-0 right-0 z-10 p-3" data-astro-cid-d4yttbaw> <button title="Close dialog" aria-label="Close dialog" data-dd-action-name="dialog: icon: close dialog" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
|
||
"><span class="sr-only">Close dialog</span><svg aria-hidden="true" focusable="false" width="32" height="32" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg" class="h-6 w-6 fill-grey-400 hover:fill-purple-300 order-2 transition-default">
|
||
<path d="M7.6029 5.72327L5.71729 7.60889L24.3849 26.2765L26.2705 24.3909L7.6029 5.72327Z"></path>
|
||
<path d="M24.3849 5.7235L5.71729 24.3911L7.6029 26.2767L26.2705 7.60911L24.3849 5.7235Z"></path>
|
||
</svg></button> </div> </span> </dialog> </dialog-wrapper> </div> </div>
|
||
<p>With these changes in, the performance of the system was adequate for our use cases. However, since we changed the design of the system, our TLA+ model was out of date.</p>
|
||
<h2 id="updating-the-tla-model"><a href="#updating-the-tla-model">Updating the TLA+ model</a></h2>
|
||
<p>We updated the TLA+ model to include a sequencer:</p>
|
||
<div class="dato-code-block-wrapper" data-astro-cid-twmmywjp> <div class="expressive-code"><figure class="frame"><figcaption class="header"></figcaption><pre data-language="tex"><code><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">1</div></div><div class="code"><span style="--0:#000000">Spec == Init /</span><span style="--0:#D40000">\ </span><span style="--0:#000000">[][Next]_vars</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">2</div></div><div class="code"><span style="--0:#000000">/</span><span style="--0:#D40000">\ </span><span style="--0:#000000">WF_vars(RestartBroker)</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">3</div></div><div class="code"><span style="--0:#000000">/</span><span style="--0:#D40000">\ </span><span style="--0:#000000">WF_vars(ClusterUnavailable)</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">4</div></div><div class="code"><span style="--0:#000000">/</span><span style="--0:#D40000">\ </span><span style="--0:#000000">WF_vars(ClusterAvailable)</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">5</div></div><div class="code"><span style="--0:#000000">/</span><span style="--0:#D40000">\ </span><span style="--0:#000000">WF_vars(Terminating)</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">6</div></div><div class="code"><span style="--0:#000000">/</span><span style="--0:#D40000">\ </span><span style="--0:#795E26">\A</span><span style="--0:#000000"> s </span><span style="--0:#795E26">\in</span><span style="--0:#000000"> Sender!ProcSet : /</span><span style="--0:#D40000">\ </span><span style="--0:#000000">WF_vars(SendMsg(s))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">7</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/</span><span style="--0:#D40000">\ </span><span style="--0:#000000">WF_vars(HandleSendMsgResponse(s))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">8</div></div><div class="code"><span style="--0:#D40000">\*</span><span style="--0:#000000"> only one receiver must always proceed, the rest can fail and stop at any stage</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">9</div></div><div class="code"><span style="--0:#000000">/</span><span style="--0:#D40000">\ </span><span style="--0:#000000">WF_vars(ReceiveMsg(1))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">10</div></div><div class="code"><span style="--0:#000000">/</span><span style="--0:#D40000">\ </span><span style="--0:#000000">WF_vars(HandleReceiveMsgResponse(1))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">11</div></div><div class="code"><span style="--0:#000000">/</span><span style="--0:#D40000">\ </span><span style="--0:#000000">WF_vars(DeleteMsg(1))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">12</div></div><div class="code"><span style="--0:#000000">/</span><span style="--0:#D40000">\ </span><span style="--0:#000000">WF_vars(HandleDeleteMsgResponse(1))</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">13</div></div><div class="code"><span style="--0:#000000">/</span><span style="--0:#D40000">\ </span><span style="--0:#795E26">\A</span><span style="--0:#000000"> b </span><span style="--0:#795E26">\in</span><span style="--0:#000000"> Broker!ProcSet : /</span><span style="--0:#D40000">\ </span><span style="--0:#000000">WF_vars(ClusterHealthcheck(b))</span></div></div><div class="ec-line highlight mark"><div class="gutter"><div class="ln" aria-hidden="true">14</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/</span><span style="--0:#ac0000">\ </span><span style="--0:#000000">WF_vars(SequencerRoutine(b))</span></div></div></code></pre><div class="copy"><div aria-live="polite"></div><button title="Copy to clipboard" data-copied="Copied!" data-code="Spec == Init /\ [][Next]_vars/\ WF_vars(RestartBroker)/\ WF_vars(ClusterUnavailable)/\ WF_vars(ClusterAvailable)/\ WF_vars(Terminating)/\ \A s \in Sender!ProcSet : /\ WF_vars(SendMsg(s)) /\ WF_vars(HandleSendMsgResponse(s))\* only one receiver must always proceed, the rest can fail and stop at any stage/\ WF_vars(ReceiveMsg(1))/\ WF_vars(HandleReceiveMsgResponse(1))/\ WF_vars(DeleteMsg(1))/\ WF_vars(HandleDeleteMsgResponse(1))/\ \A b \in Broker!ProcSet : /\ WF_vars(ClusterHealthcheck(b)) /\ WF_vars(SequencerRoutine(b))"><div></div></button></div></figure></div> </div>
|
||
<p>When we re-ran model checking, TLC found a case where the NoLostMsg property failed. TLC outputs traces that show the exact sequence of operations that led to the failure:</p>
|
||
<div class="content-image-wrapper tablet:col-span-12 desktop-sm:col-span-8"> <div class="content-image-component"> <!-- Render an image with optional popup functionality and caption --> <figure class> <picture><source media="(max-width: 767px)" srcset="https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=767&dpr=4 4x"><source media="(min-width: 768px) and (max-width: 1024px)" srcset="https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1024&dpr=4 4x"><source media="(min-width: 1025px)" srcset="https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=4 4x"><img loading="lazy" class="h-auto max-w-full tablet:cursor-zoom-in" id="open-popup--1776351421-formal-modeling-and-simulation-courier-stack-trace" src="https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Courier model stack trace." title="Courier model stack trace." width="2054" height="3459"></picture> </figure> <!-- TODO: Update src with widths related to dialog size constraints + make img src dynamic (to avoid load unless requested via click) --> <dialog-wrapper data-id="dialog-popup--1776351421-formal-modeling-and-simulation-courier-stack-trace" data-trigger-id="open-popup--1776351421-formal-modeling-and-simulation-courier-stack-trace" data-disable-mobile-popup="true" data-astro-cid-d4yttbaw="true"> <dialog id="dialog-popup--1776351421-formal-modeling-and-simulation-courier-stack-trace" class="container h-full content-center bg-transparent backdrop:bg-black/70 backdrop:backdrop-blur-[2px]" data-astro-cid-d4yttbaw> <span class="relative mx-auto [&>*:first-child]:bg-white block tablet:p-9 desktop-sm:p-12 rounded-3xl bg-white max-h-[calc(100vh-6rem)] overflow-y-auto" data-astro-cid-d4yttbaw> <figure class="content-image-component overflow-scroll max-h-[calc(100vh-12rem)]"> <img loading="lazy" class="w-full h-auto" srcset="https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1 1x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2 2x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3 3x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1026&dpr=4 4x" src="https://web-assets.dd-static.net/42588/1776351421-formal-modeling-and-simulation-courier-stack-trace.png?format=auto&fit=bounds&quality=75&disable=upscale&width=1400&dpr=1" alt="Courier model stack trace." width="1026"> </figure> <div class="desktop-sm:p-6 absolute top-0 right-0 z-10 p-3" data-astro-cid-d4yttbaw> <button title="Close dialog" aria-label="Close dialog" data-dd-action-name="dialog: icon: close dialog" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
|
||
"><span class="sr-only">Close dialog</span><svg aria-hidden="true" focusable="false" width="32" height="32" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg" class="h-6 w-6 fill-grey-400 hover:fill-purple-300 order-2 transition-default">
|
||
<path d="M7.6029 5.72327L5.71729 7.60889L24.3849 26.2765L26.2705 24.3909L7.6029 5.72327Z"></path>
|
||
<path d="M24.3849 5.7235L5.71729 24.3911L7.6029 26.2767L26.2705 7.60911L24.3849 5.7235Z"></path>
|
||
</svg></button> </div> </span> </dialog> </dialog-wrapper> </div> </div>
|
||
<p>In the trace above, the ‘initial predicate’ step displays the initial state of all variables in the model, while subsequent steps show only variables that have changed since the previous step. Steps 2, 3, 6, 15, and 18 reflect the broker performing health checks, with step 4 indicating a broker restart. Although step 2 shows a health check failure, all other health checks appear to pass and don’t contribute to the model failure.</p>
|
||
<p>Step 5 and 7 are interesting—this is where a message is sent to the broker and it responds. Step 6 shows the sequencer routine running, but it does not load any messages since a receive message request has not been registered yet. This happens in Step 8, where the receiver is sending a receive message request and the variable <code>redisReceiveCount</code> gets updated. Since there are no messages in the sequencer at this point, an error is returned.</p>
|
||
<p>Steps 10, 11, and 12 show another receive message request starting and completing. In step 13, the broker responds to a new receive message request, and in step 14, the sequencer finally runs. In our model, a receiver only attempts three requests before terminating, which occurs in steps 16 and 17, where <code>pcReceiver = "done"</code> indicates termination.</p>
|
||
<div class="dato-code-block-wrapper" data-astro-cid-twmmywjp> <div class="expressive-code"><figure class="frame"><figcaption class="header"></figcaption><pre data-language="tex"><code><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">1</div></div><div class="code"><span style="--0:#000000">ReceiveMsg(self) == /</span><span style="--0:#D40000">\ </span><span style="--0:#000000">pcReceiver[self] = "ReceiveMsg"</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">2</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/</span><span style="--0:#D40000">\ </span><span style="--0:#000000">IF (Cardinality(senderMsgs) = 0 /</span><span style="--0:#D40000">\ </span><span style="--0:#000000">QueueEmpty(clusters, receiverRequests[self].tenant, receiverRequests[self].queue) /</span><span style="--0:#D40000">\ </span><span style="--0:#000000">Len(msgs[self]) = 0)</span></div></div><div class="ec-line highlight mark"><div class="gutter"><div class="ln" aria-hidden="true">3</div></div><div class="code"><span class="indent"> </span><span style="--0:#ac0000">\/</span><span style="--0:#000000"> receiverRequests[self].attempts = 3 THEN</span></div></div><div class="ec-line"><div class="gutter"><div class="ln" aria-hidden="true">4</div></div><div class="code"><span class="indent"><span style="--0:#000000"> </span></span><span style="--0:#000000">/</span><span style="--0:#D40000">\ </span><span style="--0:#000000">pcReceiver' = [pcReceiver EXCEPT ![self] = "Done"]</span></div></div></code></pre><div class="copy"><div aria-live="polite"></div><button title="Copy to clipboard" data-copied="Copied!" data-code="ReceiveMsg(self) == /\ pcReceiver[self] = "ReceiveMsg" /\ IF (Cardinality(senderMsgs) = 0 /\ QueueEmpty(clusters, receiverRequests[self].tenant, receiverRequests[self].queue) /\ Len(msgs[self]) = 0) \/ receiverRequests[self].attempts = 3 THEN /\ pcReceiver' = [pcReceiver EXCEPT ![self] = "Done"]"><div></div></button></div></figure></div> </div>
|
||
<p>Because the sequencer didn’t run within three receive attempts, the receiver was not able to receive the message. In theory, the sequencer is not guaranteed to run within any number of attempts of the receiver and can remain permanently out-of-date with what’s stored in FoundationDB.</p>
|
||
<p>But in practice, the sequencer is guaranteed to run at some point—we programmed it to run every three seconds and, if it became unavailable or slow, it would find the FoundationDB messages when it recovers. Therefore, our <code>NoLostMsgs</code> property should really be checking that messages fulfill one of three properties: persisted in FoundationDB, delivered and deleted, or moved to the dead letter queue. This tradeoff—where we rely on the periodic running of the sequencer instead of immediate processing and accept that there may be a period when messages are sent but not delivered—was one that we knew about and deemed adequate for our use cases. The availability of SendMessages is not impacted with this change. The model gave us an assurance that we didn’t introduce any other failure modes with this change.</p>
|
||
<h2 id="next-steps"><a href="#next-steps">Next Steps</a></h2>
|
||
<p>As noted at the beginning of this blog post, formal modeling and lightweight simulations are powerful tools for verifying the design of distributed systems. However, they do not guarantee that the implementation fully aligns with the model and simulations, as subtle differences may exist. Determining whether an implementation sufficiently aligns with the models and simulations requires a deep understanding of both the model and simulations, as well as the implementation itself, and ultimately relies on human judgment. Following internal reviews, we partnered with <a href="https://cs.uwaterloo.ca/~nday/" target="_blank" rel="noopener noreferrer">Professor Nancy Day’s</a> lab at the University of Waterloo to review our work and bring the latest advances in formal modeling into industry applications. This work is still ongoing.</p>
|
||
<p>Moreover, once a system is built and in production, we find it tedious to keep the models and simulations up-to-date. We may choose to update our models for large design changes (such as when we added a sequencer), but for routine feature work and bug fixes, we often take a more pragmatic approach and verify our changes through unit, integration, and chaos testing. Oftentimes, an update to our models may not even be required, since the model is a higher-level abstraction than our implementation.</p>
|
||
<p>A more recent technique, called deterministic simulations, popularized by FoundationDB, would allow us to perform similar tests as part of our CI/CD process, helping to verify every release. In <a href="https://apple.github.io/foundationdb/testing.html" target="_blank" rel="noopener noreferrer">deterministic simulations</a>, an actual implementation of a system is executed in tightly controlled environments to verify the correctness of a system. Any non-deterministic inputs in the system such as time, random numbers, I/O operations, etc. are mocked so that any bugs found in the simulation can be reproduced along with the exact sequence of operations that led to the bug. Since time is itself mocked, a deterministic simulator can execute a system much faster than traditional unit and integration tests and can cover a large number of test cases. Failure injection is often combined with deterministic simulations.</p>
|
||
<p>Applying deterministic simulation techniques to Courier has been challenging. Courier is written in Go, and the Go runtime itself introduces non-determinism into our application: iteration over maps, scheduling of goroutines, and execution of select statements are all non-deterministic. Courier also comprises three different binaries: the brokers, Redis, and FoundationDB—which adds another layer of complexity. Recently <a href="https://antithesis.com/" target="_blank" rel="noopener noreferrer">Antithesis</a>, a company founded by former FoundationDB engineers, introduced a generalized approach to deterministic testing through a custom-built deterministic hypervisor. This enables deterministic simulations in a language- and technology-agnostic way. While we weren’t able to use deterministic simulations for Courier, it remains an active area of research for us.</p>
|
||
<p>If you’re interested in working with us at Datadog, <a href="https://careers.datadoghq.com/engineering/">we’re hiring</a>.</p> </div></div><div class="desktop-sm:col-span-full desktop-sm:mb-12 mb-9"></div><div class="print:hidden"><ul class="inline-flex gap-2"> <li> <a href="https://twitter.com/intent/tweet?url=https://www.datadoghq.com/blog/engineering/formal-modeling-and-simulation/" aria-label="Share on X" target="_blank" rel="noopener noreferrer" class="group" data-dd-action-name="share: link: x"> <svg aria-hidden="true" focusable="false" width="36" height="36" viewBox="0 0 36 36" xmlns="http://www.w3.org/2000/svg" class="[&>g>rect]:fill-grey-500 [&_path]:!fill-white w-7 h-7 tablet:w-8 tablet:h-8 desktop-sm:w-9 desktop-sm:h-9 group-hover:[&>g>rect]:fill-black">
|
||
<g id="_atom_social-icons">
|
||
<rect width="36" height="36" rx="18" fill=""></rect>
|
||
<path id="Vector" d="M19.5222 16.5277L25.4785 9H24.0671L18.8952 15.5362L14.7644 9H10L16.2466 18.8839L10 26.7778H11.4115L16.8732 19.8753L21.2356 26.7778H26L19.5222 16.5277ZM17.5889 18.9709L16.956 17.9867L11.9201 10.1553H14.0882L18.1522 16.4755L18.7851 17.4597L24.0677 25.675H21.8997L17.5889 18.9709Z" fill=""></path>
|
||
</g>
|
||
</svg> </a> </li><li> <a href="https://www.reddit.com/submit?url=https://www.datadoghq.com/blog/engineering/formal-modeling-and-simulation/" aria-label="Share on Reddit" target="_blank" rel="noopener noreferrer" class="group" data-dd-action-name="share: link: reddit"> <svg aria-hidden="true" focusable="false" width="36" height="36" viewBox="0 0 36 36" xmlns="http://www.w3.org/2000/svg" class="[&>g>rect]:fill-grey-500 [&_path]:!fill-white w-7 h-7 tablet:w-8 tablet:h-8 desktop-sm:w-9 desktop-sm:h-9 group-hover:[&>g>rect]:fill-[#FF4500]">
|
||
<g id="_atom_social-icons">
|
||
<rect width="36" height="36" rx="18" fill=""></rect>
|
||
<path id="Vector" d="M15.7419 20.129C15.1613 20.129 14.7097 19.6774 14.7097 19.0968C14.7097 18.5161 15.1613 18.0645 15.7419 18.0645C16.3226 18.0645 16.7742 18.5161 16.7742 19.0968C16.7742 19.6129 16.2581 20.129 15.7419 20.129ZM20.129 21.4194C19.6774 21.871 18.6452 22 18 22C17.3548 22 16.3226 21.8065 15.871 21.4194C15.7419 21.2903 15.6129 21.2903 15.4839 21.4194C15.3548 21.5484 15.3548 21.6774 15.4839 21.8065C16.1935 22.5161 17.5484 22.5806 18 22.5806C18.3871 22.5806 19.8065 22.5161 20.5161 21.8065C20.6452 21.6774 20.6452 21.5484 20.5161 21.4194C20.3871 21.3548 20.1935 21.3548 20.129 21.4194ZM20.2581 18C19.6774 18 19.2258 18.4516 19.2258 19.0323C19.2258 19.6129 19.6774 20.0645 20.2581 20.0645C20.8387 20.0645 21.2903 19.6129 21.2903 19.0323C21.3548 18.4516 20.9032 18 20.2581 18ZM28 18C28 23.4839 23.4839 28 18 28C12.5161 28 8 23.4839 8 18C8 12.5161 12.5161 8 18 8C23.4839 8 28 12.5161 28 18ZM24.6452 18C24.6452 17.2258 24 16.5806 23.2258 16.5806C22.8387 16.5806 22.5161 16.7742 22.2581 16.9677C21.2903 16.2581 19.871 15.8065 18.3871 15.7419L19.0323 12.6452L21.2258 13.0968C21.2258 13.6774 21.7419 14.0645 22.3226 14.0645C22.9032 14.0645 23.2903 13.5484 23.2903 12.9677C23.2903 12.3871 22.7742 11.9355 22.1935 11.9355C21.8065 11.9355 21.4839 12.129 21.2903 12.5161L18.8387 12C18.7742 12 18.7097 12 18.6452 12.0645C18.5806 12.129 18.5161 12.1935 18.5161 12.2581L17.8065 15.7419C16.2581 15.8065 14.8387 16.2581 13.871 16.9677C13.2903 16.3871 12.3871 16.4516 11.8065 17.0323C11.2258 17.6129 11.2903 18.5161 11.871 19.0968C12 19.2258 12.0645 19.2903 12.2581 19.3548C12.2581 19.5484 12.1935 19.6129 12.1935 19.8065C12.1935 22 14.7742 23.871 18 23.871C21.2258 23.871 23.8065 22.0645 23.8065 19.8065C23.8065 19.6129 23.8065 19.5484 23.7419 19.3548C24.3226 19.0968 24.6452 18.5806 24.6452 18Z" fill=""></path>
|
||
</g>
|
||
</svg> </a> </li><li> <a href="https://www.linkedin.com/shareArticle?url=https://www.datadoghq.com/blog/engineering/formal-modeling-and-simulation/" aria-label="Share on LinkedIn" target="_blank" rel="noopener noreferrer" class="group" data-dd-action-name="share: link: linkedin"> <svg aria-hidden="true" focusable="false" width="36" height="36" viewBox="0 0 36 36" xmlns="http://www.w3.org/2000/svg" class="[&>g>rect]:fill-grey-500 [&_path]:!fill-white w-7 h-7 tablet:w-8 tablet:h-8 desktop-sm:w-9 desktop-sm:h-9 group-hover:[&>g>rect]:fill-[#0A66C2]">
|
||
<g id="_atom_social-icons">
|
||
<rect width="36" height="36" rx="18" fill=""></rect>
|
||
<g id="Group">
|
||
<path id="Vector" fill-rule="evenodd" clip-rule="evenodd" d="M24.9355 9H11C9.90323 9 9 9.90323 9 11V25C9 26.0968 9.90323 27 11 27H25C26.0968 27 27 26.0968 27 25V11.0645C27 9.90323 26.0968 9 24.9355 9ZM14.4194 24.5484H11.7097V15.8387H14.4194V24.5484ZM13.0645 14.7419C12.1613 14.7419 11.5161 14.0323 11.5161 13.1935C11.5161 12.2903 12.2258 11.5806 13.0645 11.5806C13.9032 11.5806 14.6129 12.2903 14.6129 13.1935C14.6129 14.0968 13.9032 14.7419 13.0645 14.7419ZM24.4839 24.5484H21.7742V19.9677C21.7742 18.7419 21.3226 17.9677 20.2903 17.9677C19.1935 17.9677 18.6774 18.6774 18.6774 19.9677V24.5484H16.0968V15.8387H18.6774V17C18.6774 17 19.4516 15.5806 21.3226 15.5806C23.1935 15.5806 24.4839 16.6774 24.4839 19V24.5484Z" fill=""></path>
|
||
</g>
|
||
</g>
|
||
</svg> </a> </li> </ul></div></section><!-- Sidebar — second in DOM; CSS Grid places it visually left on desktop --><div class="desktop-sm:col-span-3 desktop-sm:col-start-1 desktop-sm:row-start-1 hidden desktop-sm:flex flex-col justify-between desktop-sm:pr-10 print:hidden"><aside class="aside-toc desktop-sm:block desktop-sm:sticky desktop-sm:top-[115px] hidden font-national-book border-t border-t-purple-600"> <nav class="toc desktop-sm:pt-4"> <p class="desktop-sm:mb-4 font-bold font-national text-16 leading-20">How we use formal modeling, lightweight simulations, and chaos testing to design reliable distributed systems</p> <ul class="flex flex-col gap-1">
|
||
<li class="leading-20 group">
|
||
<a href="#courier-a-mission-critical-multi-tenant-message-queuing-service" data-dd-action-name="toc: heading: Courier: a mission-critical, multi-tenant message queuing service" class="inline-block font-national font-normal border-l-2 hover:text-purple-700 border-grey-300 desktop-sm:pl-3 hover:border-purple-300 [.active>&]:border-purple-600 active:border-purple-600 [.active>&]:text-purple-600 group-has-[.active]:text-purple-600 group-has-[.active]:border-purple-600 active:text-purple-800 [.active>&]:font-bold group-has-[.active]:font-bold group-has-[.active]:hover:border-purple-700 group-has-[.active]:hover:text-purple-700 [.active]:hover:border-purple-700 [.active>&]:hover:text-purple-700 [.active]:active:border-purple-800 [.active>&]:active:text-purple-800">
|
||
Courier: a mission-critical, multi-tenant message queuing service
|
||
</a>
|
||
<ul class="flex-col gap-1 font-normal pl-9 my-1 hidden [.active_&]:flex has-[.active]:flex has-[.active]:font-bold"><li class="leading-4 hover:text-purple-300"><a href="#solving-for-multi-tenancy" data-dd-action-name="toc: heading: Solving for multi-tenancy" class="inline-block font-national font-normal border-l-2 [.active]:font-bold hover:text-purple-700 border-grey-300 desktop-sm:pl-3 hover:border-purple-300 [.active>&]:border-purple-600 active:border-purple-600 [.active>&]:text-purple-600 active:text-purple-800 [.active>&]:font-bold [.active]:hover:border-purple-700 [.active>&]:hover:text-purple-700 [.active]:active:border-purple-800 [.active>&]:active:text-purple-8000">Solving for multi-tenancy</a></li></ul>
|
||
</li>
|
||
|
||
<li class="leading-20 group">
|
||
<a href="#tla" data-dd-action-name="toc: heading: TLA+" class="inline-block font-national font-normal border-l-2 hover:text-purple-700 border-grey-300 desktop-sm:pl-3 hover:border-purple-300 [.active>&]:border-purple-600 active:border-purple-600 [.active>&]:text-purple-600 group-has-[.active]:text-purple-600 group-has-[.active]:border-purple-600 active:text-purple-800 [.active>&]:font-bold group-has-[.active]:font-bold group-has-[.active]:hover:border-purple-700 group-has-[.active]:hover:text-purple-700 [.active]:hover:border-purple-700 [.active>&]:hover:text-purple-700 [.active]:active:border-purple-800 [.active>&]:active:text-purple-800">
|
||
TLA+
|
||
</a>
|
||
<ul class="flex-col gap-1 font-normal pl-9 my-1 hidden [.active_&]:flex has-[.active]:flex has-[.active]:font-bold"><li class="leading-4 hover:text-purple-300"><a href="#tla-model-overview" data-dd-action-name="toc: heading: TLA+ model overview" class="inline-block font-national font-normal border-l-2 [.active]:font-bold hover:text-purple-700 border-grey-300 desktop-sm:pl-3 hover:border-purple-300 [.active>&]:border-purple-600 active:border-purple-600 [.active>&]:text-purple-600 active:text-purple-800 [.active>&]:font-bold [.active]:hover:border-purple-700 [.active>&]:hover:text-purple-700 [.active]:active:border-purple-800 [.active>&]:active:text-purple-8000">TLA+ model overview</a></li><li class="leading-4 hover:text-purple-300"><a href="#results" data-dd-action-name="toc: heading: Results" class="inline-block font-national font-normal border-l-2 [.active]:font-bold hover:text-purple-700 border-grey-300 desktop-sm:pl-3 hover:border-purple-300 [.active>&]:border-purple-600 active:border-purple-600 [.active>&]:text-purple-600 active:text-purple-800 [.active>&]:font-bold [.active]:hover:border-purple-700 [.active>&]:hover:text-purple-700 [.active]:active:border-purple-800 [.active>&]:active:text-purple-8000">Results</a></li></ul>
|
||
</li>
|
||
|
||
<li class="leading-20 group">
|
||
<a href="#simulations" data-dd-action-name="toc: heading: Simulations " class="inline-block font-national font-normal border-l-2 hover:text-purple-700 border-grey-300 desktop-sm:pl-3 hover:border-purple-300 [.active>&]:border-purple-600 active:border-purple-600 [.active>&]:text-purple-600 group-has-[.active]:text-purple-600 group-has-[.active]:border-purple-600 active:text-purple-800 [.active>&]:font-bold group-has-[.active]:font-bold group-has-[.active]:hover:border-purple-700 group-has-[.active]:hover:text-purple-700 [.active]:hover:border-purple-700 [.active>&]:hover:text-purple-700 [.active]:active:border-purple-800 [.active>&]:active:text-purple-800">
|
||
Simulations
|
||
</a>
|
||
<ul class="flex-col gap-1 font-normal pl-9 my-1 hidden [.active_&]:flex has-[.active]:flex has-[.active]:font-bold"><li class="leading-4 hover:text-purple-300"><a href="#closing-the-loop" data-dd-action-name="toc: heading: Closing the loop" class="inline-block font-national font-normal border-l-2 [.active]:font-bold hover:text-purple-700 border-grey-300 desktop-sm:pl-3 hover:border-purple-300 [.active>&]:border-purple-600 active:border-purple-600 [.active>&]:text-purple-600 active:text-purple-800 [.active>&]:font-bold [.active]:hover:border-purple-700 [.active>&]:hover:text-purple-700 [.active]:active:border-purple-800 [.active>&]:active:text-purple-8000">Closing the loop</a></li></ul>
|
||
</li>
|
||
|
||
<li class="leading-20 group">
|
||
<a href="#performance-testing" data-dd-action-name="toc: heading: Performance testing " class="inline-block font-national font-normal border-l-2 hover:text-purple-700 border-grey-300 desktop-sm:pl-3 hover:border-purple-300 [.active>&]:border-purple-600 active:border-purple-600 [.active>&]:text-purple-600 group-has-[.active]:text-purple-600 group-has-[.active]:border-purple-600 active:text-purple-800 [.active>&]:font-bold group-has-[.active]:font-bold group-has-[.active]:hover:border-purple-700 group-has-[.active]:hover:text-purple-700 [.active]:hover:border-purple-700 [.active>&]:hover:text-purple-700 [.active]:active:border-purple-800 [.active>&]:active:text-purple-800">
|
||
Performance testing
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="leading-20 group">
|
||
<a href="#updating-the-tla-model" data-dd-action-name="toc: heading: Updating the TLA+ model" class="inline-block font-national font-normal border-l-2 hover:text-purple-700 border-grey-300 desktop-sm:pl-3 hover:border-purple-300 [.active>&]:border-purple-600 active:border-purple-600 [.active>&]:text-purple-600 group-has-[.active]:text-purple-600 group-has-[.active]:border-purple-600 active:text-purple-800 [.active>&]:font-bold group-has-[.active]:font-bold group-has-[.active]:hover:border-purple-700 group-has-[.active]:hover:text-purple-700 [.active]:hover:border-purple-700 [.active>&]:hover:text-purple-700 [.active]:active:border-purple-800 [.active>&]:active:text-purple-800">
|
||
Updating the TLA+ model
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="leading-20 group">
|
||
<a href="#next-steps" data-dd-action-name="toc: heading: Next Steps" class="inline-block font-national font-normal border-l-2 hover:text-purple-700 border-grey-300 desktop-sm:pl-3 hover:border-purple-300 [.active>&]:border-purple-600 active:border-purple-600 [.active>&]:text-purple-600 group-has-[.active]:text-purple-600 group-has-[.active]:border-purple-600 active:text-purple-800 [.active>&]:font-bold group-has-[.active]:font-bold group-has-[.active]:hover:border-purple-700 group-has-[.active]:hover:text-purple-700 [.active]:hover:border-purple-700 [.active>&]:hover:text-purple-700 [.active]:active:border-purple-800 [.active>&]:active:text-purple-800">
|
||
Next Steps
|
||
</a>
|
||
|
||
</li>
|
||
</ul> </nav> <div x-data="{ activeButton: null, cardVisible: true, headerText: 'Did you find this article helpful?' }" x-show="cardVisible" x-transition:leave="transition ease-out duration-500" x-transition:leave-start="opacity-100 transform scale-100" x-transition:leave-end="opacity-0 transform scale-95" class="article-feedback-card w-[-webkit-fill-available] desktop-sm:-ml-8 relative opacity-0 transition-opacity duration-500 ease-in-out mt-12"> <div class="absolute top-3 -right-6"> <button title="Close feedback prompt" aria-label="Close feedback prompt" data-dd-action-name="side-nav: card: close feedback prompt" x-on:click="cardVisible = false" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
|
||
"><span class="sr-only">Close feedback prompt</span><svg aria-hidden="true" focusable="false" width="32" height="32" viewBox="0 0 32 32" xmlns="http://www.w3.org/2000/svg" class="h-4 w-4 fill-grey-400 hover:fill-purple-300 order-2 transition-default">
|
||
<path d="M7.6029 5.72327L5.71729 7.60889L24.3849 26.2765L26.2705 24.3909L7.6029 5.72327Z"></path>
|
||
<path d="M24.3849 5.7235L5.71729 24.3911L7.6029 26.2767L26.2705 7.60911L24.3849 5.7235Z"></path>
|
||
</svg></button> </div> <div class="flex flex-col gap-3 bg-white rounded shadow desktop-sm:p-9 desktop-sm:-mr-10"> <p x-text="headerText" class="p text-purple-neue italic">Did you find this article helpful?</p> <div class="article-feedback-button-container flex gap-4"> <button title="Article thumbs up" aria-label="Article thumbs up" data-dd-action-name="side-nav: card: article thumbs up" x-on:click="activeButton = 0; headerText = 'Thank you for your feedback'" x-bind:class="activeButton === 0 ? 'bg-purple-600 border-purple-600 [&_svg]:fill-white hover:bg-purple-600 cursor-default pointer-events-none' : ''" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
|
||
group p-2 rounded border border-grey-400 hover:bg-grey-200 active:bg-grey-400"><span class="sr-only">Article thumbs up</span><svg aria-hidden="true" focusable="false" width="24" height="24" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" class="h-6 w-6 fill-black order-2 transition-default">
|
||
<g clip-path="url(#clip0_1206_6)">
|
||
<path d="M23.05 7.275C22.6625 6.7875 22.0875 6.5 21.475 6.5H14.7875L15.7875 2.4875C15.9375 1.8875 15.8 1.2625 15.425 0.775C15.0375 0.2875 14.4625 0 13.85 0H10.9875C10.2375 0 9.5625 0.4125 9.2125 1.075L6.3875 6.5H1.5V20.5H6H8H18.975C19.9 20.5 20.6875 19.875 20.9125 18.9875L23.4125 8.9875C23.5625 8.3875 23.425 7.7625 23.05 7.275ZM6 18.5H3.5V8.5H6V18.5ZM21.475 8.5L18.975 18.5H8V7.75L10.9875 2H13.85L12.225 8.5H21.475Z" fill=""></path>
|
||
</g>
|
||
<defs>
|
||
<clipPath id="clip0_1206_6">
|
||
<rect width="24" height="24" fill=""></rect>
|
||
</clipPath>
|
||
</defs>
|
||
</svg></button> <button title="Article thumbs down" aria-label="Article thumbs down" data-dd-action-name="side-nav: card: article thumbs down" x-on:click="activeButton = 1; headerText = 'Thank you for your feedback'" x-bind:class="activeButton === 1 ? 'bg-purple-600 border-purple-600 [&_svg]:fill-white hover:bg-purple-600 cursor-default pointer-events-none' : ''" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
|
||
group p-2 rounded border border-grey-400 hover:bg-grey-200 active:bg-grey-400"><span class="sr-only">Article thumbs down</span><svg aria-hidden="true" focusable="false" width="24" height="24" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" class="h-6 w-6 fill-black order-2 transition-default">
|
||
<path d="M0.95 16.725C1.3375 17.2125 1.9125 17.5 2.525 17.5H9.2125L8.2125 21.5125C8.0625 22.1125 8.2 22.7375 8.575 23.225C8.9625 23.7125 9.5375 24 10.15 24H13.0125C13.7625 24 14.4375 23.5875 14.7875 22.925L17.6 17.5H22.5V3.5H18H16H5.025C4.1 3.5 3.3125 4.125 3.0875 5.0125L0.5875 15.0125C0.4375 15.6125 0.575 16.2375 0.95 16.725ZM18 5.5H20.5V15.5H18V5.5ZM2.525 15.5L5.025 5.5H16V16.25L13.0125 22H10.15L11.775 15.5H2.525Z" fill=""></path>
|
||
</svg></button> </div> </div> </div> </aside> <script type="module">const o=new IntersectionObserver(e=>{e.forEach(t=>{const n=t.target.getAttribute("id"),r=document.querySelector(`nav.toc li a[href="#${n}"]`);r&&t.isIntersecting&&(document.querySelectorAll("nav.toc li").forEach(c=>{c.classList.remove("active")}),r.parentElement.classList.add("active"))})},{root:null,rootMargin:"0px 0px -33% 0px",threshold:[1]});document.querySelectorAll(".article-content h1, .article-content h2, .article-content h3").forEach(e=>{o.observe(e)});</script><div class="further-reading-card border-2 border-purple-300 rounded-lg desktop-sm:mt-auto desktop-sm:p-9 desktop-sm:-mr-10 desktop-sm:-ml-8"> <p class="h5 uppercase"> Further reading </p> <div class="desktop-sm:mb-2"></div> <p class="h4 text-purple-600">Datadog Platform Datasheet</p> <div class="desktop-sm:mb-4"></div> <picture class> <source srcset="https://web-assets.dd-static.net/42588/1777497550-thumbnail-datadog-platform-datasheet_updated.png?format=auto&fit=bounds&quality=90&disable=upscale&width=276&dpr=1 1x, https://web-assets.dd-static.net/42588/1777497550-thumbnail-datadog-platform-datasheet_updated.png?format=auto&fit=bounds&quality=90&disable=upscale&width=276&dpr=1.5 1.5x, https://web-assets.dd-static.net/42588/1777497550-thumbnail-datadog-platform-datasheet_updated.png?format=auto&fit=bounds&quality=90&disable=upscale&width=276&dpr=2 2x, https://web-assets.dd-static.net/42588/1777497550-thumbnail-datadog-platform-datasheet_updated.png?format=auto&fit=bounds&quality=90&disable=upscale&width=276&dpr=2.5 2.5x, https://web-assets.dd-static.net/42588/1777497550-thumbnail-datadog-platform-datasheet_updated.png?format=auto&fit=bounds&quality=90&disable=upscale&width=276&dpr=3 3x, https://web-assets.dd-static.net/42588/1777497550-thumbnail-datadog-platform-datasheet_updated.png?format=auto&fit=bounds&quality=90&disable=upscale&width=276&dpr=3.5 3.5x, https://web-assets.dd-static.net/42588/1777497550-thumbnail-datadog-platform-datasheet_updated.png?format=auto&fit=bounds&quality=90&disable=upscale&width=276&dpr=4 4x"> <img src="https://web-assets.dd-static.net/42588/1777497550-thumbnail-datadog-platform-datasheet_updated.png?format=auto&fit=bounds&quality=90&disable=upscale&width=276&dpr=1" alt="Datadog Platform Datasheet" loading="lazy" fetchpriority="low" class="w-full"> </picture> <div class="desktop-sm:mb-4"></div> <p class="p-small">Learn about the key components, capabilities, and features of the Datadog platform.</p> <div class="desktop-sm:mb-6"></div> <a title="Download to learn more" aria-label="Download to learn more" href="https://www.datadoghq.com/resources/datadog-datasheet/?utm_source=inbound&utm_medium=corpsite-display&utm_campaign=dg-coreplatform-ww-blog-toc-datasheet-datadog" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
text-white bg-purple-neue hover:bg-purple-secondary active:bg-purple-neue rounded py-1.5 px-3 w-fit button-roboto-mono"><span class="block false">Download to learn more</span></a> </div></div></div><div class="print:hidden"><div class="desktop:mb-[120px] tablet:mb-24 mb-[72px]"></div><!-- Related Posts (client-side, fetched from Typesense) --><div id="related-posts" data-tags="["message queue"]" data-current-id="/blog/engineering/formal-modeling-and-simulation/" data-lang="en" data-path-to-home="/blog/"></div> <script type="module" src="/blog/_astro/RelatedPosts.astro_astro_type_script_index_0_lang.jg5bITdA.js"></script><!-- Related Jobs --><div id="related-jobs-section" data-tags="["message queue"]" data-related-job-ids="[]"> <div id="related-jobs-sentinel" class="h-px" aria-hidden="true"></div> <div id="related-jobs-content" class="hidden"> <div class="desktop-sm:mb-[76px] tablet:mb-15 mb-12"></div> <section class="container"> <h2 class="h1">Related jobs at Datadog</h2> <div class="desktop-sm:mb-4 tablet:mb-3 mb-2"></div> <h3 class="subheader3">We're always looking for talented people to collaborate with</h3> <div class="desktop-sm:mb-12 tablet:mb-9 mb-7"></div> <p class="p-small font-bold font-national uppercase">Featured positions</p> <div class="desktop-sm:mb-6 tablet:mb-5 mb-3"></div> <div id="related-jobs-table"> <table class="table job-listing w-full"> <tbody> </tbody> </table> </div> <div class="desktop-sm:mt-9 tablet:mb-7 mb-5"></div> <div class="flex flex-col"> <div id="total-jobs-count" class="hidden"> <p class="subheader3">
|
||
We have <span class="text-purple-neue" id="jobs-count"></span> positions
|
||
</p> <div class="desktop-sm:mb-6 tablet:mb-5 mb-3"></div> </div> <div id="view-all-button-container"> <button title="View all" aria-label="View all" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
|
||
text-purple-600 hover:bg-purple-200 active:bg-purple-400 rounded py-1.5 px-2.5 tablet:py-2 tablet:px-4 desktop-sm:py-2.5 desktop-sm:px-6 w-fit border-2 border-purple-60 button-text"><span class="block false">View all</span></button> </div> </div> </section> </div> </div> <script type="module" src="/blog/_astro/RelatedJobs.astro_astro_type_script_index_0_lang.Duc5wx_Q.js"></script><div class="desktop:mb-[76px] tablet:mb-[68px] mb-12"></div><div class="bg-gradient-to-r from-purple-600 from-0% to-purple-800 to-100%"> <div class="flex flex-col tablet:flex-row justify-between tablet:items-center gap-4 tablet:gap-0 py-7 px-4.5 tablet:py-9 tablet:px-[45px] desktop-sm:p-12 desktop-lg:px-14 desktop-lg:max-w-screen-desktop-lg mx-auto"> <h2 class="text-white font-bold font-national tracking-[-0.24px] tablet:tracking-[-0.28px] desktop-sm:tracking-[-0.34px] text-24 leading-24 tablet:text-28 tablet:leading-28 desktop-sm:text-34 desktop-sm:leading-34"> Start monitoring your metrics in minutes </h2> <button title="find out how" aria-label="find out how" class="flex items-center gap-1 tablet:gap-1.5 desktop-sm:gap-2 h-fit group
|
||
signup-trigger
|
||
bg-white text-purple-600 hover:bg-purple-200 active:bg-purple-400 rounded py-2 px-4 tablet:py-2.5 tablet:px-6 desktop-sm:py-3 desktop-sm:px-[26px] w-fit button-text"><span class="block false">find out how</span></button> </div> </div></div> </main> <div><html><head></head><body><footer x-data="{
|
||
openSection: '',
|
||
openCategory: '',
|
||
mobile: false,
|
||
toggle(name) {
|
||
if (!this.mobile) return;
|
||
this.openCategory = '';
|
||
this.openSection = this.openSection === name ? false : name;
|
||
},
|
||
toggleCategory(name) {
|
||
if (!this.mobile) return;
|
||
this.openCategory = this.openCategory === name ? '' : name;
|
||
},
|
||
isOpen(name) { return this.mobile && this.openSection === name },
|
||
isCategoryOpen(name) { return this.mobile && this.openCategory === name },
|
||
isVisible(name) { return !this.mobile || this.openSection === name },
|
||
isCategoryVisible(name) { return !this.mobile || this.openCategory === name },
|
||
reactOnResize() {
|
||
this.mobile = window.innerWidth < 768;
|
||
if (!this.mobile) { this.openSection = ''; this.openCategory = ''; }
|
||
}
|
||
}" x-init="reactOnResize()" @resize.window.throttle.200ms="reactOnResize()" class="footer bg-[#110617] text-white"><div class="footer-content py-5 tablet:py-7"><nav aria-label="Footer" class="footer-nav container mb-7 tablet:mb-9 [@media(min-width:1200px)]:mb-12"><ul class="footer-sections flex flex-col list-none m-0 p-0 gap-y-4 tablet:gap-y-9 [@media(min-width:1200px)]:mx-[-0.75rem] [@media(min-width:1200px)]:flex-row [@media(min-width:1200px)]:flex-wrap"><li class="footer-product-section w-full [@media(min-width:1200px)]:w-2/3 [@media(min-width:1200px)]:px-3" x-data="{ section: 'product' }"><button type="button" @click="toggle(section)" :class="isOpen(section) ? 'open text-[#B37BFF] border-b-white' : 'text-white border-b-[#323232]'" :aria-expanded="isVisible(section)" aria-controls="footer-section-product" :disabled="!mobile" class="group/section footer-section-header flex w-full items-center justify-between border-0 border-b border-solid bg-transparent p-0 pb-4 text-left font-bold font-national text-[18px] leading-[22px] cursor-pointer tablet:pb-1.5 tablet:mb-2 tablet:cursor-default tablet:border-b-white/50 tablet:block [@media(min-width:1200px)]:pb-2 [@media(min-width:1200px)]:mb-3">
|
||
<span class="capitalize">Product</span><svg class="block tablet:hidden w-[18px] h-[18px] fill-current transition-transform group-[.open]/section:rotate-180" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M96,144.313,32.343,80.657,43.657,69.343,96,121.687l52.343-52.344,11.314,11.314Z"></path>
|
||
</svg></button><ul id="footer-section-product" x-show.important="isVisible(section)" class="footer-category-list flex flex-col list-none m-0 p-0 tablet:mx-[-0.75rem] tablet:flex-wrap tablet:max-h-[1050px] tablet:gap-y-4 [@media(min-width:992px)]:gap-y-5 [@media(min-width:1200px)]:max-h-[870px]"><li class="footer-category-wrapper w-full tablet:px-3 tablet:w-1/3 [@media(min-width:768px)_and_(max-width:1199.98px)]:[order:var(--footer-tablet-order,0)] [@media(min-width:1200px)]:w-1/4" style="--footer-tablet-order:1" x-data="{ category: 'observability' }"><div class="footer-category-block border-b border-solid border-[#323232] tablet:border-b-0"><button type="button" @click="toggleCategory(category)" :class="isCategoryOpen(category) ? 'open pb-1.5' : ''" :aria-expanded="isCategoryVisible(category)" aria-controls="footer-category-observability" :disabled="!mobile" class="group/category footer-product-category flex w-full items-center justify-between border-0 bg-transparent px-0 py-4 text-left font-bold font-national text-[#969696] text-[18px] leading-[22px] cursor-pointer tablet:block tablet:w-fit tablet:py-0 tablet:mb-2 tablet:cursor-default tablet:text-[16px] tablet:leading-[20px] [@media(min-width:1200px)]:mb-3">
|
||
<span class="flex items-center justify-start text-left uppercase gap-x-1 [@media(min-width:1200px)]:gap-x-2"><svg class="footer-category-icon w-[18px] h-[18px] fill-[#B37BFF]" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M96,136a40,40,0,1,1,40-40A40.045,40.045,0,0,1,96,136Zm0-64a24,24,0,1,0,24,24A24.028,24.028,0,0,0,96,72Zm0,92C35.749,164,4.16,102.208,2.845,99.578L1.056,96l1.789-3.578C4.16,89.792,35.749,28,96,28s91.84,61.792,93.155,64.422L190.944,96l-1.789,3.578C187.84,102.208,156.252,164,96,164ZM19.165,96C26.457,108.305,53.3,148,96,148s69.544-39.7,76.836-52C165.551,83.708,138.706,44,96,44S26.457,83.7,19.165,96Z"></path>
|
||
</svg>
|
||
Observability</span><svg class="block tablet:hidden w-[18px] h-[18px] fill-current transition-transform group-[.open]/category:rotate-180" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M96,144.313,32.343,80.657,43.657,69.343,96,121.687l52.343-52.344,11.314,11.314Z"></path>
|
||
</svg></button><ul id="footer-category-observability" x-show.important="isCategoryVisible(category)" class="footer-subcategory-list flex flex-col list-none mt-0 mb-5 p-0 gap-y-3 tablet:mb-0"><li class="footer-subcategory"><span class="footer-subcategory-label block font-bold font-national text-[16px] leading-[20px]">Infrastructure</span><ul class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-1.5 tablet:mt-2"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/infrastructure-monitoring/">Infrastructure Monitoring</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/metrics/">Custom Metrics</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/network-monitoring/">Network Monitoring</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/container-monitoring/">Container Monitoring</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/kubernetes-autoscaling/">Kubernetes Autoscaling</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/serverless-monitoring/">Serverless Monitoring</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/cloud-cost-management/">Cloud Cost Management</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/storage-management/">Storage Management</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/gpu-monitoring/">GPU Monitoring</a></li></ul></li><li class="footer-subcategory"><span class="footer-subcategory-label block font-bold font-national text-[16px] leading-[20px]">Applications</span><ul class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-1.5 tablet:mt-2"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/apm/">Application Performance Monitoring</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/universal-service-monitoring/">Universal Service Monitoring</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/code-profiling/">Continuous Profiler</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/dynamic-instrumentation/">Dynamic Instrumentation</a></li></ul></li><li class="footer-subcategory"><span class="footer-subcategory-label block font-bold font-national text-[16px] leading-[20px]">Data</span><ul class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-1.5 tablet:mt-2"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/database-monitoring/">Database Monitoring</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/products/observability/data-observability/">Data Observability</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/data-streams-monitoring/">Data Streams Monitoring</a></li></ul></li><li class="footer-subcategory"><span class="footer-subcategory-label block font-bold font-national text-[16px] leading-[20px]">Logs</span><ul class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-1.5 tablet:mt-2"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/log-management/">Log Management</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/audit-trail/">Audit Trail</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/observability-pipelines/">Observability Pipelines</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/error-tracking/">Error Tracking</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/byoc-logs/">Bring Your Own Cloud Log Management</a></li></ul></li></ul></div></li><li class="footer-column-break hidden w-0 min-w-0 shrink-0 m-0 p-0 [@media(min-width:768px)_and_(max-width:1199.98px)]:basis-[900px] [@media(min-width:768px)_and_(max-width:1199.98px)]:[order:var(--footer-tablet-order,0)] [@media(min-width:1200px)]:basis-[750px] footer-column-break--desktop [@media(min-width:1200px)]:block" aria-hidden="true" style="--footer-tablet-order:1"></li><li class="footer-category-wrapper w-full tablet:px-3 tablet:w-1/3 [@media(min-width:768px)_and_(max-width:1199.98px)]:[order:var(--footer-tablet-order,0)] [@media(min-width:1200px)]:w-1/4" style="--footer-tablet-order:3" x-data="{ category: 'security' }"><div class="footer-category-block border-b border-solid border-[#323232] tablet:border-b-0"><button type="button" @click="toggleCategory(category)" :class="isCategoryOpen(category) ? 'open pb-1.5' : ''" :aria-expanded="isCategoryVisible(category)" aria-controls="footer-category-security" :disabled="!mobile" class="group/category footer-product-category flex w-full items-center justify-between border-0 bg-transparent px-0 py-4 text-left font-bold font-national text-[#969696] text-[18px] leading-[22px] cursor-pointer tablet:block tablet:w-fit tablet:py-0 tablet:mb-2 tablet:cursor-default tablet:text-[16px] tablet:leading-[20px] [@media(min-width:1200px)]:mb-3">
|
||
<span class="flex items-center justify-start text-left uppercase gap-x-1 [@media(min-width:1200px)]:gap-x-2"><svg class="footer-category-icon w-[18px] h-[18px] fill-[#B37BFF]" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>security-platform</title>
|
||
<path d="M16 0.793l-12 6.938v9.604c0.007 4.42 2.392 8.281 5.944 10.372l0.056 0.031 6 3.468 6-3.469c3.608-2.122 5.993-5.983 6-10.402v-9.605zM6.667 10.811l8 4.625v4.42l-8-4.625zM7.997 8.5l6.669-3.856v7.711zM6.723 18.343l7.944 4.593v4.42l-3.333-1.926c-2.521-1.484-4.269-4.050-4.607-7.044l-0.004-0.043zM25.333 17.335c-0.005 3.439-1.86 6.443-4.623 8.070l-0.044 0.024-3.333 1.927v-22.712l8 4.625z"></path>
|
||
</svg>
|
||
Security</span><svg class="block tablet:hidden w-[18px] h-[18px] fill-current transition-transform group-[.open]/category:rotate-180" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M96,144.313,32.343,80.657,43.657,69.343,96,121.687l52.343-52.344,11.314,11.314Z"></path>
|
||
</svg></button><ul id="footer-category-security" x-show.important="isCategoryVisible(category)" class="footer-subcategory-list flex flex-col list-none mt-0 mb-5 p-0 gap-y-3 tablet:mb-0"><li class="footer-subcategory"><span class="footer-subcategory-label block font-bold font-national text-[16px] leading-[20px]">Code Security</span><ul class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-1.5 tablet:mt-2"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/code-security/">Code Security</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/software-composition-analysis/">Software Composition Analysis</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/sast/">Static Code Analysis</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/iast/">Runtime Code Analysis</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/iac-security">IaC Security</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/secret-scanning/">Secret Scanning</a></li></ul></li><li class="footer-subcategory"><span class="footer-subcategory-label block font-bold font-national text-[16px] leading-[20px]">Cloud Security</span><ul class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-1.5 tablet:mt-2"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/cloud-security/">Cloud Security</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/sensitive-data-scanner/">Sensitive Data Scanner</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/cloud-security/#posture-management">Cloud Security Posture Management</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/cloud-security/#entitlement-management">Cloud Infrastructure Entitlement Management</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/cloud-security/#vulnerability-management">Vulnerability Management</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/cloud-security/#compliance">Compliance</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/workload-protection/">Cloud Workload Protection</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/app-and-api-protection/">App & API Protection</a></li></ul></li><li class="footer-subcategory"><span class="footer-subcategory-label block font-bold font-national text-[16px] leading-[20px]">SIEM</span><ul class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-1.5 tablet:mt-2"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/cloud-siem/">Cloud SIEM</a></li></ul></li></ul></div></li><li class="footer-column-break hidden w-0 min-w-0 shrink-0 m-0 p-0 [@media(min-width:768px)_and_(max-width:1199.98px)]:basis-[900px] [@media(min-width:768px)_and_(max-width:1199.98px)]:[order:var(--footer-tablet-order,0)] [@media(min-width:1200px)]:basis-[750px] footer-column-break--desktop [@media(min-width:1200px)]:block" aria-hidden="true" style="--footer-tablet-order:3"></li><li class="footer-category-wrapper w-full tablet:px-3 tablet:w-1/3 [@media(min-width:768px)_and_(max-width:1199.98px)]:[order:var(--footer-tablet-order,0)] [@media(min-width:1200px)]:w-1/4" style="--footer-tablet-order:2" x-data="{ category: 'digital-experience' }"><div class="footer-category-block border-b border-solid border-[#323232] tablet:border-b-0"><button type="button" @click="toggleCategory(category)" :class="isCategoryOpen(category) ? 'open pb-1.5' : ''" :aria-expanded="isCategoryVisible(category)" aria-controls="footer-category-digital-experience" :disabled="!mobile" class="group/category footer-product-category flex w-full items-center justify-between border-0 bg-transparent px-0 py-4 text-left font-bold font-national text-[#969696] text-[18px] leading-[22px] cursor-pointer tablet:block tablet:w-fit tablet:py-0 tablet:mb-2 tablet:cursor-default tablet:text-[16px] tablet:leading-[20px] [@media(min-width:1200px)]:mb-3">
|
||
<span class="flex items-center justify-start text-left uppercase gap-x-1 [@media(min-width:1200px)]:gap-x-2"><svg class="footer-category-icon w-[18px] h-[18px] fill-[#B37BFF]" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>rum</title>
|
||
<path d="M8.728 32h-8.728v-14.544h8.728v14.544zM2.912 29.088h2.912v-8.728h-2.912v8.728zM18.908 29.088h-7.269v-2.912h4.364v-14.544h-10.187v2.912h-2.905v-5.816h16.003v20.361zM32 20.361h-10.18v-2.912h7.276v-14.544h-20.367v2.912h-2.912v-5.816h26.184v20.361z"></path>
|
||
</svg>
|
||
Digital Experience</span><svg class="block tablet:hidden w-[18px] h-[18px] fill-current transition-transform group-[.open]/category:rotate-180" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M96,144.313,32.343,80.657,43.657,69.343,96,121.687l52.343-52.344,11.314,11.314Z"></path>
|
||
</svg></button><ul id="footer-category-digital-experience" x-show.important="isCategoryVisible(category)" class="footer-subcategory-list flex flex-col list-none mt-0 mb-5 p-0 gap-y-3 tablet:mb-0"><li class="footer-subcategory"><span class="footer-subcategory-label block font-bold font-national text-[16px] leading-[20px]">Digital Experience</span><ul class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-1.5 tablet:mt-2"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/product-analytics/">Product Analytics</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/experiments/">Experiments</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/real-user-monitoring/">Browser Real User Monitoring</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/real-user-monitoring/mobile-rum/">Mobile Real User Monitoring</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/real-user-monitoring/session-replay/">Session Replay</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/synthetic-monitoring/">Synthetic Monitoring</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/mobile-app-testing/">Mobile App Testing</a></li></ul></li></ul></div></li><li class="footer-column-break hidden w-0 min-w-0 shrink-0 m-0 p-0 [@media(min-width:768px)_and_(max-width:1199.98px)]:basis-[900px] [@media(min-width:768px)_and_(max-width:1199.98px)]:[order:var(--footer-tablet-order,0)] [@media(min-width:1200px)]:basis-[750px] footer-column-break--tablet [@media(min-width:768px)_and_(max-width:1199.98px)]:block" aria-hidden="true" style="--footer-tablet-order:2"></li><li class="footer-category-wrapper w-full tablet:px-3 tablet:w-1/3 [@media(min-width:768px)_and_(max-width:1199.98px)]:[order:var(--footer-tablet-order,0)] [@media(min-width:1200px)]:w-1/4" style="--footer-tablet-order:4" x-data="{ category: 'software-delivery' }"><div class="footer-category-block border-b border-solid border-[#323232] tablet:border-b-0"><button type="button" @click="toggleCategory(category)" :class="isCategoryOpen(category) ? 'open pb-1.5' : ''" :aria-expanded="isCategoryVisible(category)" aria-controls="footer-category-software-delivery" :disabled="!mobile" class="group/category footer-product-category flex w-full items-center justify-between border-0 bg-transparent px-0 py-4 text-left font-bold font-national text-[#969696] text-[18px] leading-[22px] cursor-pointer tablet:block tablet:w-fit tablet:py-0 tablet:mb-2 tablet:cursor-default tablet:text-[16px] tablet:leading-[20px] [@media(min-width:1200px)]:mb-3">
|
||
<span class="flex items-center justify-start text-left uppercase gap-x-1 [@media(min-width:1200px)]:gap-x-2"><svg class="footer-category-icon w-[18px] h-[18px] fill-[#B37BFF]" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>ci</title>
|
||
<path d="M30.667 22.625v0.087c-0.006 4.392-3.565 7.95-7.956 7.955h-0.085c-3.937-0.005-7.205-2.864-7.845-6.62l-0.007-0.047h-3.107l4.333-4.333 4.333 4.333h-2.833c0.6 2.315 2.669 3.998 5.131 4h0.087c2.916-0.009 5.278-2.373 5.282-5.29v-0.085c-0.003-2.921-2.369-5.288-5.29-5.292h-13.42c-4.393-0.004-7.954-3.565-7.957-7.958v-0.087c0.006-4.392 3.565-7.95 7.956-7.955h0.085c4.393 0.004 7.954 3.564 7.958 7.956v2.044h-2.667v-2.043c-0.004-2.921-2.371-5.287-5.291-5.29h-0.087c-2.919 0.005-5.285 2.371-5.288 5.29v0.085c0.003 2.921 2.369 5.288 5.29 5.292h13.42c4.393 0.004 7.954 3.565 7.957 7.958v0z"></path>
|
||
</svg>
|
||
Software Delivery</span><svg class="block tablet:hidden w-[18px] h-[18px] fill-current transition-transform group-[.open]/category:rotate-180" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M96,144.313,32.343,80.657,43.657,69.343,96,121.687l52.343-52.344,11.314,11.314Z"></path>
|
||
</svg></button><ul id="footer-category-software-delivery" x-show.important="isCategoryVisible(category)" class="footer-subcategory-list flex flex-col list-none mt-0 mb-5 p-0 gap-y-3 tablet:mb-0"><li class="footer-subcategory"><span class="footer-subcategory-label block font-bold font-national text-[16px] leading-[20px]">Software Delivery</span><ul class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-1.5 tablet:mt-2"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/continuous-testing/">Continuous Testing</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/internal-developer-portal/">Internal Developer Portal (IDP)</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/ci-cd-monitoring/">CI Visibility</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/test-optimization/">Test Optimization</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/feature-flags/">Feature Flags</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/code-coverage/">Code Coverage</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/platform/dora-metrics/">DORA Metrics + AI Impact</a></li></ul></li></ul></div></li><li class="footer-category-wrapper w-full tablet:px-3 tablet:w-1/3 [@media(min-width:768px)_and_(max-width:1199.98px)]:[order:var(--footer-tablet-order,0)] [@media(min-width:1200px)]:w-1/4" style="--footer-tablet-order:5" x-data="{ category: 'service-management' }"><div class="footer-category-block border-b border-solid border-[#323232] tablet:border-b-0"><button type="button" @click="toggleCategory(category)" :class="isCategoryOpen(category) ? 'open pb-1.5' : ''" :aria-expanded="isCategoryVisible(category)" aria-controls="footer-category-service-management" :disabled="!mobile" class="group/category footer-product-category flex w-full items-center justify-between border-0 bg-transparent px-0 py-4 text-left font-bold font-national text-[#969696] text-[18px] leading-[22px] cursor-pointer tablet:block tablet:w-fit tablet:py-0 tablet:mb-2 tablet:cursor-default tablet:text-[16px] tablet:leading-[20px] [@media(min-width:1200px)]:mb-3">
|
||
<span class="flex items-center justify-start text-left uppercase gap-x-1 [@media(min-width:1200px)]:gap-x-2"><svg class="footer-category-icon w-[18px] h-[18px] fill-[#B37BFF]" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M88,188.38V91.38l84-48.5v96.99l-84,48.51Zm16-87.76v60.04l52-30.02v-60.04l-52,30.02Zm-50-28.87v97l16-9.24V80.99l52-30.02,16-9.24V23.25L54,71.75ZM20,52.12v97l16-9.24V61.36l52-30.02,16-9.24V3.62L20,52.12Z"></path>
|
||
</svg>
|
||
Service Management</span><svg class="block tablet:hidden w-[18px] h-[18px] fill-current transition-transform group-[.open]/category:rotate-180" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M96,144.313,32.343,80.657,43.657,69.343,96,121.687l52.343-52.344,11.314,11.314Z"></path>
|
||
</svg></button><ul id="footer-category-service-management" x-show.important="isCategoryVisible(category)" class="footer-subcategory-list flex flex-col list-none mt-0 mb-5 p-0 gap-y-3 tablet:mb-0"><li class="footer-subcategory"><span class="footer-subcategory-label block font-bold font-national text-[16px] leading-[20px]">Incident Response</span><ul class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-1.5 tablet:mt-2"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/incident-response/">Incident Management</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/case-management/">Work Management</a></li></ul></li><li class="footer-subcategory"><span class="footer-subcategory-label block font-bold font-national text-[16px] leading-[20px]">Actions</span><ul class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-1.5 tablet:mt-2"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/workflow-automation/">Workflow Automation</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/app-builder/">Datadog Apps</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/event-management/">Event Management</a></li></ul></li><li class="footer-subcategory"><span class="footer-subcategory-label block font-bold font-national text-[16px] leading-[20px]">Context Graph</span><ul class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-1.5 tablet:mt-2"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/software-catalog/">Software Catalog</a></li></ul></li></ul></div></li><li class="footer-column-break hidden w-0 min-w-0 shrink-0 m-0 p-0 [@media(min-width:768px)_and_(max-width:1199.98px)]:basis-[900px] [@media(min-width:768px)_and_(max-width:1199.98px)]:[order:var(--footer-tablet-order,0)] [@media(min-width:1200px)]:basis-[750px] footer-column-break--tablet [@media(min-width:768px)_and_(max-width:1199.98px)]:block footer-column-break--desktop [@media(min-width:1200px)]:block" aria-hidden="true" style="--footer-tablet-order:5"></li><li class="footer-category-wrapper w-full tablet:px-3 tablet:w-1/3 [@media(min-width:768px)_and_(max-width:1199.98px)]:[order:var(--footer-tablet-order,0)] [@media(min-width:1200px)]:w-1/4" style="--footer-tablet-order:6" x-data="{ category: 'ai' }"><div class="footer-category-block border-b border-solid border-[#323232] tablet:border-b-0"><button type="button" @click="toggleCategory(category)" :class="isCategoryOpen(category) ? 'open pb-1.5' : ''" :aria-expanded="isCategoryVisible(category)" aria-controls="footer-category-ai" :disabled="!mobile" class="group/category footer-product-category flex w-full items-center justify-between border-0 bg-transparent px-0 py-4 text-left font-bold font-national text-[#969696] text-[18px] leading-[22px] cursor-pointer tablet:block tablet:w-fit tablet:py-0 tablet:mb-2 tablet:cursor-default tablet:text-[16px] tablet:leading-[20px] [@media(min-width:1200px)]:mb-3">
|
||
<span class="flex items-center justify-start text-left uppercase gap-x-1 [@media(min-width:1200px)]:gap-x-2"><svg class="footer-category-icon w-[18px] h-[18px] fill-[#B37BFF]" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" fill="" viewBox="0 0 192 192" style="enable-background:new 0 0 192 192;" xml:space="preserve"><g id="Layer_7"><g><linearGradient id="SVGID_1_" gradientUnits="userSpaceOnUse" x1="89.0192" y1="122.732" x2="142.9808" y2="29.268"><stop offset="0" style="stop-color:#5500FF"></stop><stop offset="0.1715" style="stop-color:#4D09FF"></stop><stop offset="0.4511" style="stop-color:#3821FF"></stop><stop offset="0.802" style="stop-color:#1649FF"></stop><stop offset="1" style="stop-color:#0062FF"></stop></linearGradient><path class="st5" d="M120,16h-8c0,30.88-25.12,56-56,56v8c30.88,0,56,25.12,56,56h8c0-30.88,25.12-56,56-56v-8 C145.12,72,120,46.88,120,16z"></path><linearGradient id="SVGID_00000049207151058930698180000011839073544639225000_" gradientUnits="userSpaceOnUse" x1="35.4115" y1="168.7311" x2="68.5885" y2="111.267"><stop offset="0" style="stop-color:#5500FF"></stop><stop offset="0.1715" style="stop-color:#4D09FF"></stop><stop offset="0.4511" style="stop-color:#3821FF"></stop><stop offset="0.802" style="stop-color:#1649FF"></stop><stop offset="1" style="stop-color:#0062FF"></stop></linearGradient><path d="M56,104h-4c0,0,0,0,0,0c0,0,0,0,0,0h-4 c0,17.64-14.35,32-32,32v8c17.65,0,32,14.36,32,32h8c0-17.64,14.35-32,32-32v-8C70.35,136,56,121.64,56,104z"></path></g></g></svg>
|
||
AI</span><svg class="block tablet:hidden w-[18px] h-[18px] fill-current transition-transform group-[.open]/category:rotate-180" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M96,144.313,32.343,80.657,43.657,69.343,96,121.687l52.343-52.344,11.314,11.314Z"></path>
|
||
</svg></button><ul id="footer-category-ai" x-show.important="isCategoryVisible(category)" class="footer-subcategory-list flex flex-col list-none mt-0 mb-5 p-0 gap-y-3 tablet:mb-0"><li class="footer-subcategory"><span class="footer-subcategory-label block font-bold font-national text-[16px] leading-[20px]">AI for Datadog</span><ul class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-1.5 tablet:mt-2"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/ai/bits-investigation/">Bits Investigation</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/ai/bits-ai-agents/">Bits AI</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/ai/bits-chat/">Bits Chat</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/ai/bits-agent-builder/">Bits Agent Builder</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/ai/bits-code/">Bits Code</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/ai/bits-security-analyst/">Bits Security Analyst</a></li></ul></li><li class="footer-subcategory"><span class="footer-subcategory-label block font-bold font-national text-[16px] leading-[20px]">Datadog for AI</span><ul class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-1.5 tablet:mt-2"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/products/ai/agent-observability/">Agent Observability</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/ai/mcp-server/">MCP Server</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/ai/mcp-server/">Pup CLI</a></li></ul></li></ul></div></li><li class="footer-category-wrapper w-full tablet:px-3 tablet:w-1/3 [@media(min-width:768px)_and_(max-width:1199.98px)]:[order:var(--footer-tablet-order,0)] [@media(min-width:1200px)]:w-1/4" style="--footer-tablet-order:7" x-data="{ category: 'platform' }"><div class="footer-category-block border-b border-solid border-[#323232] tablet:border-b-0"><button type="button" @click="toggleCategory(category)" :class="isCategoryOpen(category) ? 'open pb-1.5' : ''" :aria-expanded="isCategoryVisible(category)" aria-controls="footer-category-platform" :disabled="!mobile" class="group/category footer-product-category flex w-full items-center justify-between border-0 bg-transparent px-0 py-4 text-left font-bold font-national text-[#969696] text-[18px] leading-[22px] cursor-pointer tablet:block tablet:w-fit tablet:py-0 tablet:mb-2 tablet:cursor-default tablet:text-[16px] tablet:leading-[20px] [@media(min-width:1200px)]:mb-3">
|
||
<span class="flex items-center justify-start text-left uppercase gap-x-1 [@media(min-width:1200px)]:gap-x-2"><svg class="footer-category-icon w-[18px] h-[18px] fill-[#B37BFF]" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>dashboard</title>
|
||
<path d="M32 24.876v1.79h-32v-21.333h1.988v19.543h30.012zM26.009 10.679l3.976 12.432h-25.982v-8.006l6.999-7.981 8.987 7.981 6.020-4.426z"></path>
|
||
</svg>
|
||
Platform Capabilities</span><svg class="block tablet:hidden w-[18px] h-[18px] fill-current transition-transform group-[.open]/category:rotate-180" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M96,144.313,32.343,80.657,43.657,69.343,96,121.687l52.343-52.344,11.314,11.314Z"></path>
|
||
</svg></button><ul id="footer-category-platform" x-show.important="isCategoryVisible(category)" class="footer-subcategory-list flex flex-col list-none mt-0 mb-5 p-0 gap-y-3 tablet:mb-0"><li class="footer-subcategory"><span class="footer-subcategory-label block font-bold font-national text-[16px] leading-[20px]">Built-in Features</span><ul class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-1.5 tablet:mt-2"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/platform/alerts/">Alerts</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/platform/dashboards/">Dashboards</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://docs.datadoghq.com/notebooks/">Notebooks</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://docs.datadoghq.com/service_management/mobile/?tab=ios">Mobile App</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/fleet-automation/">Fleet Automation</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://docs.datadoghq.com/account_management/rbac/?tab=datadogapplication">Access Control</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/governance-console/">Governance Console</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://docs.datadoghq.com/account_management/teams/">Teams</a></li></ul></li><li class="footer-subcategory"><span class="footer-subcategory-label block font-bold font-national text-[16px] leading-[20px]">Extensibility</span><ul class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-1.5 tablet:mt-2"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/solutions/opentelemetry/">OpenTelemetry</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/platform/integrations/">Integrations</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/platform/ides/">IDE Plugins</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/ai/agent-directory/">Agent Directory</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://docs.datadoghq.com/api/">API</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/marketplacepartners/">Marketplace</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product/service-level-objectives/">Service Level Objectives</a></li></ul></li></ul></div></li></ul></li><li class="w-full [@media(min-width:1200px)]:w-1/3 [@media(min-width:1200px)]:px-3"><ul class="footer-link-sections flex flex-col list-none m-0 p-0 gap-y-4 tablet:grid tablet:grid-cols-3 tablet:gap-x-[30px] tablet:gap-y-0 [@media(min-width:1200px)]:grid-cols-2 [@media(min-width:1200px)]:content-start [@media(min-width:1200px)]:gap-y-9"><li class="footer-resources-section [@media(min-width:1200px)]:col-start-1 [@media(min-width:1200px)]:row-span-2" x-data="{ section: 'resources' }"><button type="button" @click="toggle(section)" :class="isOpen(section) ? 'open text-[#B37BFF] border-b-white' : 'text-white border-b-[#323232]'" :aria-expanded="isVisible(section)" aria-controls="footer-section-resources" :disabled="!mobile" class="group/section footer-section-header flex w-full items-center justify-between border-0 border-b border-solid bg-transparent p-0 pb-4 text-left font-bold font-national text-[18px] leading-[22px] cursor-pointer tablet:pb-1.5 tablet:mb-2 tablet:cursor-default tablet:border-b-white/50 tablet:block [@media(min-width:1200px)]:pb-2 [@media(min-width:1200px)]:mb-3">
|
||
Resources<svg class="block tablet:hidden w-[18px] h-[18px] fill-current transition-transform group-[.open]/section:rotate-180" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M96,144.313,32.343,80.657,43.657,69.343,96,121.687l52.343-52.344,11.314,11.314Z"></path>
|
||
</svg></button><ul id="footer-section-resources" x-show.important="isVisible(section)" class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-4 pb-5 border-b border-solid border-[#323232] tablet:mt-2 tablet:pb-0 tablet:border-b-0"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/pricing/">Pricing</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://docs.datadoghq.com/">Documentation</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/support/">Support</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/support-services/">Services & Enablement</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/product-preview/">Product Preview Program</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/certification/overview/">Certification</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://opensource.datadoghq.com/">Open Source</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/events-webinars/">Events and Webinars</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/security/">Security</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/privacy/">Privacy Center</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/knowledge-center/">Knowledge Center</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/learn/">Learning Resources</a></li></ul></li><li class="footer-blog-section [@media(min-width:1200px)]:col-start-2 [@media(min-width:1200px)]:row-start-1" x-data="{ section: 'blog' }"><button type="button" @click="toggle(section)" :class="isOpen(section) ? 'open text-[#B37BFF] border-b-white' : 'text-white border-b-[#323232]'" :aria-expanded="isVisible(section)" aria-controls="footer-section-blog" :disabled="!mobile" class="group/section footer-section-header flex w-full items-center justify-between border-0 border-b border-solid bg-transparent p-0 pb-4 text-left font-bold font-national text-[18px] leading-[22px] cursor-pointer tablet:pb-1.5 tablet:mb-2 tablet:cursor-default tablet:border-b-white/50 tablet:block [@media(min-width:1200px)]:pb-2 [@media(min-width:1200px)]:mb-3">
|
||
Blog<svg class="block tablet:hidden w-[18px] h-[18px] fill-current transition-transform group-[.open]/section:rotate-180" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M96,144.313,32.343,80.657,43.657,69.343,96,121.687l52.343-52.344,11.314,11.314Z"></path>
|
||
</svg></button><ul id="footer-section-blog" x-show.important="isVisible(section)" class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-4 pb-5 border-b border-solid border-[#323232] tablet:mt-2 tablet:pb-0 tablet:border-b-0"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/blog/">The Monitor</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/blog/engineering/">Engineering</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/blog/ai/">AI</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://securitylabs.datadoghq.com/">Security Labs</a></li></ul></li><li class="footer-about-section [@media(min-width:1200px)]:col-start-2 [@media(min-width:1200px)]:row-start-2" x-data="{ section: 'about' }"><button type="button" @click="toggle(section)" :class="isOpen(section) ? 'open text-[#B37BFF] border-b-white' : 'text-white border-b-[#323232]'" :aria-expanded="isVisible(section)" aria-controls="footer-section-about" :disabled="!mobile" class="group/section footer-section-header flex w-full items-center justify-between border-0 border-b border-solid bg-transparent p-0 pb-4 text-left font-bold font-national text-[18px] leading-[22px] cursor-pointer tablet:pb-1.5 tablet:mb-2 tablet:cursor-default tablet:border-b-white/50 tablet:block [@media(min-width:1200px)]:pb-2 [@media(min-width:1200px)]:mb-3">
|
||
About<svg class="block tablet:hidden w-[18px] h-[18px] fill-current transition-transform group-[.open]/section:rotate-180" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 192 192" width="192" height="192">
|
||
<path d="M96,144.313,32.343,80.657,43.657,69.343,96,121.687l52.343-52.344,11.314,11.314Z"></path>
|
||
</svg></button><ul id="footer-section-about" x-show.important="isVisible(section)" class="footer-menu-list flex flex-col list-none m-0 p-0 gap-y-1.5 mt-4 pb-5 border-b border-solid border-[#323232] tablet:mt-2 tablet:pb-0 tablet:border-b-0"><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/about/contact/">Contact Us</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/partner/network/">Partners</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/about/latest-news/press-releases/">Press</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/about/leadership/">Leadership</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://careers.datadoghq.com/">Careers</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/legal/">Legal</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://investors.datadoghq.com/">Investor Relations</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/about/analyst/">Analyst Reports</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/esg-report/">ESG Report</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/vendor-help/">Vendor Help</a></li><li class="footer-menu-item font-normal text-[#E2E2E2] text-[14px] leading-[18px]"><a class="text-inherit no-underline hover:text-[#969696] active:text-[#969696] focus-visible:text-[#969696] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[#B37BFF]" href="https://www.datadoghq.com/trust/">Trust Hub</a></li></ul></li></ul></li></ul></nav><div class="container"><div class="flex items-center tablet:justify-between"><div class="w-1/2"><a href="https://www.datadoghq.com/"><img class="logo max-w-[114px] tablet:max-w-[196px]" src="https://corp.dd-static.net/img/dd_logos/png_logos/horizontal/dd_logo_h_white.png?format=png" alt="Datadog logo"></a></div><div class="w-1/2 tablet:w-1/3"><p class="download-app mt-0 mb-0 pb-1 text-left font-semibold font-national text-white/50 text-[16px] leading-[16px] [@media(min-width:1200px)]:text-start">Download mobile app</p><div class="footer-app-links flex justify-start gap-5 [@media(min-width:1200px)]:items-end"><a href="https://apps.apple.com/app/datadog/id1391380318" aria-label="Apple Store Link"><svg class="block" width="26" height="31" viewBox="0 0 26 31" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M20.8866 16.3001C20.9017 15.1253 21.2147 13.9734 21.7962 12.9516C22.3778 11.9297 23.2091 11.0712 24.2129 10.4559C23.5752 9.54776 22.734 8.80043 21.7561 8.27322C20.7782 7.74602 19.6904 7.45346 18.5793 7.4188C16.209 7.1707 13.9112 8.83315 12.7033 8.83315C11.4721 8.83315 9.61244 7.44343 7.60989 7.48452C6.31458 7.52625 5.05221 7.90186 3.94575 8.57475C2.8393 9.24765 1.92651 10.1949 1.29632 11.3242C-1.43354 16.0372 0.602689 22.9639 3.21767 26.7736C4.52599 28.6391 6.05506 30.723 8.0558 30.6491C10.0136 30.5681 10.7449 29.4041 13.1082 29.4041C15.4495 29.4041 16.1355 30.6491 18.1769 30.6021C20.2778 30.5681 21.6014 28.7283 22.8638 26.8451C23.8039 25.5159 24.5272 24.0468 25.0071 22.4922C23.7865 21.9774 22.745 21.1158 22.0122 20.0146C21.2795 18.9135 20.888 17.6216 20.8866 16.3001Z" fill="white"></path>
|
||
<path d="M17.0298 4.91325C18.1753 3.542 18.7396 1.77948 18.603 0C16.8529 0.183293 15.2364 1.01735 14.0754 2.336C13.5078 2.98019 13.0731 3.72962 12.7961 4.54146C12.5191 5.3533 12.4052 6.21162 12.4611 7.06735C13.3364 7.07634 14.2024 6.88714 14.9938 6.51402C15.7851 6.14089 16.4813 5.59356 17.0298 4.91325Z" fill="white"></path>
|
||
</svg>
|
||
</a><a href="https://play.google.com/store/apps/details?id=com.datadog.app" aria-label="Google Play Store Link"><svg class="block" width="25" height="27" viewBox="0 0 25 27" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M17.8452 17.8161L13.4478 13.4082L0.463867 26.3921C0.758323 26.6542 1.13522 26.805 1.52921 26.8183C1.92319 26.8317 2.30942 26.7067 2.62091 26.4651L17.8452 17.8161Z" fill="url(#paint0_linear4)"></path>
|
||
<path d="M0.463776 0.424506C0.131609 0.835131 -0.0322666 1.35655 0.00527509 1.88337V24.9335C-0.0301817 25.4653 0.145841 25.9897 0.495038 26.3924L0.567982 26.4757L13.4477 13.5647V13.2521L0.53672 0.351562L0.463776 0.424506Z" fill="url(#paint1_linear3)"></path>
|
||
<path d="M17.7197 17.8682L13.4473 13.5645V13.2519L17.7197 8.94824L17.8135 9.01076L22.9299 11.9077C24.3888 12.7309 24.3888 14.0855 22.9299 14.9192L17.8343 17.8161L17.7197 17.8682Z" fill="url(#paint2_linear3)"></path>
|
||
<path d="M17.8452 9.01127L2.62091 0.362264C2.31145 0.117293 1.92552 -0.0107306 1.531 0.000704822C1.13648 0.0121402 0.758621 0.162302 0.463867 0.424787L13.4478 13.4087L17.8452 9.01127Z" fill="url(#paint3_linear3)"></path>
|
||
<defs>
|
||
<linearGradient id="paint0_linear4" x1="15.459" y1="15.8049" x2="-8.25809" y2="39.5115" gradientUnits="userSpaceOnUse">
|
||
<stop stop-color="#FF3A44"></stop>
|
||
<stop offset="1" stop-color="#C31162"></stop>
|
||
</linearGradient>
|
||
<linearGradient id="paint1_linear3" x1="12.3015" y1="1.6437" x2="-5.18413" y2="19.1293" gradientUnits="userSpaceOnUse">
|
||
<stop stop-color="#00A0FF"></stop>
|
||
<stop offset="0.01" stop-color="#00A1FF"></stop>
|
||
<stop offset="0.26" stop-color="#00BEFF"></stop>
|
||
<stop offset="0.51" stop-color="#00D2FF"></stop>
|
||
<stop offset="0.76" stop-color="#00DFFF"></stop>
|
||
<stop offset="1" stop-color="#00E3FF"></stop>
|
||
</linearGradient>
|
||
<linearGradient id="paint2_linear3" x1="24.8369" y1="13.4082" x2="-0.370305" y2="13.4082" gradientUnits="userSpaceOnUse">
|
||
<stop stop-color="#FFE000"></stop>
|
||
<stop offset="0.41" stop-color="#FFBD00"></stop>
|
||
<stop offset="0.78" stop-color="#FFA500"></stop>
|
||
<stop offset="1" stop-color="#FF9C00"></stop>
|
||
</linearGradient>
|
||
<linearGradient id="paint3_linear3" x1="-2.80817" y1="-7.2447" x2="7.77905" y2="3.34252" gradientUnits="userSpaceOnUse">
|
||
<stop stop-color="#32A071"></stop>
|
||
<stop offset="0.07" stop-color="#2DA771"></stop>
|
||
<stop offset="0.48" stop-color="#15CF74"></stop>
|
||
<stop offset="0.8" stop-color="#06E775"></stop>
|
||
<stop offset="1" stop-color="#00F076"></stop>
|
||
</linearGradient>
|
||
</defs>
|
||
</svg></a></div></div></div></div></div><div class="footer-legal container py-4"><div class="border-t border-solid border-[#323232] pt-9 [@media(min-width:1200px)]:pt-3"></div><div class="flex flex-col [@media(min-width:1200px)]:flex-row [@media(min-width:1200px)]:items-center"><div class="footer-legal-menu order-2 mb-7 flex flex-wrap items-center justify-center tablet:flex-nowrap [@media(min-width:1200px)]:order-1 [@media(min-width:1200px)]:mb-0 [@media(min-width:1200px)]:w-2/3 [@media(min-width:1200px)]:justify-start"><span class="copyright w-full text-center text-[14px] leading-[18px] mr-2 tablet:w-auto tablet:text-start [@media(min-width:992px)]:text-[16px] [@media(min-width:992px)]:leading-[20px] [@media(min-width:992px)]:mr-3">© Datadog 2026
|
||
</span><span class="flex items-center"><a class="py-1.5 text-white text-[16px] leading-[16px] no-underline whitespace-normal hover:opacity-50 [@media(min-width:992px)]:py-1" href="https://www.datadoghq.com/legal/terms/">Terms</a>
|
||
<span> | </span>
|
||
</span><span class="flex items-center"><a class="py-1.5 text-white text-[16px] leading-[16px] no-underline whitespace-normal hover:opacity-50 [@media(min-width:992px)]:py-1" href="https://www.datadoghq.com/legal/privacy/">Privacy</a>
|
||
<span> | </span>
|
||
</span><span class="flex items-center"><a class="py-1.5 text-white text-[16px] leading-[16px] no-underline whitespace-normal hover:opacity-50 [@media(min-width:992px)]:py-1" href="https://www.datadoghq.com/legal/cookies/">Cookies</a></span></div><div class="order-3 flex justify-center [@media(min-width:1200px)]:order-2 [@media(min-width:1200px)]:w-1/6 [@media(min-width:1200px)]:justify-end"><div x-data="{ popup_open: false}" class="footer-menu relative flex items-center h-full justify-center desktop-sm:justify-end cursor-pointer w-auto min-w-[120px] max-w-[166px] desktop-sm:w-[166px]"><button type="button" @click="popup_open = !popup_open" @keydown.escape="popup_open = false" :aria-expanded="popup_open" aria-haspopup="listbox" class="flex items-center text-white text-[13px] leading-[18px] [&:lang(ja)]:text-[11px] [&:lang(ko)]:text-[11px] tablet:text-[15px] tablet:leading-[20px] tablet:[&:lang(ja)]:text-[13px] tablet:[&:lang(ko)]:text-[13px] desktop-sm:text-[17px] desktop-sm:leading-[22px] desktop-sm:[&:lang(ja)]:text-[14px] desktop-sm:[&:lang(ko)]:text-[14px] rounded-sm tablet:rounded w-full h-[32px] border border-[#616161] justify-between px-2 py-0.5 hover:bg-[#323232]">
|
||
<span class="flex items-center"><span class="flex items-center mr-2"><svg class="<nil>" width="16px" height="16px" viewBox="0 0 48 48" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||
<!-- Generator: Sketch 52.6 (67491) - http://www.bohemiancoding.com/sketch -->
|
||
<title>Icon/world</title>
|
||
<desc>Created with Sketch.</desc>
|
||
<g stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
|
||
<path d="M23,18.7142857 L15.1972484,18.7142857 C14.8808707,20.3582799 14.7142857,22.1185995 14.7142857,24 C14.7142857,26.101197 14.9220672,28.0513698 15.3142384,29.8571429 L23,29.8571429 L23,18.7142857 Z M25,18.7142857 L25,29.8571429 L32.6856474,29.8571429 C33.0778185,28.0513698 33.2856,26.101197 33.2856,24 C33.2856,22.1185995 33.119015,20.3582799 32.8026373,18.7142857 L25,18.7142857 Z M23,31.8571429 L15.8410328,31.8571429 C16.6874013,34.5709824 17.9826731,36.9218861 19.6311425,38.9369568 C20.7014808,40.2453264 21.8466701,41.314437 22.9882288,42.1603531 C22.9921555,42.1632629 22.9960793,42.1661678 23,42.1690678 L23,31.8571429 Z M25,31.8571429 L25,42.1689833 C25.0038827,42.1661114 25.0077683,42.1632346 25.0116569,42.1603531 C26.1532156,41.314437 27.298405,40.2453264 28.3687432,38.9369568 C30.0172126,36.9218861 31.3124844,34.5709824 32.1588529,31.8571429 L25,31.8571429 Z M23,16.7142857 L23,5.83093215 C22.9960793,5.83383218 22.9921555,5.83673708 22.9882288,5.83964686 C21.8466701,6.68556302 20.7014808,7.75467359 19.6311425,9.06304321 C17.8680399,11.2182402 16.508965,13.7576013 15.67101,16.7142857 L23,16.7142857 Z M25,16.7142857 L32.3288757,16.7142857 C31.4909207,13.7576013 30.1318458,11.2182402 28.3687432,9.06304321 C27.298405,7.75467359 26.1532156,6.68556302 25.0116569,5.83964686 C25.0077683,5.83676535 25.0038827,5.83388863 25,5.83101669 L25,16.7142857 Z M13.1635581,18.7142857 L5.74490713,18.7142857 C5.25990904,20.3922708 5,22.1657822 5,24 C5,26.043811 5.32270158,28.0122495 5.91993233,29.8571429 L13.2714898,29.8571429 C12.906587,28.0336164 12.7142857,26.0823166 12.7142857,24 C12.7142857,22.1336211 12.8687714,20.3724948 13.1635581,18.7142857 Z M20.4443338,42.6678567 C19.6363334,41.950688 18.8419739,41.130915 18.0831432,40.2033289 C16.1536635,37.8447543 14.6685687,35.0696382 13.7544275,31.8571429 L6.69559884,31.8571429 C9.21247658,37.3914264 14.3062352,41.5058446 20.4443338,42.6678567 Z M20.4443338,5.33214327 C14.0967882,6.53380609 8.86606956,10.8927992 6.44701478,16.7142857 L13.5990681,16.7142857 C14.4953973,13.2582719 16.04017,10.2939789 18.0831432,7.79667107 C18.8419739,6.86908497 19.6363334,6.04931197 20.4443338,5.33214327 Z M34.8363276,18.7142857 C35.1311143,20.3724948 35.2856,22.1336211 35.2856,24 C35.2856,26.0823166 35.0932987,28.0336164 34.7283959,29.8571429 L42.0800677,29.8571429 C42.6772984,28.0122495 43,26.043811 43,24 C43,22.1657822 42.740091,20.3922708 42.2550929,18.7142857 L34.8363276,18.7142857 Z M27.5555209,42.6678842 C33.6936845,41.5059095 38.7875036,37.39147 41.3044012,31.8571429 L34.2454582,31.8571429 C33.3313171,35.0696382 31.8462222,37.8447543 29.9167425,40.2033289 C29.1579022,41.1309269 28.363532,41.950709 27.5555209,42.6678842 Z M27.5555209,5.33211577 C28.363532,6.04929101 29.1579022,6.86907311 29.9167425,7.79667107 C31.9597158,10.2939789 33.5044884,13.2582719 34.4008176,16.7142857 L41.5529852,16.7142857 C39.133912,10.8927548 33.9031319,6.53373958 27.5555209,5.33211577 Z M45,24 C45,35.597999 35.597999,45 24,45 C12.402001,45 3,35.597999 3,24 C3,12.402001 12.402001,3 24,3 C35.597999,3 45,12.402001 45,24 Z" fill="#ffffff" fill-rule="nonzero"></path>
|
||
</g>
|
||
</svg>
|
||
</span><span class="flex items-center mb-0.5 [&:lang(ja)]:break-keep [&:lang(ko)]:break-keep">English
|
||
</span></span><span class="flex items-center transition-transform duration-200" :class="popup_open ? 'rotate-180' : ''"><svg class="<nil>" width="10" height="10" viewBox="0 0 10 10" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M0.999547 5.41459L5.24219 1.17195L9.48483 5.41459" stroke="white"></path>
|
||
</svg></span></button><div x-show="popup_open" x-cloak="" @click.outside="popup_open = false" class="absolute bottom-[calc(100%_+_8px)] desktop-sm:bottom-[calc(100%_+_4px)] left-0 w-full bg-[#323232] flex flex-col gap-0.5 p-0.5 tablet:gap-1 tablet:p-1 rounded-sm tablet:rounded-md desktop-sm:rounded-lg z-10"><a class="flex flex-col w-full px-2 py-0.5 tablet:px-3 tablet:py-1 desktop-sm:px-4 desktop-sm:py-1.5 text-white no-underline bg-[#323232] rounded-sm tablet:rounded hover:bg-[#616161] transition-colors" href="https://www.datadoghq.com/?lang_pref=en"><div class="text-[13px] leading-[18px] [&:lang(ja)]:text-[11px] [&:lang(ko)]:text-[11px] tablet:text-[15px] tablet:leading-[20px] tablet:[&:lang(ja)]:text-[13px] tablet:[&:lang(ko)]:text-[13px] desktop-sm:text-[17px] desktop-sm:leading-[22px] desktop-sm:[&:lang(ja)]:text-[14px] desktop-sm:[&:lang(ko)]:text-[14px] font-normal mb-1">English</div><div class="text-[12px] leading-[14px] [&:lang(ja)]:text-[9px] [&:lang(ko)]:text-[9px] tablet:text-[13px] tablet:leading-[16px] tablet:[&:lang(ja)]:text-[10px] tablet:[&:lang(ko)]:text-[10px] desktop-sm:text-[14px] desktop-sm:leading-[18px] desktop-sm:[&:lang(ja)]:text-[11px] desktop-sm:[&:lang(ko)]:text-[11px] text-[#c7c7c7] font-normal">English</div></a><a class="flex flex-col w-full px-2 py-0.5 tablet:px-3 tablet:py-1 desktop-sm:px-4 desktop-sm:py-1.5 text-white no-underline bg-[#323232] rounded-sm tablet:rounded hover:bg-[#616161] transition-colors" href="https://www.datadoghq.com/ja/?lang_pref=ja"><div class="text-[13px] leading-[18px] [&:lang(ja)]:text-[11px] [&:lang(ko)]:text-[11px] tablet:text-[15px] tablet:leading-[20px] tablet:[&:lang(ja)]:text-[13px] tablet:[&:lang(ko)]:text-[13px] desktop-sm:text-[17px] desktop-sm:leading-[22px] desktop-sm:[&:lang(ja)]:text-[14px] desktop-sm:[&:lang(ko)]:text-[14px] font-normal mb-1">日本語</div><div class="text-[12px] leading-[14px] [&:lang(ja)]:text-[9px] [&:lang(ko)]:text-[9px] tablet:text-[13px] tablet:leading-[16px] tablet:[&:lang(ja)]:text-[10px] tablet:[&:lang(ko)]:text-[10px] desktop-sm:text-[14px] desktop-sm:leading-[18px] desktop-sm:[&:lang(ja)]:text-[11px] desktop-sm:[&:lang(ko)]:text-[11px] text-[#c7c7c7] font-normal">Japanese</div></a><a class="flex flex-col w-full px-2 py-0.5 tablet:px-3 tablet:py-1 desktop-sm:px-4 desktop-sm:py-1.5 text-white no-underline bg-[#323232] rounded-sm tablet:rounded hover:bg-[#616161] transition-colors" href="https://www.datadoghq.com/ko/?lang_pref=ko"><div class="text-[13px] leading-[18px] [&:lang(ja)]:text-[11px] [&:lang(ko)]:text-[11px] tablet:text-[15px] tablet:leading-[20px] tablet:[&:lang(ja)]:text-[13px] tablet:[&:lang(ko)]:text-[13px] desktop-sm:text-[17px] desktop-sm:leading-[22px] desktop-sm:[&:lang(ja)]:text-[14px] desktop-sm:[&:lang(ko)]:text-[14px] font-normal mb-1">한국어</div><div class="text-[12px] leading-[14px] [&:lang(ja)]:text-[9px] [&:lang(ko)]:text-[9px] tablet:text-[13px] tablet:leading-[16px] tablet:[&:lang(ja)]:text-[10px] tablet:[&:lang(ko)]:text-[10px] desktop-sm:text-[14px] desktop-sm:leading-[18px] desktop-sm:[&:lang(ja)]:text-[11px] desktop-sm:[&:lang(ko)]:text-[11px] text-[#c7c7c7] font-normal">Korean</div></a></div></div></div><div class="order-1 mb-7 [@media(min-width:1200px)]:order-3 [@media(min-width:1200px)]:mb-0 [@media(min-width:1200px)]:w-1/6"><div class="footer-legal-menu flex h-full items-center justify-center [@media(min-width:1200px)]:justify-end"><span class="flex items-center mr-7 [@media(min-width:1200px)]:mr-2"><a class="social-link flex items-center text-white" href="https://twitter.com/datadoghq" aria-label="Twitter link"><svg class="w-[17px] h-auto fill-current" width="1200" height="1227" viewBox="0 0 1200 1227" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M714.163 519.284L1160.89 0H1055.03L667.137 450.887L357.328 0H0L468.492 681.821L0 1226.37H105.866L515.491 750.218L842.672 1226.37H1200L714.137 519.284H714.163ZM569.165 687.828L521.697 619.934L144.011 79.6944H306.615L611.412 515.685L658.88 583.579L1055.08 1150.3H892.476L569.165 687.854V687.828Z" fill="white"></path>
|
||
</svg>
|
||
</a></span><span class="flex items-center mr-7 [@media(min-width:1200px)]:mr-2"><a class="social-link flex items-center text-white" href="https://www.instagram.com/datadoghq/" aria-label="Instagram link"><svg class="w-[17px] h-auto fill-current" version="1.1" xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 32 32">
|
||
<title>instagram</title>
|
||
<path d="M15.966 2.952c4.226 0 4.763 0 6.44 0.067 1.006 0 2.013 0.201 2.952 0.537 1.409 0.537 2.482 1.61 3.019 3.019 0.335 0.939 0.537 1.946 0.537 2.952 0.067 1.677 0.067 2.214 0.067 6.44s0 4.763-0.067 6.44c0 1.006-0.201 2.013-0.537 2.952-0.537 1.409-1.61 2.482-3.019 3.019-0.939 0.335-1.946 0.537-2.952 0.537-1.677 0.067-2.214 0.067-6.44 0.067s-4.763 0-6.44-0.067c-1.006 0-2.013-0.201-2.952-0.537-1.409-0.537-2.482-1.677-3.019-3.019-0.335-0.939-0.537-1.945-0.537-2.952-0.067-1.677-0.067-2.214-0.067-6.44s0-4.763 0.067-6.44c0-1.006 0.201-2.013 0.537-2.952 0.537-1.342 1.61-2.482 3.019-3.019 0.939-0.335 1.946-0.537 2.952-0.537 1.677 0 2.147-0.067 6.44-0.067zM15.966 0.067c-4.361 0-4.897 0-6.574 0.067-1.342 0-2.616 0.268-3.891 0.738-2.147 0.805-3.891 2.549-4.696 4.696-0.47 1.275-0.738 2.549-0.738 3.891-0.067 1.677-0.067 2.281-0.067 6.574s0 4.897 0.067 6.574c0.067 1.342 0.335 2.616 0.805 3.824 0.805 2.147 2.549 3.891 4.696 4.696 1.275 0.47 2.549 0.738 3.891 0.738 1.677 0.067 2.214 0.067 6.574 0.067s4.897 0 6.574-0.067c1.342 0 2.616-0.268 3.891-0.738 2.147-0.805 3.891-2.549 4.696-4.696 0.47-1.275 0.738-2.549 0.738-3.891 0.067-1.677 0.067-2.281 0.067-6.574s0-4.897-0.067-6.574c0-1.342-0.268-2.616-0.738-3.891-0.805-2.147-2.549-3.891-4.696-4.696-1.275-0.47-2.549-0.738-3.891-0.738-1.677 0-2.214 0-6.642 0zM15.968 7.85c-4.495 0-8.184 3.69-8.184 8.184s3.69 8.184 8.184 8.184c4.495 0 8.184-3.69 8.184-8.184s-3.69-8.184-8.184-8.184zM15.968 21.335c-2.952 0-5.3-2.415-5.3-5.3 0-2.952 2.415-5.3 5.3-5.3s5.3 2.415 5.3 5.3c0 2.952-2.348 5.3-5.3 5.3zM24.484 9.46c1.075 0 1.946-0.871 1.946-1.946s-0.871-1.946-1.946-1.946c-1.074 0-1.945 0.871-1.945 1.946s0.871 1.946 1.945 1.946z"></path>
|
||
</svg>
|
||
</a></span><span class="flex items-center mr-7 [@media(min-width:1200px)]:mr-2"><a class="social-link flex items-center text-white" href="https://www.youtube.com/user/DatadogHQ" aria-label="Youtube link"><svg class="w-[17px] h-auto fill-current" width="42" height="42" viewBox="0 0 42 42" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M35.3752 13.7796C35.0295 12.488 34.0145 11.473 32.7229 11.1273C30.3818 10.5 21 10.5 21 10.5C21 10.5 11.6182 10.5 9.27955 11.1273C7.98798 11.473 6.973 12.488 6.62726 13.7796C6 16.1182 6 21.0005 6 21.0005C6 21.0005 6 25.8828 6.62726 28.2214C6.973 29.513 7.98798 30.528 9.27955 30.8737C11.6182 31.501 21 31.501 21 31.501C21 31.501 30.3818 31.501 32.7204 30.8737C34.012 30.528 35.027 29.513 35.3727 28.2214C36 25.8828 36 21.0005 36 21.0005C36 21.0005 36 16.1182 35.3727 13.7796H35.3752ZM17.9995 25.5V16.501L25.7934 21.0005L17.9995 25.5Z" fill="white"></path>
|
||
</svg>
|
||
</a></span><span class="flex items-center"><a class="social-link flex items-center text-white" href="https://www.linkedin.com/company/datadog" aria-label="LinkedIn link"><svg class="w-[17px] h-auto fill-current" width="32" height="32" viewBox="0 0 30 30" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||
<path d="M27.8504 0H2.14956C0.963597 0 0 0.963597 0 2.14956V27.8504C0 29.0364 0.963597 30 2.14956 30H27.8504C29.0364 30 30 29.0364 30 27.8504V2.14956C30 0.963597 29.0364 0 27.8504 0ZM8.7465 25.7898H4.29419C4.29419 25.7898 4.27936 25.7799 4.27936 25.775V11.3112C4.27936 11.3112 4.28924 11.2963 4.29419 11.2963H8.75144C8.75144 11.2963 8.76627 11.3062 8.76627 11.3112V25.775C8.76627 25.775 8.75638 25.7898 8.75144 25.7898H8.7465ZM6.49811 9.39878C5.03047 9.39878 3.84451 8.20293 3.84451 6.72542C3.84451 5.2479 5.03047 4.05205 6.49811 4.05205C7.96574 4.05205 9.15171 5.2479 9.15171 6.72542C9.15171 8.20293 7.96574 9.39878 6.49811 9.39878V9.39878ZM25.5675 25.7898H21.1349C21.1349 25.7898 21.1201 25.7799 21.1201 25.775V18.1848C21.1201 16.0995 20.3294 14.9333 18.674 14.9333C16.8753 14.9333 15.9414 16.1489 15.9414 18.1848V25.775C15.9414 25.775 15.9315 25.7898 15.9265 25.7898H11.6571C11.6571 25.7898 11.6422 25.7799 11.6422 25.775V11.3112C11.6422 11.3112 11.6521 11.2963 11.6571 11.2963H15.9265C15.9265 11.2963 15.9414 11.3062 15.9414 11.3112V13.184C15.9414 13.184 15.9611 13.2087 15.971 13.1939C16.1835 12.843 17.5078 10.8516 20.3097 10.8516C23.3833 10.8516 25.5872 12.7294 25.5872 16.6134V25.775C25.5872 25.775 25.5773 25.7898 25.5724 25.7898H25.5675Z" fill="white"></path>
|
||
</svg></a></span></div></div></div></div></footer></body></html></div> <datadog-logs-rum data-is-prod="true" data-dd-client-token="pub0b466265cd4de08394d4e1979fb79787" data-dd-application-id="c0bd0f01-a704-40c6-a9ef-b130b5bf397e" data-env="production" data-service="websites-blog"></datadog-logs-rum> <script type="module" src="/blog/_astro/DatadogLogsRum.astro_astro_type_script_index_0_lang.CrAKTPh3.js"></script> <script type="module" src="/blog/_astro/Base.astro_astro_type_script_index_0_lang.TdQLbLHm.js"></script> <script type="module" src="/blog/_astro/Base.astro_astro_type_script_index_1_lang.DABIRTca.js"></script> <script src="/assets/ane-popup-banner.js"></script> <script>
|
||
document.addEventListener('alpine:init', () => {
|
||
Alpine.data('searchbar', () => ({
|
||
search(e) {
|
||
let searchbarValue = '';
|
||
if (e.key === 'Enter') {
|
||
searchbarValue = e.target.value;
|
||
} else {
|
||
searchbarValue = (e.target.nextElementSibling || e.target.previousElementSibling)
|
||
?.value;
|
||
}
|
||
const pageLangPrefix =
|
||
document.documentElement.lang === 'en' ? '' : '/' + document.documentElement.lang;
|
||
const pathName = window.location.pathname;
|
||
let blogType = 'The-Monitor';
|
||
if (pathName.includes('engineering')) blogType = 'Engineering';
|
||
else if (pathName.includes('pup-culture')) blogType = 'Pup-Culture';
|
||
else if (pathName.includes('ai')) blogType = 'AI';
|
||
const blogTypeParam = pageLangPrefix === '' ? '?blog-0=' + blogType : '';
|
||
const searchParam =
|
||
searchbarValue ? (blogTypeParam === '' ? '?' : '&') + 's=' + searchbarValue : '';
|
||
window.location.href = pageLangPrefix + '/blog/search/' + blogTypeParam + searchParam;
|
||
},
|
||
}));
|
||
});
|
||
</script> </body></html><script type="module">const k=()=>{const t=document.querySelector(".aside-toc"),e=document.querySelector(".article-feedback-card"),o=document.querySelector(".further-reading-card");if(!e)return;const s=o?.querySelector("img"),u=()=>{t&&o&&(t.style.marginBottom=`${o.offsetHeight+42}px`)};t&&setTimeout(u,100),s&&(s.complete?u():s.addEventListener("load",u));const f=()=>{window.scrollY>=2e3&&e.classList.remove("opacity-0"),window.scrollY<2e3&&e.classList.add("opacity-0")},c=(p,i,w)=>{const g=e.querySelector(".article-feedback-button-container");if(!g)return;g.querySelectorAll("button").forEach(y=>{y.addEventListener("click",()=>{const L={pageURL:p,feedback:y.dataset.ddActionName},S=w===null?[]:JSON.parse(w);S.push(L),i.setItem("datadogBlogFeedback",JSON.stringify(S)),setTimeout(()=>{e.classList.add("opacity-0","transition-opacity","duration-500","ease-out"),e.addEventListener("transitionend",()=>{e.remove()},{once:!0})},2500)})})},a=window.location.pathname,b=window.localStorage,l=b.getItem("datadogBlogFeedback");let m=!1;if(l&&(m=!!JSON.parse(l).filter(p=>p.pageURL===a).length,m)){e?.remove();return}c(a,b,l),window.addEventListener("scroll",f,{passive:!0})};function O(){const n=document.querySelectorAll(".table-component"),t=e=>{e.scrollWidth>e.clientWidth?e.classList.add("has-scrollbar"):e.classList.remove("has-scrollbar")};n.forEach(e=>{const o=e.querySelector(".table-component-box .table-component-box-inner");if(!o)return;t(o),new ResizeObserver(()=>{t(o)}).observe(o)})}let d=!1,r=null,v=null;function h(){d=!0,r&&(r.disconnect(),r=null),clearTimeout(v),v=setTimeout(()=>{d=!1},3e3)}window.addEventListener("wheel",h,{passive:!0});window.addEventListener("touchstart",h,{passive:!0});window.addEventListener("keydown",n=>{["ArrowUp","ArrowDown","PageUp","PageDown","Home","End"," "].includes(n.key)&&h()},{passive:!0});function E(){const n=document.querySelector("[data-dd-secondary-nav]");if(!n)return 0;const t=getComputedStyle(n);if(t.display==="none"||t.visibility==="hidden")return 0;const e=n.getBoundingClientRect();return e.height<1?0:Math.ceil(e.height)}function C(n){if(typeof n=="number"&&!Number.isNaN(n))return n;const t=typeof window<"u"?window.__FIX_HASH_SCROLL_TOP_OFFSET__:void 0;if(typeof t=="number"&&!Number.isNaN(t))return t;let e=100;const o=document.querySelector("main");if(o){const s=parseFloat(getComputedStyle(o).marginTop);Number.isFinite(s)&&s>=0&&(e=s+4)}return e+E()}function T(n={}){const{maxDuration:t=1e4,stableThreshold:e=3,topOffset:o}=n,s=location.hash;if(!s||d)return;const u=decodeURIComponent(s),f=document.querySelector(u);if(!f)return;r&&r.disconnect();let c=null,a=0;const b=performance.now(),l=()=>{if(d)return;const m=C(o),i=f.getBoundingClientRect().top+window.scrollY-m;window.scrollTo({top:i,behavior:"instant"})};r=new ResizeObserver(()=>{if(d){r.disconnect(),r=null;return}if(performance.now()-b>t){r.disconnect(),r=null;return}const i=f.getBoundingClientRect().top+window.scrollY;if(c===null){c=i,l();return}Math.abs(i-c)<1?(a++,a>=e&&(r.disconnect(),r=null)):(c=i,a=0,l())}),r.observe(document.documentElement)}location.hash&&setTimeout(()=>{T()},100);window.addEventListener("hashchange",()=>{d=!1,location.hash&&setTimeout(()=>{T()},100)});k();O();</script> |