Files
nexus/sreweekly/articles/441/01-how-we-migrated-from-statsd-to-prometheus-in-one-month.html
2026-09-12 17:23:01 +08:00

68 lines
199 KiB
HTML
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><meta name="viewport" content="width=device-width,minimum-scale=1,initial-scale=1,maximum-scale=1"/><style data-precedence="high" data-href="lite-color-scheme-tokens">:root{color-scheme:light;--invert-in-dark-mode: 0;--blend-in-dark-mode: normal;--color-bg-neutral-primary: #ffffff;--color-bg-neutral-secondary: #f2f2f2;--color-bg-neutral-tertiary: #f9f9f9;--color-bg-neutral-quaternary: #242424;--color-bg-brand-primary: #191919;--color-bg-brand-primary-hover: #080808;--color-bg-accent-secondary: #bbdbba;--color-bg-accent-secondary-hover: #9fcc9e;--color-bg-accent-tertiary: #d2e7d1;--color-bg-accent-quaternary: #e8f3e8;--color-bg-error-primary: #c94a4a;--color-bg-error-primary-hover: #b63636;--color-bg-utility-yellow-primary: #fffae1;--color-bg-utility-blue-primary: #e5f2ff;--color-bg-verified-book-author-intro: #e5f2ff;--color-border-verified-book-author-intro: transparent;--verified-book-author-intro-illustration-scale: 1;--color-fg-neutral-primary: #242424;--color-fg-neutral-primary-hover: #000000;--color-fg-neutral-secondary: #6b6b6b;--color-fg-neutral-secondary-hover: #3f3f3f;--color-fg-neutral-tertiary: #ffffff;--color-fg-accent-primary: #1a8917;--color-fg-accent-primary-hover: #156d12;--color-fg-error-primary: #c94a4a;--color-fg-error-primary-hover: #b63636;--color-fg-utility-blue-primary: #437aff;--color-border-neutral-primary: #f2f2f2;--color-border-neutral-primary-hover: #e5e5e5;--color-border-neutral-secondary: #242424;--color-border-neutral-secondary-hover: #000000;--color-border-neutral-tertiary: #e5e5e5;--color-border-neutral-tertiary-hover: #d2d2d2;--color-border-brand-primary: #191919;--color-border-accent-primary: #1a8917;--color-border-accent-primary-hover: #156d12;--color-border-error-primary: #c94a4a;--color-border-error-primary-hover: #b63636;--color-chart-accent-primary: #1a8917;--color-chart-accent-secondary: #9fcc9e;--color-chart-accent-tertiary: #bbdbba;--color-chart-accent-quaternary: #e8f3e8;--color-chart-utility-yellow-primary: #fea010;--color-chart-utility-yellow-secondary: #ffc017;--color-chart-utility-yellow-tertiary: #fffae1;--color-chart-neutral-primary: #b3b3b3;--color-chart-neutral-secondary: #b3b3b3;--color-chart-neutral-tertiary: #e5e5e5;--color-status-in-review-background: #e5f2ff;--color-status-in-review-text: #437aff;--color-status-edits-requested-background: #fffae1;--color-status-edits-requested-text: #be5b04;--color-story-footer-bg-subtle: #f7f4ed;--color-story-footer-bg-contrast: #242424;--color-story-footer-fg-subtle: #242424;--color-story-footer-fg-contrast: #ffffff;--color-story-footer-placeholder-light-grey: #d2d2d2;--color-story-footer-placeholder-dark-grey: #6b6b6b}</style><script src="https://www.google.com/recaptcha/enterprise.js?render=6Le-uGgpAAAAAPprRaokM8AKthQ9KNGdoxaGUvVp" async=""></script><link rel="icon" href="https://miro.medium.com/v2/5d8de952517e8160e40ef9841c781cdc14a5db313057fa3c3de41c6f5b494b19"/><title>Medium</title><meta name="theme-color" content="#000000"/><meta name="twitter:app:name:iphone" content="Medium"/><meta name="twitter:app:id:iphone" content="828256236"/><meta property="al:ios:app_name" content="Medium"/><meta property="al:ios:app_store_id" content="828256236"/><meta property="al:android:package" content="com.medium.reader"/><meta property="al:android:app_name" content="Medium"/><meta property="fb:app_id" content="542599432471018"/><meta property="og:site_name" content="Medium"/><meta name="apple-itunes-app" content="app-id=828256236, app-argument=/mixpaneleng/how-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5, affiliate-data=pt=698524&amp;ct=smart_app_banner&amp;mt=8"/><link rel="search" type="application/opensearchdescription+xml" title="Medium" href="/osd.xml"/><link rel="apple-touch-icon" sizes="152x152" href="https://miro.medium.com/v2/resize:fill:304:304/10fd5c419ac61637245384e7099e131627900034828f4f386bdaa47a74eae156"/><link rel="apple-touch-icon" sizes="120x120" href="https://miro.medium.com/v2/resize:fill:240:240/10fd5c419ac61637245384e7099e131627900034828f4f386bdaa47a74eae156"/><link rel="apple-touch-icon" sizes="76x76" href="https://miro.medium.com/v2/resize:fill:152:152/10fd5c419ac61637245384e7099e131627900034828f4f386bdaa47a74eae156"/><link rel="apple-touch-icon" sizes="60x60" href="https://miro.medium.com/v2/resize:fill:120:120/10fd5c419ac61637245384e7099e131627900034828f4f386bdaa47a74eae156"/><link rel="mask-icon" href="https://miro.medium.com/v2/resize:fill:1000:1000/7*GAOKVe--MXbEJmV9230oOQ.png" color="#171717"/><link rel="preconnect" href="https://glyph.medium.com" crossorigin=""/><link rel="manifest" href="/manifest.json"/><link rel="preconnect" href="https://www.google.com"/><link rel="preconnect" href="https://www.gstatic.com" crossorigin=""/><link id="glyph_preload_link" rel="preload" as="style" type="text/css" href="https://glyph.medium.com/css/unbound.css"/><title>How We Migrated from StatsD to Prometheus in One Month | by Eddie Bracho | Mixpanel Engineering | Medium</title><meta property="og:type" content="article"/><meta property="article:published_time" content="2025-05-14T20:11:25.049Z"/><meta name="title" content="How We Migrated from StatsD to Prometheus in One Month | by Eddie Bracho | Mixpanel Engineering | Medium"/><meta property="og:title" content="How We Migrated from StatsD to Prometheus in One Month"/><meta property="al:android:url" content="medium://p/fb973af124f5"/><meta property="al:ios:url" content="medium://p/fb973af124f5"/><meta property="al:android:app_name" content="Medium"/><meta name="description" content="How We Migrated from StatsD to Prometheus in One Month We recently migrated all of our infrastructure metrics from StatsD to Prometheus and are very pleased with the results. The migration was a ton …"/><meta property="og:description" content="How We Migrated from StatsD to Prometheus in One Month We recently migrated all of our infrastructure metrics from StatsD to Prometheus and are very pleased with the results. The migration was a ton …"/><meta property="og:url" content="https://medium.com/mixpaneleng/how-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5"/><meta property="al:web:url" content="https://medium.com/mixpaneleng/how-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5"/><meta property="og:image" content="https://miro.medium.com/v2/resize:fit:653/1*uDXTt2XtbxzDnaGG3SU2fQ.png"/><meta property="article:author" content="https://medium.com/@eddie.bracho"/><meta name="author" content="Eddie Bracho"/><meta name="robots" content="index,follow,max-image-preview:large"/><meta name="referrer" content="unsafe-url"/><meta property="twitter:title" content="How We Migrated from StatsD to Prometheus in One Month"/><meta name="twitter:site" content="@mixpanel"/><meta name="twitter:app:url:iphone" content="medium://p/fb973af124f5"/><meta property="twitter:description" content="How We Migrated from StatsD to Prometheus in One Month We recently migrated all of our infrastructure metrics from StatsD to Prometheus and are very pleased with the results. The migration was a ton …"/><meta name="twitter:image:src" content="https://miro.medium.com/v2/resize:fit:653/1*uDXTt2XtbxzDnaGG3SU2fQ.png"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:label1" content="Reading time"/><meta name="twitter:data1" content="8 min read"/><link rel="author" href="https://medium.com/@eddie.bracho"/><link rel="canonical" href="https://medium.com/mixpaneleng/how-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5"/><link rel="alternate" href="android-app://com.medium.reader/https/medium.com/p/fb973af124f5"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/static/manifest-3cf8583e.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/entry-C9P2t20qJW.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-D0WnhmqRYc.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-C11jW0FiNe.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CrjQH3m-gW.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-BZidLkaas_.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/entry-CwxkgI60OV.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DXScYH4iYt.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DpwhkAmq4W.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/entry-fBNTF2ig9x.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CXnHTW_oeV.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DnPkHBw3I_.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CYdEVPzgBO.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CQtFW-cnCb.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-Ccx4bzDBz8.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CD5QG4Nr4u.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DdonUN-_p_.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-C0nqQaN4zj.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-Dyqv3_ywby.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DlxwlCRnKY.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-x3yMr9Hj2M.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-D6hKjcUA3T.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-6-s_y3aqFJ.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-BQYOJ-mO15.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DP3_7SVASi.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CjgOIT_e7P.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CzYDeZxwQf.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-BHu-NPXwPu.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-HvNxSL9udh.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-ByFaBDnwQT.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk---sLyFDcIU.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-Cwg9icNWml.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-I8cQABJjKJ.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-EPww8yA76c.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-BW6VYw0uzE.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CZabPPB8tI.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CPynNJqWgE.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-6BSpMonAeB.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DuJ3wvjuqM.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-BDd81HQSRF.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CH0gmOPEMW.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DxkCkufq5c.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CUCbM7l4Dj.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-cNcrfj78jd.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DTqgPoTD1a.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-Bne5AQ4h-5.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CGhflpm2Hm.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CNPxQmRnXf.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-BX3UAvcF5y.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CRPL1qQoWW.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-BfU8uMdn42.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-Bn3GEJ9G7e.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-4m7R-C0Uxl.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CeP5820RVI.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-Q5o9bjyr7s.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-E3gD0vz4qE.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-C839jH4HYq.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-Bn4kth2VY1.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-Cgs_EYwel6.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-zgIIvDkvAP.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-Cl_V_h5b8P.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-wS9mSISbp0.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-BJsxkluEP9.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CvnM9G2Pye.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-BSxQZl1sLA.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-KzP5Y5LLQP.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DfTA5uCdpU.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CmwWkCGGjS.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DTGFyXOJOa.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DuxEVP_C5g.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CBXapKR9BK.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/entry-DJjt4aY773.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-qI8VCxaJvk.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DCC7-BJDuV.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-BWWdZr0USU.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-Ba_Uf5H8ZL.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DSy2ZEdAjr.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CbMw0xIIxH.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-B4miwgxpFB.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-BJ72pzEf0s.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-D9y1tiYsrF.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-Cy2g0X2OZx.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-48RYi7P5w4.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CJ9k-wmTwA.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-C5Yqs-oE3u.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DTomAmq_pV.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-vH1AZzDYbp.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-BvfKfi9j2g.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-B9W5mp7Pc-.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CL-ZvphHYY.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DvptD6K6cu.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-D5ciu3l8Qw.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CbFLM138qE.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CMTIR1k2Vp.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DetynJx9PY.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-1s_R4bonAJ.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DsHhLSyYeg.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-Z6E5M310NG.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-CIAVaunxWo.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-BxOL7DBeLv.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DdbWhtMyTh.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-BaolDDrFXO.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DJdM89KVgu.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-BBEAsIkFV6.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-LSHJIEs1W4.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-StD-hACT-A.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-BL0RVjf1MM.js"/><link rel="modulepreload" href="https://cdn-client.medium.com/lite/assets/chunk-DYqHmMoGMW.js"/><style type="text/css" data-fela-rehydration="646" data-fela-type="STATIC">html{box-sizing:border-box;-webkit-text-size-adjust:100%}*, *:before, *:after{box-sizing:inherit}body{margin:0;padding:0;text-rendering:optimizeLegibility;-webkit-font-smoothing:antialiased;color:var(--color-fg-neutral-primary);position:relative;min-height:100vh}h1, h2, h3, h4, h5, h6, dl, dd, ol, ul, menu, figure, blockquote, p, pre, form{margin:0}menu, ol, ul{padding:0;list-style:none;list-style-image:none}main{display:block}a{color:inherit;text-decoration:none}a, button, input{-webkit-tap-highlight-color:transparent}img, svg{vertical-align:middle}button{background:transparent;overflow:visible}button, input, optgroup, select, textarea{margin:0}:root{--reach-tabs:1;--reach-menu-button:1}#speechify-root{font-family:Sohne, sans-serif}div[data-popper-reference-hidden="true"]{visibility:hidden;pointer-events:none}.grecaptcha-badge{visibility:hidden}
:root {--color-syntax-keyword: #cf222e;--color-syntax-keyword: light-dark(#cf222e, #ff7b72);--color-syntax-entity: #8250df;--color-syntax-entity: light-dark(#8250df, #d2a8ff);--color-syntax-constant: #0550ae;--color-syntax-constant: light-dark(#0550ae, #79c0ff);--color-syntax-string: #0a3069;--color-syntax-string: light-dark(#0a3069, #a5d6ff);--color-syntax-variable: #953800;--color-syntax-variable: light-dark(#953800, #ffa657);--color-syntax-comment: #656d76;--color-syntax-comment: light-dark(#656d76, #8b949e);--color-syntax-tag: #116329;--color-syntax-tag: light-dark(#116329, #7ee787);--color-syntax-subst: #24292f;--color-syntax-subst: light-dark(#24292f, #c9d1d9);--color-syntax-heading: #0550ae;--color-syntax-heading: light-dark(#0550ae, #58a6ff);--color-syntax-bullet: #3b2300;--color-syntax-bullet: light-dark(#3b2300, #f2cc60);--color-syntax-inserted: #116329;--color-syntax-inserted: light-dark(#116329, #aff5b4);--color-syntax-inserted-bg: #dafbe1;--color-syntax-inserted-bg: light-dark(#dafbe1, #033a16);--color-syntax-deleted: #82071e;--color-syntax-deleted: light-dark(#82071e, #ffdcd7);--color-syntax-deleted-bg: #ffebe9;--color-syntax-deleted-bg: light-dark(#ffebe9, #67060c);
}.hljs-doctag,
.hljs-keyword,
.hljs-meta .hljs-keyword,
.hljs-template-tag,
.hljs-template-variable,
.hljs-type,
.hljs-variable.language_ {color: var(--color-syntax-keyword);
}.hljs-title,
.hljs-title.class_,
.hljs-title.class_.inherited__,
.hljs-title.function_ {color: var(--color-syntax-entity);
}.hljs-attr,
.hljs-attribute,
.hljs-literal,
.hljs-meta,
.hljs-number,
.hljs-operator,
.hljs-variable,
.hljs-selector-attr,
.hljs-selector-class,
.hljs-selector-id {color: var(--color-syntax-constant);
}.hljs-regexp,
.hljs-string,
.hljs-meta .hljs-string {color: var(--color-syntax-string);
}.hljs-built_in,
.hljs-symbol {color: var(--color-syntax-variable);
}.hljs-code,
.hljs-comment,
.hljs-formula {color: var(--color-syntax-comment);
}.hljs-name,
.hljs-quote,
.hljs-selector-tag,
.hljs-selector-pseudo {color: var(--color-syntax-tag);
}.hljs-subst {color: var(--color-syntax-subst);
}.hljs-section {color: var(--color-syntax-heading);font-weight: bold;
}.hljs-bullet {color: var(--color-syntax-bullet);
}.hljs-addition {color: var(--color-syntax-inserted);background-color: var(--color-syntax-inserted-bg);
}.hljs-deletion {color: var(--color-syntax-deleted);background-color: var(--color-syntax-deleted-bg);
}.hljs-doctag,
.hljs-strong {font-weight: bold;
}.hljs-emphasis,
.hljs-formula {font-style: italic;
}
</style><style type="text/css" data-fela-rehydration="646" data-fela-type="KEYFRAME">@-webkit-keyframes k1{0%{opacity:0.8}50%{opacity:0.5}100%{opacity:0.8}}@-moz-keyframes k1{0%{opacity:0.8}50%{opacity:0.5}100%{opacity:0.8}}@keyframes k1{0%{opacity:0.8}50%{opacity:0.5}100%{opacity:0.8}}</style><style type="text/css" data-fela-rehydration="646" data-fela-type="RULE">.a{font-family:medium-content-sans-serif-font, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu, Cantarell, "Open Sans", "Helvetica Neue", sans-serif}.b{font-weight:400}.c{background-color:var(--color-bg-neutral-primary)}.d{display:none}.e{display:block}.f{position:sticky}.g{top:0}.h{z-index:500}.i{padding:0 24px}.j{align-items:center}.k{border-bottom:solid 1px var(--color-border-neutral-primary)}.t{height:41px}.u{line-height:20px}.v{margin-left:8px}.w{color:var(--color-fg-neutral-secondary)}.x{fill:inherit}.y{font-size:13px}.z{font-family:sohne, "Helvetica Neue", Helvetica, Arial, sans-serif}.ab{letter-spacing:inherit}.ac{padding:0}.ae{margin:0}.af{cursor:pointer}.ag:disabled{cursor:not-allowed}.ah:disabled{color:var(--color-fg-neutral-secondary)}.ai:disabled{fill:var(--color-fg-neutral-secondary)}.al{height:100%}.am{display:flex}.be{color:#FFFFFF}.bf{fill:#FFFFFF}.bg{background:#1A8917}.bh{border-color:var(--color-border-accent-primary)}.bl:disabled{cursor:inherit !important}.bm:disabled{opacity:0.3}.bn:disabled:hover{background:#1A8917}.bo:disabled:hover{border-color:var(--color-border-accent-primary)}.bp{border-radius:99em}.bq{border-width:1px}.br{border-style:solid}.bs{box-sizing:border-box}.bt{display:inline-block}.bu{text-decoration:none}.bv{text-align:center}.bw{margin-left:16px}.bx{color:inherit}.by{font-size:inherit}.bz{border:inherit}.ca{font-family:inherit}.cb{font-weight:inherit}.cc{gap:24px}.cd{height:57px}.ce{flex:1 0 auto}.cf{border:none}.cg{width:auto}.ch path{fill:var(--color-fg-neutral-primary)}.ci{height:25px}.cj{margin-left:24px}.cl{border-radius:20px}.cm{width:240px}.cn{background:var(--color-bg-neutral-tertiary)}.co path{fill:var(--color-fg-neutral-secondary)}.cq{outline:none}.cr{font-size:14px}.cs{width:100%}.ct{padding:10px 20px 10px 0}.cu{background-color:transparent}.cv{color:var(--color-fg-neutral-primary)}.cw::placeholder{color:var(--color-fg-neutral-secondary)}.cx{margin-left:12px}.cy{margin-right:12px}.db{gap:4px}.dc{padding:6px 12px}.dd{white-space:nowrap}.de{background-color:var(--color-bg-neutral-quaternary)}.df{width:24px}.dg{height:24px}.dh{flex-shrink:0}.di path{fill:var(--color-fg-neutral-tertiary)}.dj{color:var(--color-fg-neutral-tertiary)}.dm{position:relative}.dn{fill:var(--color-fg-neutral-secondary)}.do{gap:8px}.dr{position:absolute}.ds{width:1px}.dt{height:1px}.du{margin:-1px}.dv{overflow:hidden}.dw{clip:rect(0, 0, 0, 0)}.dx{border-width:0}.dy{margin-right:32px}.dz{background:transparent}.ea svg{margin-left:4px}.eb svg{fill:var(--color-fg-neutral-secondary)}.ed{box-shadow:inset 0 0 0 1px rgba(0, 0, 0, 0.05)}.ee{border-radius:50%}.ef{height:32px}.eg{width:32px}.ei{background-color:var(--color-bg-neutral-secondary)}.ej{flex:0 0 auto}.el{flex:1 1 auto}.er{max-width:100%}.es{text-overflow:ellipsis}.et{padding-top:3px}.eu{border-bottom:1px solid var(--color-border-neutral-primary)}.ev{justify-content:center}.fb{max-width:1192px}.fc{min-width:0}.fi{font-weight:500}.fw{margin:0 8px}.fx{display:inline}.fy{font-size:16px}.fz{line-height:24px}.gb{position:fixed}.gc{top:50%}.gd{right:16px}.ge{transform:translateY(-50%)}.gf{z-index:499}.gh{flex-direction:column}.gi{align-items:flex-end}.gj{padding:12px 4px}.gk{border-radius:999px}.gl{background-color:color-mix(in srgb, var(--color-bg-neutral-primary) 38%, transparent)}.gm{backdrop-filter:blur(10px)}.gn{-webkit-backdrop-filter:blur(10px)}.go{opacity:1}.gp{pointer-events:auto}.gq{transition:opacity 0.18s ease, visibility 0.18s ease}.gs{width:16px}.gt{height:2px}.gu{border-radius:1px}.gv{background-color:var(--color-fg-neutral-primary)}.gw{transition:opacity 0.15s ease, background-color 0.15s ease}.gx{opacity:0.15}.gy{max-width:300px}.gz{max-height:70vh}.ha{border-radius:8px}.hb{box-shadow:0px 0px 4px 0px color-mix(in srgb, var(--color-bg-neutral-quaternary) 5%, transparent), 0px 2px 8px 0px color-mix(in srgb, var(--color-bg-neutral-quaternary) 15%, transparent)}.hc{opacity:0}.hd{visibility:hidden}.he{pointer-events:none}.hf{overflow-y:auto}.hg{overscroll-behavior:contain}.hh{min-height:0}.hi{padding:12px 40px 12px 16px}.hj{list-style-type:none}.hk{padding:2px 0}.hl{width:4px}.hm{height:4px}.hn{background-color:var(--color-fg-accent-primary)}.ho{flex:1}.hp{max-height:20px}.hq{display:-webkit-box}.hr{-webkit-line-clamp:1}.hs{-webkit-box-orient:vertical}.ht{word-break:break-all}.hv{color:var(--color-fg-accent-primary)}.hw{will-change:opacity, transform}.hx{width:calc(100% - 0px)}.ia{transform:translateY(89px)}.ib::before{content:""}.ic::before{position:absolute}.id::before{inset:-12px}.ie::before{z-index:-1}.if::before{border-radius:12px}.ig::before{pointer-events:none}.ih::before{background-color:color-mix(in srgb, var(--color-bg-neutral-primary) 90%, transparent)}.ii::before{backdrop-filter:blur(10px)}.ij::before{-webkit-backdrop-filter:blur(10px)}.ik{width:160px}.il{align-items:flex-start}.im{gap:16px}.in{border-radius:2px}.io{height:38px}.ip{width:38px}.ir{text-decoration:underline}.is{word-wrap:break-word}.it{word-break:break-word}.iz{margin:0 24px}.jd{background:var(--color-bg-neutral-primary)}.je{border:1px solid var(--color-border-neutral-primary)}.jf{border-radius:4px}.jg{box-shadow:0 1px 4px var(--color-border-neutral-primary)}.jh{max-height:100vh}.ji{left:0}.jj{top:calc(100vh + 100px)}.jk{bottom:calc(100vh + 100px)}.jl{width:10px}.jm{margin-bottom:12px}.jn{margin-top:32px}.jo{flex-wrap:wrap}.jp{display:inline-flex}.jq{align-items:stretch}.jr{box-shadow:inset 0 0 0 1px var(--color-border-neutral-primary)}.js{display:contents}.jt{padding:4px 8px 4px 12px}.ju{border-radius:99em 0 0 99em}.jw{padding:4px 8px}.jx{border-radius:0 99em 99em 0}.jy{border-left:1px solid var(--color-border-neutral-primary)}.ka:after{display:block}.kb:after{content:""}.kc:after{clear:both}.kd{line-height:1.23}.ke{letter-spacing:0}.kf{font-style:normal}.kg{font-weight:700}.lb{margin-top:0px}.lc{justify-content:space-between}.ln{gap:12px}.lo{align-items:baseline}.lp{width:36px}.lq{height:36px}.lr{border:2px solid var(--color-bg-neutral-primary)}.ls{z-index:0}.lt{box-shadow:none}.lu{border:1px solid rgba(0, 0, 0, 0.05)}.lv{margin-bottom:2px}.lw{flex-wrap:nowrap}.ly{width:12px}.lz{padding-left:8px}.ma{padding-right:8px}.nb> *{flex-shrink:0}.nc{overflow-x:scroll}.nd::-webkit-scrollbar{display:none}.ne{scrollbar-width:none}.nf{-ms-overflow-style:none}.ng{max-width:74px}.nh{flex-direction:row}.ni{z-index:2}.nj{margin-right:4px}.nm{-webkit-user-select:none}.nn{border:0}.nq{outline:0}.nr{user-select:none}.ns> svg{pointer-events:none}.ob{cursor:progress}.oc{margin-left:32px}.od{margin-left:4px}.oe{padding:4px 0}.oo{padding:8px 2px}.op svg{color:var(--color-fg-neutral-secondary)}.ph{line-height:1.58}.pi{letter-spacing:-0.004em}.pj{font-family:source-serif-pro, Georgia, Cambria, "Times New Roman", Times, serif}.qb{margin-bottom:-0.46em}.qc{line-height:1.12}.qd{letter-spacing:-0.022em}.qe{font-weight:600}.qx{margin-bottom:-0.28em}.rd{list-style-type:decimal}.re{margin-left:30px}.rf{padding-left:0px}.rl{font-style:italic}.rm{margin-left:auto}.rn{margin-right:auto}.ro{max-width:649px}.ru{clear:both}.rw{cursor:zoom-in}.rx{z-index:auto}.rz{height:auto}.sa{margin-top:10px}.sb{max-width:728px}.se{max-width:653px}.sf{list-style-type:disc}.sg{max-width:700px}.sh{overflow-x:auto}.si{font-family:source-code-pro, Menlo, Monaco, "Courier New", Courier, monospace}.sj{padding:32px}.sk{border:1px solid var(--color-border-neutral-tertiary)}.sl{line-height:1.4}.sm{margin-top:-0.2em}.sn{margin-bottom:-0.2em}.so{white-space:pre}.sp{min-width:fit-content}.sq{max-width:928px}.sr{margin-bottom:26px}.ss{margin-top:6px}.st{padding:8px 12px 8px 16px}.su{padding:8px 12px}.sv{border-top:none}.sw{margin-bottom:50px}.sx{height:52px}.sy{max-height:52px}.sz{box-sizing:content-box}.ta{position:static}.tb{z-index:1}.tc{max-width:155px}.tk{margin:0 32px}.tl{margin-right:20px}.tm{margin-bottom:64px}.tn{gap:48px}.to{white-space:pre-wrap}.ug{height:48px}.uh{width:48px}.ui{height:64px}.uj{width:64px}.uk{align-self:flex-end}.uw{margin-top:8px}.ux{margin-top:16px}.uy{animation:k1 1.2s ease-in-out infinite}.uz{padding:32px 0}.va{border-radius:100%}.vb{margin-top:4px}.vc{margin-bottom:8px}.vd{height:8px}.ve{width:120px}.vf{width:80px}.vg{margin-top:12px}.vh{height:10px}.vi{width:90%}.vo{height:0px}.vr{height:20px}.vs{width:220px}.vt{width:180px}.vu{margin:24px 0}.vv{height:148px}.vw{margin-bottom:32px}.xj{flex-grow:0}.xk{height:200px}.xl{margin-bottom:16px}.xm{width:158px}.xn{height:16px}.xo{width:288px}.xp{width:296px}.xq{margin-bottom:56px}.xr{width:224px}.xs{border-bottom:solid 1px var(--color-border-neutral-tertiary)}.xt{margin-top:72px}.xu{padding:24px 0}.xv{margin-bottom:0px}.xw{margin-right:16px}.aj:hover:not(:disabled){color:var(--color-fg-neutral-primary)}.ak:hover:not(:disabled){fill:var(--color-fg-neutral-primary)}.bi:hover{background:#156D12}.bj:hover{border-color:var(--color-border-accent-primary-hover)}.bk:hover{cursor:pointer}.dp:hover{color:var(--color-fg-neutral-primary)}.dq:hover{fill:var(--color-fg-neutral-primary)}.ec:hover svg{fill:var(--color-fg-neutral-primary)}.eh:hover{background-color:rgba(0, 0, 0, 0.1)}.iq:hover{background-color:none}.jv:hover{background-color:var(--color-bg-neutral-secondary)}.jz:hover{border-left-color:transparent}.lx:hover{text-decoration:underline}.np:hover{fill:var(--color-fg-neutral-secondary-hover)}.of:hover p{color:var(--color-fg-neutral-secondary-hover)}.og:hover{color:var(--color-fg-neutral-secondary-hover)}.oh:hover path{stroke:var(--color-fg-neutral-secondary-hover) !important}.oi:hover{color:var(--color-fg-neutral-primary-hover)}.oq:hover svg{color:var(--color-fg-neutral-primary-hover)}.or:hover p{color:var(--color-fg-neutral-primary-hover)}.cp:focus-within path{fill:var(--color-fg-neutral-primary)}.no:focus-visible{fill:var(--color-fg-neutral-secondary-hover)}.os:focus svg{color:var(--color-fg-neutral-primary-hover)}.ry:focus{transform:scale(1.01)}.nt:active{border-style:none}</style><style type="text/css" data-fela-rehydration="646" data-fela-type="RULE" media="all and (min-width: 1080px)">.s{display:none}.av{font-size:14px}.aw{line-height:20px}.bc{font-size:13px}.bd{padding:5px 12px}.dl{display:flex}.ek{display:block}.fa{margin:0 64px}.fh{max-width:250px}.ft{font-size:20px}.fu{line-height:24px}.fv{letter-spacing:0}.iy{margin-bottom:50px}.jc{max-width:680px}.kx{font-size:42px}.ky{margin-bottom:32px}.kz{line-height:52px}.la{letter-spacing:-0.011em}.ll{align-items:center}.lm{flex-direction:row}.mn{border-top:solid 1px var(--color-bg-neutral-secondary)}.mo{border-bottom:solid 1px var(--color-bg-neutral-secondary)}.mp{margin:32px 0 0}.mq{padding:3px 8px}.mz> *{margin-right:24px}.na> :last-child{margin-right:0}.oa{margin-top:0px}.on{margin:0}.py{margin-top:2.14em}.pz{line-height:32px}.qa{letter-spacing:-0.003em}.qt{font-size:24px}.qu{margin-top:1.95em}.qv{line-height:30px}.qw{letter-spacing:-0.016em}.rc{margin-top:0.94em}.rk{margin-top:1.14em}.rt{margin-top:56px}.tj{display:inline-block}.tp{gap:8px}.tw{margin-bottom:0}.tx{margin-right:20px}.ul{max-width:500px}.vn{margin-bottom:52px}.wb{padding-top:72px}.wo{width:calc(100% + 32px)}.wp{margin-left:-16px}.wq{margin-right:-16px}.xf{padding-left:16px}.xg{padding-right:16px}.xh{flex-basis:50%}.xi{max-width:50%}</style><style type="text/css" data-fela-rehydration="646" data-fela-type="RULE" media="all and (max-width: 1079.98px)">.nz{margin-top:0px}.sc{margin-left:auto}.sd{text-align:center}.tg{display:none}.ti{display:inline-block}</style><style type="text/css" data-fela-rehydration="646" data-fela-type="RULE" media="all and (max-width: 903.98px)">.ga{display:none}.ny{margin-top:0px}.tf{display:inline-block}</style><style type="text/css" data-fela-rehydration="646" data-fela-type="RULE" media="all and (max-width: 727.98px)">.nw{margin-top:0px}.nx{margin-right:0px}.te{display:inline-block}.th{display:none}</style><style type="text/css" data-fela-rehydration="646" data-fela-type="RULE" media="all and (max-width: 551.98px)">.l{display:flex}.m{justify-content:space-between}.an{font-size:13px}.ao{line-height:20px}.ax{padding:0px 8px 1px}.ck{display:none}.em{min-width:100%}.ew{margin:0 24px}.fd{max-width:150px}.fj{font-size:16px}.fk{letter-spacing:0}.iu{margin-bottom:2px}.kh{font-size:32px}.ki{margin-bottom:24px}.kj{line-height:38px}.kk{letter-spacing:-0.014em}.ld{align-items:flex-start}.le{flex-direction:column-reverse}.mb{margin:24px -24px 0}.mc{padding:0}.mr> *{margin-right:8px}.ms> :last-child{margin-right:24px}.nk{margin-left:0px}.nu{margin-top:0px}.nv{margin-right:0px}.oj{margin:0}.ot{border:1px solid var(--color-border-neutral-primary)}.ou{border-radius:99em}.ov{padding:0px 16px 0px 12px}.ow{height:38px}.ox{align-items:center}.oz svg{margin-right:8px}.pk{font-size:18px}.pl{margin-top:1.56em}.pm{line-height:28px}.pn{letter-spacing:-0.003em}.qf{font-size:20px}.qg{margin-top:1.2em}.qh{line-height:24px}.qy{margin-top:0.67em}.rg{margin-top:1.34em}.rp{margin-top:40px}.td{display:inline-block}.tu{flex-direction:column}.tv{gap:0}.ue{margin-bottom:20px}.uf{margin-right:0}.up{max-width:100%}.uq{font-size:24px}.ur{line-height:30px}.us{letter-spacing:-0.016em}.vj{margin-bottom:40px}.vp{margin-bottom:30px}.vx{padding-top:48px}.wc{width:calc(100% + 24px)}.wd{margin-left:-12px}.we{margin-right:-12px}.wr{padding-left:12px}.ws{padding-right:12px}.wt{flex-basis:100%}.oy:hover{border-color:var(--color-border-neutral-tertiary)}</style><style type="text/css" data-fela-rehydration="646" data-fela-type="RULE" media="all and (min-width: 904px) and (max-width: 1079.98px)">.r{display:none}.at{font-size:14px}.au{line-height:20px}.ba{font-size:13px}.bb{padding:5px 12px}.dk{display:flex}.eq{min-width:100%}.ez{margin:0 64px}.fg{max-width:250px}.fq{font-size:20px}.fr{line-height:24px}.fs{letter-spacing:0}.ix{margin-bottom:50px}.jb{max-width:680px}.kt{font-size:42px}.ku{margin-bottom:32px}.kv{line-height:52px}.kw{letter-spacing:-0.011em}.lj{align-items:center}.lk{flex-direction:row}.mj{border-top:solid 1px var(--color-bg-neutral-secondary)}.mk{border-bottom:solid 1px var(--color-bg-neutral-secondary)}.ml{margin:32px 0 0}.mm{padding:3px 8px}.mx> *{margin-right:24px}.my> :last-child{margin-right:0}.om{margin:0}.pv{margin-top:2.14em}.pw{line-height:32px}.px{letter-spacing:-0.003em}.qp{font-size:24px}.qq{margin-top:1.95em}.qr{line-height:30px}.qs{letter-spacing:-0.016em}.rb{margin-top:0.94em}.rj{margin-top:1.14em}.rs{margin-top:56px}.tq{gap:8px}.ty{margin-bottom:0}.tz{margin-right:20px}.um{max-width:500px}.vm{margin-bottom:52px}.wa{padding-top:72px}.wl{width:calc(100% + 32px)}.wm{margin-left:-16px}.wn{margin-right:-16px}.xb{padding-left:16px}.xc{padding-right:16px}.xd{flex-basis:50%}.xe{max-width:50%}</style><style type="text/css" data-fela-rehydration="646" data-fela-type="RULE" media="all and (min-width: 728px) and (max-width: 903.98px)">.p{display:flex}.q{justify-content:space-between}.ar{font-size:13px}.as{line-height:20px}.az{padding:0px 8px 1px}.da{display:none}.ep{min-width:100%}.ey{margin:0 48px}.ff{max-width:250px}.fn{font-size:20px}.fo{line-height:24px}.fp{letter-spacing:0}.iw{margin-bottom:50px}.ja{max-width:680px}.kp{font-size:42px}.kq{margin-bottom:32px}.kr{line-height:52px}.ks{letter-spacing:-0.011em}.lh{align-items:center}.li{flex-direction:row}.mf{border-top:solid 1px var(--color-bg-neutral-secondary)}.mg{border-bottom:solid 1px var(--color-bg-neutral-secondary)}.mh{margin:32px 0 0}.mi{padding:3px 8px}.mv> *{margin-right:24px}.mw> :last-child{margin-right:0}.ol{margin:0}.ps{margin-top:2.14em}.pt{line-height:32px}.pu{letter-spacing:-0.003em}.ql{font-size:24px}.qm{margin-top:1.95em}.qn{line-height:30px}.qo{letter-spacing:-0.016em}.ra{margin-top:0.94em}.ri{margin-top:1.14em}.rr{margin-top:56px}.tr{gap:8px}.ua{margin-bottom:0}.ub{margin-right:20px}.un{max-width:500px}.vl{margin-bottom:52px}.vz{padding-top:72px}.wi{width:calc(100% + 28px)}.wj{margin-left:-14px}.wk{margin-right:-14px}.wx{padding-left:14px}.wy{padding-right:14px}.wz{flex-basis:50%}.xa{max-width:50%}</style><style type="text/css" data-fela-rehydration="646" data-fela-type="RULE" media="all and (min-width: 552px) and (max-width: 727.98px)">.n{display:flex}.o{justify-content:space-between}.ap{font-size:13px}.aq{line-height:20px}.ay{padding:0px 8px 1px}.cz{display:none}.eo{min-width:100%}.ex{margin:0 24px}.fe{max-width:150px}.fl{font-size:16px}.fm{letter-spacing:0}.iv{margin-bottom:2px}.kl{font-size:32px}.km{margin-bottom:24px}.kn{line-height:38px}.ko{letter-spacing:-0.014em}.lf{align-items:flex-start}.lg{flex-direction:column-reverse}.md{margin:24px 0 0}.me{padding:0}.mt> *{margin-right:8px}.mu> :last-child{margin-right:8px}.nl{margin-left:0px}.ok{margin:0}.pa{border:1px solid var(--color-border-neutral-primary)}.pb{border-radius:99em}.pc{padding:0px 16px 0px 12px}.pd{height:38px}.pe{align-items:center}.pg svg{margin-right:8px}.po{font-size:18px}.pp{margin-top:1.56em}.pq{line-height:28px}.pr{letter-spacing:-0.003em}.qi{font-size:20px}.qj{margin-top:1.2em}.qk{line-height:24px}.qz{margin-top:0.67em}.rh{margin-top:1.34em}.rq{margin-top:40px}.ts{flex-direction:column}.tt{gap:0}.uc{margin-bottom:20px}.ud{margin-right:0}.uo{max-width:100%}.ut{font-size:24px}.uu{line-height:30px}.uv{letter-spacing:-0.016em}.vk{margin-bottom:40px}.vq{margin-bottom:30px}.vy{padding-top:48px}.wf{width:calc(100% + 24px)}.wg{margin-left:-12px}.wh{margin-right:-12px}.wu{padding-left:12px}.wv{padding-right:12px}.ww{flex-basis:100%}.pf:hover{border-color:var(--color-border-neutral-tertiary)}</style><style type="text/css" data-fela-rehydration="646" data-fela-type="RULE" media="print">.gg{display:none}</style><style type="text/css" data-fela-rehydration="646" data-fela-type="RULE" media="(prefers-reduced-motion: reduce)">.gr{transition:none}</style><style type="text/css" data-fela-rehydration="646" data-fela-type="RULE" media="(orientation: landscape) and (max-width: 903.98px)">.hu{max-height:none}</style><style type="text/css" data-fela-rehydration="646" data-fela-type="RULE" media="(prefers-reduced-motion: no-preference)">.hy{transition:opacity 200ms}.rv{transition:transform 300ms cubic-bezier(0.2, 0, 0.2, 1)}</style><style type="text/css" data-fela-rehydration="646" data-fela-type="RULE" media="all and (max-width: 1256px)">.hz{display:none}</style></head><body><!--$--><!--/$--><div class="a b c"><a href="/sitemap/sitemap.xml" class="d">Sitemap</a><link id="glyph_link" rel="stylesheet" type="text/css" href="https://glyph.medium.com/css/unbound.css"/><script>document.domain = document.domain;</script><div class="e f g h c"><div class="i j k l m n o p q r s t u"><a class="w x y z ab b ac ae af ag ah ai aj ak l n p r s j al u" href="https://play.google.com/store/apps/details?id=com.medium.reader&amp;referrer=utm_source%3DmobileNavBar&amp;source=---top_nav_layout_nav-------------------------------------------" rel="noopener follow">Open in app<svg width="10" height="10" viewBox="0 0 10 10" fill="none" xmlns="http://www.w3.org/2000/svg" class="v"><path d="M0.984835 8.48484C0.838388 8.63128 0.838388 8.86872 0.984835 9.01516C1.13128 9.16161 1.36872 9.16161 1.51517 9.01516L0.984835 8.48484ZM8.75 1.25H9.125C9.125 1.04289 8.95711 0.875 8.75 0.875V1.25ZM8.375 6.5C8.375 6.70711 8.54289 6.875 8.75 6.875C8.95711 6.875 9.125 6.70711 9.125 6.5H8.375ZM3.5 0.875C3.29289 0.875 3.125 1.04289 3.125 1.25C3.125 1.45711 3.29289 1.625 3.5 1.625V0.875ZM1.51517 9.01516L9.01516 1.51517L8.48484 0.984835L0.984835 8.48484L1.51517 9.01516ZM8.375 1.25V6.5H9.125V1.25H8.375ZM3.5 1.625H8.75V0.875H3.5V1.625Z" fill="currentColor"></path></svg></a><div class="am j"><p class="z b an ao ap aq ar as at au av aw w"><span data-dd-action-name="Susi presentation tracker global_nav"><button class="z b an ao ax ap aq ay ar as az ba au bb bc aw bd be bf bg bh bi bj bk bl bm bn bo bp bq br bs bt bu bv" data-testid="headerSignUpButton">Sign up</button></span></p><div class="bw e"><p class="z b an ao ap aq ar as at au av aw w"><span data-dd-action-name="Susi presentation tracker global_nav"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" data-testid="headerSignInButton" rel="noopener follow" href="/m/signin?operation=login&amp;redirect=https%3A%2F%2Fmedium.com%2Fmixpaneleng%2Fhow-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5&amp;source=post_page---top_nav_layout_nav-----------------------global_nav--------------------" data-discover="true">Sign in</a></span></p></div></div></div><div class="i j k am cc cd"><div class="am j ce"><a class="bx x by cf ca ab cb ac ae af ag ah ai aj ak am" aria-label="Homepage" data-testid="headerMediumLogo" rel="noopener follow" href="/?source=---top_nav_layout_nav-------------------------------------------" data-discover="true"><svg xmlns="http://www.w3.org/2000/svg" width="719" height="160" fill="none" aria-labelledby="wordmark-medium-desc" viewBox="0 0 719 160" class="cg ch ci"><desc>Medium Logo</desc><path fill="#242424" d="m174.104 9.734.215-.047V8.02H130.39L89.6 103.89 48.81 8.021H1.472v1.666l.212.047c8.018 1.81 12.09 4.509 12.09 14.242V137.93c0 9.734-4.087 12.433-12.106 14.243l-.212.047v1.671h32.118v-1.665l-.213-.048c-8.018-1.809-12.089-4.509-12.089-14.242V30.586l52.399 123.305h2.972l53.925-126.743V140.75c-.687 7.688-4.721 10.062-11.982 11.701l-.215.05v1.652h55.948v-1.652l-.215-.05c-7.269-1.639-11.4-4.013-12.087-11.701l-.037-116.774h.037c0-9.733 4.071-12.432 12.087-14.242m25.555 75.488c.915-20.474 8.268-35.252 20.606-35.507 3.806.063 6.998 1.312 9.479 3.714 5.272 5.118 7.751 15.812 7.368 31.793zm-.553 5.77h65.573v-.275c-.186-15.656-4.721-27.834-13.466-36.196-7.559-7.227-18.751-11.203-30.507-11.203h-.263c-6.101 0-13.584 1.48-18.909 4.16-6.061 2.807-11.407 7.003-15.855 12.511-7.161 8.874-11.499 20.866-12.554 34.343q-.05.606-.092 1.212a50 50 0 0 0-.065 1.151 85.807 85.807 0 0 0-.094 5.689c.71 30.524 17.198 54.917 46.483 54.917 25.705 0 40.675-18.791 44.407-44.013l-1.886-.664c-6.557 13.556-18.334 21.771-31.738 20.769-18.297-1.369-32.314-19.922-31.042-42.395m139.722 41.359c-2.151 5.101-6.639 7.908-12.653 7.908s-11.513-4.129-15.418-11.63c-4.197-8.053-6.405-19.436-6.405-32.92 0-28.067 8.729-46.22 22.24-46.22 5.657 0 10.111 2.807 12.236 7.704zm43.499 20.008c-8.019-1.897-12.089-4.722-12.089-14.951V1.309l-48.716 14.353v1.757l.299-.024c6.72-.543 11.278.386 13.925 2.83 2.072 1.915 3.082 4.853 3.082 8.987v18.66c-4.803-3.067-10.516-4.56-17.448-4.56-14.059 0-26.909 5.92-36.176 16.672-9.66 11.205-14.767 26.518-14.767 44.278-.003 31.72 15.612 53.039 38.851 53.039 13.595 0 24.533-7.449 29.54-20.013v16.865h43.711v-1.746zM424.1 19.819c0-9.904-7.468-17.374-17.375-17.374-9.859 0-17.573 7.632-17.573 17.374s7.721 17.374 17.573 17.374c9.907 0 17.375-7.47 17.375-17.374m11.499 132.546c-8.019-1.897-12.089-4.722-12.089-14.951h-.035V43.635l-43.714 12.551v1.705l.263.024c9.458.842 12.047 4.1 12.047 15.152v81.086h43.751v-1.746zm112.013 0c-8.018-1.897-12.089-4.722-12.089-14.951V43.635l-41.621 12.137v1.71l.246.026c7.733.813 9.967 4.257 9.967 15.36v59.279c-2.578 5.102-7.415 8.131-13.274 8.336-9.503 0-14.736-6.419-14.736-18.073V43.638l-43.714 12.55v1.703l.262.024c9.459.84 12.05 4.097 12.05 15.152v50.17a56.3 56.3 0 0 0 .91 10.444l.787 3.423c3.701 13.262 13.398 20.197 28.59 20.197 12.868 0 24.147-7.966 29.115-20.43v17.311h43.714v-1.747zm169.818 1.788v-1.749l-.213-.05c-8.7-2.006-12.089-5.789-12.089-13.49v-63.79c0-19.89-11.171-31.761-29.883-31.761-13.64 0-25.141 7.882-29.569 20.16-3.517-13.01-13.639-20.16-28.606-20.16-13.146 0-23.449 6.938-27.869 18.657V43.643L545.487 55.68v1.715l.263.024c9.345.829 12.047 4.181 12.047 14.95v81.784h40.787v-1.746l-.215-.053c-6.941-1.631-9.181-4.606-9.181-12.239V66.998c1.836-4.289 5.537-9.37 12.853-9.37 9.086 0 13.692 6.296 13.692 18.697v77.828h40.797v-1.746l-.215-.053c-6.94-1.631-9.18-4.606-9.18-12.239V75.066a42 42 0 0 0-.578-7.26c1.947-4.661 5.86-10.177 13.475-10.177 9.214 0 13.691 6.114 13.691 18.696v77.828z"></path></svg></a><div class="cj ck"><div class="am cf cl cm cn j co cp"><div class="bt" aria-describedby="searchResults" aria-labelledby="searchResults" aria-haspopup="listbox"></div><div class="cx cy am"><svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.09234 11.06C4.09234 10.1473 4.27211 9.24358 4.62138 8.40036C4.97065 7.55715 5.48258 6.79099 6.12795 6.14562C6.77332 5.50026 7.53948 4.98832 8.38269 4.63905C9.2259 4.28978 10.1297 4.11001 11.0423 4.11001C11.955 4.11001 12.8588 4.28978 13.702 4.63905C14.5452 4.98832 15.3114 5.50026 15.9567 6.14562C16.6021 6.79099 17.114 7.55715 17.4633 8.40036C17.8126 9.24358 17.9923 10.1473 17.9923 11.06C17.9923 12.9033 17.2601 14.671 15.9567 15.9744C14.6534 17.2778 12.8856 18.01 11.0423 18.01C9.19909 18.01 7.43133 17.2778 6.12795 15.9744C4.82457 14.671 4.09234 12.9033 4.09234 11.06ZM11.0423 3.01001C9.75141 3.01124 8.4797 3.32292 7.33449 3.91876C6.18929 4.51459 5.20418 5.37711 4.46226 6.43355C3.72034 7.48999 3.24337 8.70937 3.07159 9.98883C2.89981 11.2683 3.03825 12.5703 3.47525 13.785C3.91224 14.9997 4.63497 16.0915 5.58245 16.9683C6.52994 17.8451 7.6744 18.4812 8.9193 18.8229C10.1642 19.1646 11.473 19.2019 12.7353 18.9316C13.9977 18.6613 15.1765 18.0915 16.1723 17.27L19.9223 21.02C19.9727 21.078 20.0345 21.1251 20.1039 21.1582C20.1732 21.1913 20.2487 21.2098 20.3255 21.2125C20.4023 21.2152 20.4788 21.2021 20.5504 21.1739C20.6219 21.1458 20.6868 21.1032 20.7412 21.0488C20.7955 20.9945 20.8381 20.9295 20.8662 20.858C20.8944 20.7865 20.9075 20.71 20.9048 20.6332C20.9021 20.5563 20.8836 20.4809 20.8505 20.4116C20.8174 20.3422 20.7704 20.2804 20.7123 20.23L16.9823 16.5C18.0418 15.3483 18.7426 13.9128 18.999 12.3691C19.2554 10.8253 19.0564 9.24032 18.4261 7.80795C17.7959 6.37559 16.7618 5.15798 15.4504 4.30407C14.1391 3.45017 12.6072 2.997 11.0423 3.00001V3.01001Z" fill="currentColor"></path></svg></div><input role="combobox" aria-controls="searchResults" aria-expanded="false" aria-label="search" data-testid="headerSearchInput" tabindex="0" class="cf cq z cr u cs ct cu cv cw" placeholder="Search" value=""/></div></div></div><div class="ck cz da"><button type="button" class="am j db dc bp cf af dd de" aria-label="Get the Medium app"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" class="df dg dh di"><g clip-path="url(#a)"><path fill="#000" d="M9.75 3.938h-4.5A1.313 1.313 0 0 0 3.938 5.25v4.5a1.313 1.313 0 0 0 1.312 1.313h4.5a1.31 1.31 0 0 0 1.313-1.313v-4.5A1.31 1.31 0 0 0 9.75 3.938m.188 5.812a.187.187 0 0 1-.188.188h-4.5a.187.187 0 0 1-.187-.188v-4.5a.187.187 0 0 1 .187-.187h4.5a.187.187 0 0 1 .188.187zm-.188 3.188h-4.5a1.313 1.313 0 0 0-1.312 1.312v4.5a1.313 1.313 0 0 0 1.312 1.313h4.5a1.31 1.31 0 0 0 1.313-1.313v-4.5a1.31 1.31 0 0 0-1.313-1.312m.188 5.812a.19.19 0 0 1-.188.188h-4.5a.19.19 0 0 1-.187-.188v-4.5a.19.19 0 0 1 .187-.187h4.5a.19.19 0 0 1 .188.187zM18.75 3.938h-4.5a1.313 1.313 0 0 0-1.312 1.312v4.5a1.31 1.31 0 0 0 1.312 1.313h4.5a1.31 1.31 0 0 0 1.313-1.313v-4.5a1.313 1.313 0 0 0-1.313-1.312m.188 5.812a.19.19 0 0 1-.188.188h-4.5a.19.19 0 0 1-.187-.188v-4.5a.19.19 0 0 1 .187-.187h4.5a.19.19 0 0 1 .188.187zm-6 6.75v-3a.562.562 0 1 1 1.124 0v3a.562.562 0 1 1-1.124 0m7.124-1.5a.56.56 0 0 1-.562.563h-2.437V19.5a.56.56 0 0 1-.563.563h-3a.562.562 0 1 1 0-1.125h2.438V13.5a.562.562 0 1 1 1.124 0v.938H19.5a.56.56 0 0 1 .563.562m0 3v1.5a.562.562 0 1 1-1.125 0V18a.562.562 0 1 1 1.125 0"></path></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h24v24H0z"></path></clipPath></defs></svg><span class="z b cr u dj">Get app</span></button></div><div class="ck cz p dk dl"><span data-dd-action-name="Susi presentation tracker new_post_topnav"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" data-testid="headerWriteButton" rel="noopener follow" href="/m/signin?operation=register&amp;redirect=https%3A%2F%2Fmedium.com%2Fnew-story&amp;source=---top_nav_layout_nav-----------------------new_post_topnav--------------------" data-discover="true"><div class="z b cr u w dm dn am j do dp dq"><svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M14 4C14.2761 4 14.5 3.77614 14.5 3.5C14.5 3.22386 14.2761 3 14 3V4ZM21 10C21 9.72386 20.7761 9.5 20.5 9.5C20.2239 9.5 20 9.72386 20 10H21ZM14 3H4V4H14V3ZM3 4V20H4V4H3ZM4 21H20V20H4V21ZM21 20V10H20V20H21ZM20 21C20.5523 21 21 20.5523 21 20H20V21ZM3 20C3 20.5523 3.44771 21 4 21V20H3ZM4 3C3.44772 3 3 3.44772 3 4H4H4V3Z" fill="currentColor"></path><path d="M17.5 4.5L9.04173 12.9583C9.01429 12.9857 8.99361 13.0192 8.98134 13.056L8.15811 15.5257C8.09297 15.7211 8.2789 15.907 8.47434 15.8419L10.944 15.0187C10.9808 15.0064 11.0143 14.9857 11.0417 14.9583L19.5 6.5M17.5 4.5L19.8232 2.17678C19.9209 2.07915 20.0791 2.07915 20.1768 2.17678L21.8232 3.82322C21.9209 3.92085 21.9209 4.07915 21.8232 4.17678L19.5 6.5M17.5 4.5L19.5 6.5" stroke="currentColor"></path></svg>Write</div></a></span></div><div class="cz da r s"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" data-testid="headerSearchButton" rel="noopener follow" href="/search?source=---top_nav_layout_nav-------------------------------------------" data-discover="true"><div class="z b cr u w dm dn am j do dp dq"><svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M4.09234 11.06C4.09234 10.1473 4.27211 9.24358 4.62138 8.40036C4.97065 7.55715 5.48258 6.79099 6.12795 6.14562C6.77332 5.50026 7.53948 4.98832 8.38269 4.63905C9.2259 4.28978 10.1297 4.11001 11.0423 4.11001C11.955 4.11001 12.8588 4.28978 13.702 4.63905C14.5452 4.98832 15.3114 5.50026 15.9567 6.14562C16.6021 6.79099 17.114 7.55715 17.4633 8.40036C17.8126 9.24358 17.9923 10.1473 17.9923 11.06C17.9923 12.9033 17.2601 14.671 15.9567 15.9744C14.6534 17.2778 12.8856 18.01 11.0423 18.01C9.19909 18.01 7.43133 17.2778 6.12795 15.9744C4.82457 14.671 4.09234 12.9033 4.09234 11.06ZM11.0423 3.01001C9.75141 3.01124 8.4797 3.32292 7.33449 3.91876C6.18929 4.51459 5.20418 5.37711 4.46226 6.43355C3.72034 7.48999 3.24337 8.70937 3.07159 9.98883C2.89981 11.2683 3.03825 12.5703 3.47525 13.785C3.91224 14.9997 4.63497 16.0915 5.58245 16.9683C6.52994 17.8451 7.6744 18.4812 8.9193 18.8229C10.1642 19.1646 11.473 19.2019 12.7353 18.9316C13.9977 18.6613 15.1765 18.0915 16.1723 17.27L19.9223 21.02C19.9727 21.078 20.0345 21.1251 20.1039 21.1582C20.1732 21.1913 20.2487 21.2098 20.3255 21.2125C20.4023 21.2152 20.4788 21.2021 20.5504 21.1739C20.6219 21.1458 20.6868 21.1032 20.7412 21.0488C20.7955 20.9945 20.8381 20.9295 20.8662 20.858C20.8944 20.7865 20.9075 20.71 20.9048 20.6332C20.9021 20.5563 20.8836 20.4809 20.8505 20.4116C20.8174 20.3422 20.7704 20.2804 20.7123 20.23L16.9823 16.5C18.0418 15.3483 18.7426 13.9128 18.999 12.3691C19.2554 10.8253 19.0564 9.24032 18.4261 7.80795C17.7959 6.37559 16.7618 5.15798 15.4504 4.30407C14.1391 3.45017 12.6072 2.997 11.0423 3.00001V3.01001Z" fill="currentColor"></path></svg><span class="dr ds dt ac du dv dw dd dx speechify-ignore">Search</span></div></a></div><div class="dy ck cz da"><div class="am j"><p class="z b an ao ap aq ar as at au av aw w"><span data-dd-action-name="Susi presentation tracker global_nav"><button class="z b an ao ax ap aq ay ar as az ba au bb bc aw bd be bf bg bh bi bj bk bl bm bn bo bp bq br bs bt bu bv" data-testid="headerSignUpButton">Sign up</button></span></p><div class="bw e"><p class="z b an ao ap aq ar as at au av aw w"><span data-dd-action-name="Susi presentation tracker global_nav"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" data-testid="headerSignInButton" rel="noopener follow" href="/m/signin?operation=login&amp;redirect=https%3A%2F%2Fmedium.com%2Fmixpaneleng%2Fhow-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5&amp;source=post_page---top_nav_layout_nav-----------------------global_nav--------------------" data-discover="true">Sign in</a></span></p></div></div></div><div class="e"><div><div class="bt"><div tabindex="-1" class="cq"><button class="cf dz ac am j af dm ea eb ec" aria-label="user options menu" data-testid="headerUserIcon"><div class="e dm"><img alt="Unknown user" class="e bs ee ef eg ei" src="https://miro.medium.com/v2/resize:fill:64:64/1*dmbNkD5D-u45r44go_cf0g.png" width="32" height="32" loading="lazy"/><div class="ed ee e ef eg dr g cf eh"></div></div></button></div></div></div></div></div></div><div class="am"><div class="ej ck cz da r ek" style="width:0px"></div><div class="el cs em eo ep eq" style="width:calc(100% - 0px)"><div class="e"><div><div class="et eu cs e"><div class="am ev"><div class="ew ex ey ez fa fb fc cs"><div class="cd am j"><div class="fd fe ff fg fh e"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" href="https://medium.com/mixpaneleng?source=post_page---publication_nav-b5de3443ad1f-fb973af124f5-----------------------------------------" rel="noopener follow"><h2 class="z fi fj ao fk fl aq fm fn fo fp fq fr fs ft fu fv cv"><div class="er dv es dd">Mixpanel Engineering</div></h2></a></div><div class="l n da r s"><span class="fw fx" aria-hidden="true"><span class="z b fy fz cv">·</span></span><p class="z b fy fz cv"></p></div></div></div></div></div></div><nav class="ga" aria-label="Table of contents"><button type="button" aria-label="Table of contents" aria-haspopup="true" aria-expanded="false" aria-controls="_R_fjabr5_" class="gb gc gd ge gf gg am gh gi do ae gj cf af gk gl gm gn go gp gq gr"><span aria-hidden="true" class="e gs gt gu go gv gw gr"></span><span aria-hidden="true" class="e gs gt gu gx de gw gr"></span><span aria-hidden="true" class="e gs gt gu gx de gw gr"></span><span aria-hidden="true" class="e gs gt gu gx de gw gr"></span><span aria-hidden="true" class="e gs gt gu gx de gw gr"></span><span aria-hidden="true" class="e gs gt gu gx de gw gr"></span><span aria-hidden="true" class="e gs gt gu gx de gw gr"></span><span aria-hidden="true" class="e gs gt gu gx de gw gr"></span><span aria-hidden="true" class="e gs gt gu gx de gw gr"></span><span aria-hidden="true" class="e gs gt gu gx de gw gr"></span><span aria-hidden="true" class="e gs gt gu gx de gw gr"></span><span aria-hidden="true" class="e gs gt gu gx de gw gr"></span><span aria-hidden="true" class="e gs gt gu gx de gw gr"></span><span aria-hidden="true" class="e gs gt gu gx de gw gr"></span></button><div id="_R_fjabr5_" class="gb gc gd ge gf gg am gh gy gz dv ha c hb hc hd he gq gr" data-testid="toc-popover"><div class="hf hg hh hi"><ol class="ae ac hj"><li class="hk"><a class="hv x cr bz z ab b ac ae af ag ah ai aj ak am j do u" title="Background" aria-current="location" href="/?source=post_page-----fb973af124f5-----------------------------------------#da62" rel="noopener follow"><span aria-hidden="true" class="dh hl hm ee hn"></span><span class="ho fc z b cr u dv hp es hq hr hs ht hu bx">Background</span></a></li><li class="hk"><a class="w x cr bz z ab b ac ae af ag ah ai aj ak am j do u" title="Performance" href="/?source=post_page-----fb973af124f5-----------------------------------------#05df" rel="noopener follow"><span aria-hidden="true" class="dh hl hm ee cu"></span><span class="ho fc z b cr u dv hp es hq hr hs ht hu bx">Performance</span></a></li><li class="hk"><a class="w x cr bz z ab b ac ae af ag ah ai aj ak am j do u" title="Reduce Vendor Lock-in" href="/?source=post_page-----fb973af124f5-----------------------------------------#0a8c" rel="noopener follow"><span aria-hidden="true" class="dh hl hm ee cu"></span><span class="ho fc z b cr u dv hp es hq hr hs ht hu bx">Reduce Vendor Lock-in</span></a></li><li class="hk"><a class="w x cr bz z ab b ac ae af ag ah ai aj ak am j do u" title="Ecosystem" href="/?source=post_page-----fb973af124f5-----------------------------------------#1aa4" rel="noopener follow"><span aria-hidden="true" class="dh hl hm ee cu"></span><span class="ho fc z b cr u dv hp es hq hr hs ht hu bx">Ecosystem</span></a></li><li class="hk"><a class="w x cr bz z ab b ac ae af ag ah ai aj ak am j do u" title="Challenges" href="/?source=post_page-----fb973af124f5-----------------------------------------#f66d" rel="noopener follow"><span aria-hidden="true" class="dh hl hm ee cu"></span><span class="ho fc z b cr u dv hp es hq hr hs ht hu bx">Challenges</span></a></li><li class="hk"><a class="w x cr bz z ab b ac ae af ag ah ai aj ak am j do u" title="Metrics Collection" href="/?source=post_page-----fb973af124f5-----------------------------------------#d4a5" rel="noopener follow"><span aria-hidden="true" class="dh hl hm ee cu"></span><span class="ho fc z b cr u dv hp es hq hr hs ht hu bx">Metrics Collection</span></a></li><li class="hk"><a class="w x cr bz z ab b ac ae af ag ah ai aj ak am j do u" title="Takeaways" href="/?source=post_page-----fb973af124f5-----------------------------------------#3c50" rel="noopener follow"><span aria-hidden="true" class="dh hl hm ee cu"></span><span class="ho fc z b cr u dv hp es hq hr hs ht hu bx">Takeaways</span></a></li><li class="hk"><a class="w x cr bz z ab b ac ae af ag ah ai aj ak am j do u" title="Metrics Reshaping" href="/?source=post_page-----fb973af124f5-----------------------------------------#fff7" rel="noopener follow"><span aria-hidden="true" class="dh hl hm ee cu"></span><span class="ho fc z b cr u dv hp es hq hr hs ht hu bx">Metrics Reshaping</span></a></li><li class="hk"><a class="w x cr bz z ab b ac ae af ag ah ai aj ak am j do u" title="Takeaways" href="/?source=post_page-----fb973af124f5-----------------------------------------#ead5" rel="noopener follow"><span aria-hidden="true" class="dh hl hm ee cu"></span><span class="ho fc z b cr u dv hp es hq hr hs ht hu bx">Takeaways</span></a></li><li class="hk"><a class="w x cr bz z ab b ac ae af ag ah ai aj ak am j do u" title="Query Rewrites" href="/?source=post_page-----fb973af124f5-----------------------------------------#ae8c" rel="noopener follow"><span aria-hidden="true" class="dh hl hm ee cu"></span><span class="ho fc z b cr u dv hp es hq hr hs ht hu bx">Query Rewrites</span></a></li><li class="hk"><a class="w x cr bz z ab b ac ae af ag ah ai aj ak am j do u" title="Can We Automate it?" href="/?source=post_page-----fb973af124f5-----------------------------------------#f501" rel="noopener follow"><span aria-hidden="true" class="dh hl hm ee cu"></span><span class="ho fc z b cr u dv hp es hq hr hs ht hu bx">Can We Automate it?</span></a></li><li class="hk"><a class="w x cr bz z ab b ac ae af ag ah ai aj ak am j do u" title="Reduce the Pain" href="/?source=post_page-----fb973af124f5-----------------------------------------#91d4" rel="noopener follow"><span aria-hidden="true" class="dh hl hm ee cu"></span><span class="ho fc z b cr u dv hp es hq hr hs ht hu bx">Reduce the Pain</span></a></li><li class="hk"><a class="w x cr bz z ab b ac ae af ag ah ai aj ak am j do u" title="Takeaways" href="/?source=post_page-----fb973af124f5-----------------------------------------#893e" rel="noopener follow"><span aria-hidden="true" class="dh hl hm ee cu"></span><span class="ho fc z b cr u dv hp es hq hr hs ht hu bx">Takeaways</span></a></li><li class="hk"><a class="w x cr bz z ab b ac ae af ag ah ai aj ak am j do u" title="Final Thoughts" href="/?source=post_page-----fb973af124f5-----------------------------------------#fe24" rel="noopener follow"><span aria-hidden="true" class="dh hl hm ee cu"></span><span class="ho fc z b cr u dv hp es hq hr hs ht hu bx">Final Thoughts</span></a></li></ol></div></div></nav><script type="application/ld+json">{"@context":"https://schema.org","@id":"https://medium.com/mixpaneleng/how-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5","@type":"SocialMediaPosting","image":["https://miro.medium.com/1*uDXTt2XtbxzDnaGG3SU2fQ.png"],"url":"https://medium.com/mixpaneleng/how-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5","dateCreated":"2023-09-29T01:54:06Z","datePublished":"2023-09-29T01:54:06Z","dateModified":"2025-05-14T20:11:25Z","headline":"How We Migrated from StatsD to Prometheus in One Month","name":"How We Migrated from StatsD to Prometheus in One Month","description":"How We Migrated from StatsD to Prometheus in One Month We recently migrated all of our infrastructure metrics from StatsD to Prometheus and are very pleased with the results. The migration was a ton …","identifier":"fb973af124f5","author":{"@context":"https://schema.org","@id":"https://medium.com/@eddie.bracho","@type":"Person","identifier":"eddie.bracho","name":"Eddie Bracho","url":"https://medium.com/@eddie.bracho"},"creator":{"@context":"https://schema.org","@id":"https://medium.com/@eddie.bracho","@type":"Person","identifier":"eddie.bracho","name":"Eddie Bracho","url":"https://medium.com/@eddie.bracho"},"publisher":{"@context":"https://schema.org","@type":"Organization","@id":"https://medium.com/mixpaneleng","name":"Mixpanel Engineering","description":"Blog is now hosted at https://mixpanel.substack.com","url":"https://medium.com/mixpaneleng","sameAs":["https://twitter.com/mixpanel"],"logo":{"@type":"ImageObject","width":280,"height":280,"url":"https://miro.medium.com/v2/resize:fit:280/1%2AwdBmRrZ2IldG8fNVhER_xg.png"}},"mainEntityOfPage":"https://medium.com/mixpaneleng/how-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5","isAccessibleForFree":true}</script><div class="he dr hw hx g hy hz hc ia"><div class="am ev"><div class="ew ex ey ez fa fb fc cs"><div class="ik e"><div class="he dm ib ic id ie if ig ih ii ij"><div class="am il gh im"><a href="https://medium.com/mixpaneleng?source=post_page---post_publication_sidebar-b5de3443ad1f-fb973af124f5-----------------------------------------" rel="noopener follow"><div class="dm"><img alt="Mixpanel Engineering" class="ei in e ip io" src="https://miro.medium.com/v2/resize:fill:76:76/1*wdBmRrZ2IldG8fNVhER_xg.png" width="38" height="38" loading="lazy"/><div class="in e io ip dr g ed iq"></div></div></a><p class="z b cr u w">Blog is now hosted at <a class="bx x by bz ca ab cb ac ae af ag ah ai ir is it" href="https://mixpanel.substack.com" rel="noopener ugc nofollow">https://mixpanel.substack.com</a></p><p class="z b cr u cv"></p></div></div></div></div></div></div><div class="iu iv iw ix iy e"><div class="am ev"><div class="fc cs iz ja jb jc"></div></div><article><div class="e"><div class="e"><span class="e"></span><section><div><div class="dr ji jj jk jl he"></div><div><div class="am ev"><div class="fc cs iz ja jb jc"><div class="speechify-ignore jm jn am j jo do"><div class="jp jq bp jr dd"><div class="js"><div class="am"><div class="am" aria-haspopup="dialog"><a class="bx x by cf ca ab cb jt ae af ag ah ai aj ak am j ju jv" rel="noopener follow" href="/tag/prometheus?source=post_page---header_tags--fb973af124f5-----------------------------------------" data-discover="true"><span class="z b y u cv">Prometheus</span></a></div></div></div><button aria-label="Follow Prometheus" class="w dn by cf ca ab cb jw ae af ag bm am j jx jy jv jz dp"><svg width="12" height="12" viewBox="0 0 12 12" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true"><path d="M5.96219 1V6M5.96219 11V6M5.96219 6H11M5.96219 6H1" stroke="currentColor" stroke-linecap="round"></path></svg></button></div><div class="jp jq bp jr dd"><div class="js"><div class="am"><div class="am" aria-haspopup="dialog"><a class="bx x by cf ca ab cb jt ae af ag ah ai aj ak am j ju jv" rel="noopener follow" href="/tag/metrics?source=post_page---header_tags--fb973af124f5-----------------------------------------" data-discover="true"><span class="z b y u cv">Metrics</span></a></div></div></div><button aria-label="Follow Metrics" class="w dn by cf ca ab cb jw ae af ag bm am j jx jy jv jz dp"><svg width="12" height="12" viewBox="0 0 12 12" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true"><path d="M5.96219 1V6M5.96219 11V6M5.96219 6H11M5.96219 6H1" stroke="currentColor" stroke-linecap="round"></path></svg></button></div></div></div></div></div><div class="it is ka kb kc"><div class="am ev"><div class="fc cs iz ja jb jc"><div><h1 id="a54e" class="pw-post-title kd ke kf z kg kh ki kj kk kl km kn ko kp kq kr ks kt ku kv kw kx ky kz la lb cv" data-testid="storyTitle">How We Migrated from StatsD to Prometheus in One Month</h1><div><div class="speechify-ignore am lc"><div class="speechify-ignore cs e"><div class="am ld le lf lg lh li lj lk ll lm ln"><div class="am j ln"><div class="am lo"><div><div class="bt"><a rel="noopener follow" href="/@eddie.bracho?source=post_page---byline--fb973af124f5-----------------------------------------" data-discover="true"><div class="e lp lq ee lr ls"><div class="e dm"><img alt="Eddie Bracho" class="e bs ee ef eg ei" src="https://miro.medium.com/v2/resize:fill:64:64/1*FgW6VTtMYvR9jCKWKNcecQ.jpeg" width="32" height="32" loading="lazy" data-testid="authorPhoto"/><div class="lt ee e ef eg dr g lu eh"></div></div></div></a></div></div></div><span class="z b cr u cv"><div class="lv am j"><div class="am j lw"><div class="am j"><div><div class="bt"><span class="z b cr u cv"><a class="bx x by bz ca ab cb ac ae af ag ah ai lx" data-testid="authorName" rel="noopener follow" href="/@eddie.bracho?source=post_page---byline--fb973af124f5-----------------------------------------" data-discover="true">Eddie Bracho</a></span></div></div></div><div class="ly bt"></div></div></div></span></div><div class="am j jo"><span class="z b cr u w"><div class="am ce"><span data-testid="storyReadTime">8 min read</span><div class="lz ma e" aria-hidden="true"><span class="e" aria-hidden="true"><span class="z b cr u w">·</span></span></div><span data-testid="storyPublishDate">Sep 29, 2023</span></div></span></div></div><div class="am lc mb mc md me mf mg mh mi mj mk ml mm mn mo mp mq"><div class="ck cz p dk dl j"><div class="am j"><div class="ng e"><div class="am j nh ni"><div class="pw-multi-vote-icon dm nj nk nl nm"><span data-dd-action-name="Susi presentation tracker clap_footer"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" data-testid="headerClapButton" rel="noopener follow" href="/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fvote%2Fmixpaneleng%2Ffb973af124f5&amp;operation=register&amp;redirect=https%3A%2F%2Fmedium.com%2Fmixpaneleng%2Fhow-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5&amp;user=Eddie+Bracho&amp;userId=fc8168dcb359&amp;source=---header_actions--fb973af124f5---------------------clap_footer--------------------" data-discover="true"><div><div class="bt"><div tabindex="-1" class="cq"><div class="nn af dn no np nq ac nr ns nt nm" role="presentation"><svg width="24" height="24" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-label="clap"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.3702 0.827583L12 3.2822L12.6298 0.827583H11.3702Z"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M13.9159 3.95255L15.4386 1.84138L14.2552 1.45158L13.9159 3.95255Z"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M8.58869 1.84055L10.1114 3.95172L9.77379 1.45076L8.58869 1.84055Z"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M18.523 18.9192C17.6623 19.7799 16.7727 20.1664 15.9029 20.25C16.0419 20.1341 16.1776 20.01 16.3101 19.8776C18.6976 17.4891 19.1702 14.9269 17.7087 12.2547L16.7975 10.6516L16.0072 8.9799C15.7465 8.42046 15.8127 7.99921 16.2099 7.69218C16.2866 7.63312 16.3749 7.59082 16.469 7.56801C16.5631 7.5452 16.6609 7.54239 16.7561 7.55977C17.0392 7.60611 17.3023 7.79149 17.4836 8.06046L19.8472 12.2174C20.8229 13.8411 20.9884 16.4538 18.523 18.9192ZM7.52441 18.4822L3.36992 14.3278C3.25591 14.2118 3.1786 14.0648 3.14765 13.9051C3.1167 13.7454 3.13348 13.5802 3.19591 13.43C3.25833 13.2798 3.36363 13.1513 3.49864 13.0606C3.63365 12.9699 3.79239 12.921 3.95503 12.92C4.16606 12.92 4.37875 13.0011 4.54013 13.1617L6.69765 15.3192C6.76738 15.3812 6.85815 15.4142 6.9514 15.4114C7.04465 15.4087 7.13333 15.3704 7.1993 15.3044C7.26526 15.2385 7.30353 15.1498 7.30627 15.0566C7.30902 14.9633 7.27603 14.8725 7.21406 14.8028L5.05655 12.6445L3.60827 11.1962C3.4532 11.0414 3.36595 10.8314 3.36572 10.6124C3.36549 10.3933 3.45229 10.1831 3.60703 10.028C3.76177 9.87297 3.97177 9.78572 4.19084 9.78549C4.40991 9.78526 4.6201 9.87206 4.77517 10.0268L8.21296 13.4662C8.2468 13.5003 8.28703 13.5273 8.33134 13.5457C8.37566 13.5641 8.42317 13.5736 8.47117 13.5736C8.51916 13.5736 8.56668 13.5641 8.61099 13.5457C8.6553 13.5273 8.69553 13.5003 8.72937 13.4662C8.7634 13.4324 8.79041 13.3922 8.80884 13.3479C8.82726 13.3036 8.83675 13.256 8.83675 13.208C8.83675 13.16 8.82726 13.1125 8.80884 13.0682C8.79041 13.0239 8.7634 12.9837 8.72937 12.9498L5.29323 9.51287L4.3233 8.54377C4.16876 8.38892 4.08197 8.17909 4.08197 7.96032C4.08197 7.74155 4.16876 7.53171 4.3233 7.37687C4.4802 7.22625 4.68926 7.14214 4.90675 7.14214C5.12424 7.14214 5.33331 7.22625 5.4902 7.37687L6.46013 8.34514L9.8971 11.7813C9.93077 11.8156 9.97094 11.8429 10.0153 11.8615C10.0596 11.8801 10.1072 11.8897 10.1553 11.8897C10.2034 11.8897 10.251 11.8801 10.2953 11.8615C10.3397 11.8429 10.3798 11.8156 10.4135 11.7813C10.4818 11.7127 10.5202 11.6199 10.5202 11.5231C10.5202 11.4263 10.4818 11.3334 10.4135 11.2649L6.97655 7.82956C6.89975 7.75312 6.83891 7.66217 6.79756 7.56201C6.75622 7.46185 6.7352 7.35447 6.73572 7.24611C6.73507 7.13769 6.75595 7.03021 6.79715 6.92991C6.83835 6.82962 6.89904 6.7385 6.97572 6.66183C7.13084 6.50719 7.34095 6.42035 7.55999 6.42035C7.77903 6.42035 7.98914 6.50719 8.14427 6.66183L13.9308 12.4491C13.9922 12.5076 14.0721 12.5429 14.1567 12.549C14.2413 12.5552 14.3254 12.5318 14.3947 12.4829C14.464 12.434 14.5142 12.3626 14.5367 12.2809C14.5593 12.1991 14.5528 12.112 14.5183 12.0345L13.4011 9.67094C13.1404 9.11149 13.2066 8.69025 13.6047 8.38239C13.6817 8.32396 13.77 8.28204 13.864 8.25926C13.958 8.23648 14.0556 8.23331 14.1509 8.24997C14.4339 8.29632 14.6963 8.4817 14.8783 8.75149L17.0714 12.6114C18.3732 14.9915 17.9545 17.2003 15.7936 19.3611C14.6383 20.5173 13.1917 20.9882 11.6036 20.7283C10.1859 20.4916 8.73765 19.6955 7.52524 18.4822H7.52441ZM10.7503 5.9708L12.8706 8.09108C12.4593 8.59342 12.4047 9.26128 12.7423 9.98128L12.9616 10.4464L9.43861 6.92335C9.37757 6.81002 9.34438 6.68378 9.34179 6.55508C9.34179 6.33411 9.42786 6.12721 9.58344 5.9708C9.74082 5.82112 9.9497 5.73766 10.1669 5.73766C10.3841 5.73766 10.593 5.82112 10.7503 5.9708ZM18.1051 7.67563C17.7948 7.21466 17.3586 6.91756 16.8753 6.83894C16.6812 6.80632 16.4825 6.81388 16.2915 6.86114C16.1005 6.9084 15.9212 6.99436 15.7647 7.1137C15.4535 7.3537 15.2599 7.65659 15.1754 7.99508C14.9266 7.75301 14.611 7.59102 14.2692 7.52997C13.9911 7.4859 13.7061 7.52267 13.4483 7.6359L11.2668 5.45439C10.9747 5.16339 10.5792 5.00001 10.1669 5.00001C9.7546 5.00001 9.35911 5.16339 9.06703 5.45439C8.87539 5.64887 8.73887 5.89078 8.67144 6.15535C8.5271 6.00929 8.35536 5.89315 8.16606 5.81358C7.97675 5.73401 7.77361 5.69257 7.56826 5.69165C7.36292 5.69073 7.15941 5.73034 6.9694 5.8082C6.77938 5.88607 6.60661 6.00066 6.46096 6.14542C6.25382 6.35175 6.10947 6.6126 6.04468 6.8977C5.42068 6.2737 4.39613 6.2737 3.80772 6.86128C3.51632 7.15315 3.35266 7.54872 3.35266 7.96114C3.35266 8.37357 3.51632 8.76914 3.80772 9.06101C3.56937 9.16032 3.30703 9.29935 3.09268 9.5137C2.80164 9.80592 2.63823 10.2015 2.63823 10.614C2.63823 11.0264 2.80164 11.422 3.09268 11.7142L3.6091 12.229C3.33829 12.291 3.08902 12.4243 2.88721 12.6152C2.6854 12.8061 2.53842 13.0476 2.46154 13.3146C2.38467 13.5815 2.38071 13.8642 2.45008 14.1332C2.51945 14.4022 2.65961 14.6477 2.85599 14.8442L7.00965 18.9987C8.32965 20.3178 9.91861 21.1876 11.4852 21.4475C11.806 21.5011 12.1307 21.5282 12.456 21.5286C13.3051 21.5286 14.1087 21.33 14.8494 20.9493C15.0803 20.9824 15.3087 21.0031 15.5346 21.0031C16.8008 21.0031 17.9917 20.4834 19.0403 19.4356C21.8028 16.6731 21.5917 13.7021 20.4786 11.85L18.1051 7.67645V7.67563Z"></path></svg></div></div></div></div></a></span></div><div class="pw-multi-vote-count e nu nv nw nx ny nz oa"><p class="z b y u w"><span class="ob">--</span></p></div></div></div><div class="oc dy e"><div><div class="bt"><div tabindex="-1" class="cq"><button class="af nn go oe am j dn np of" aria-label="responses"><svg width="24" height="24" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" class="lb"><path d="M18.0064 16.8029C19.5393 15.347 20.24 13.4782 20.24 11.4818C20.24 7.35659 16.7088 4 12.1914 4C7.67395 4 4 7.35659 4 11.4818C4 15.6077 7.67395 18.9643 12.1914 18.9643C13.0081 18.9643 13.8134 18.8527 14.5839 18.6374C14.815 18.8368 15.064 19.0281 15.3284 19.1956C16.3876 19.8892 17.5312 20.24 18.7266 20.24C18.9505 20.232 19.1273 20.1284 19.2132 19.953C19.2554 19.8711 19.2738 19.7795 19.2664 19.688C19.2589 19.5966 19.2259 19.5089 19.1711 19.4347C18.6763 18.765 18.326 18.0714 18.1313 17.3777C18.0754 17.1816 18.0337 16.9817 18.0064 16.7798V16.8029ZM14.884 17.8585L14.8167 17.6353L14.502 17.7309C13.7529 17.9559 12.9738 18.0698 12.1906 18.069C8.16787 18.069 4.89943 15.1142 4.89943 11.4818C4.89943 7.84932 8.16787 4.89376 12.1906 4.89376C16.2051 4.89376 19.3033 7.85171 19.3033 11.4874C19.3033 13.2813 18.695 14.9556 17.2757 16.2073L17.081 16.3747V16.6299C17.081 16.6857 17.0811 16.7814 17.0973 16.9249C17.1216 17.1561 17.1784 17.4032 17.2514 17.6584C17.4055 18.2165 17.6488 18.7746 17.9732 19.3167C17.2189 19.2051 16.489 18.9261 15.8077 18.4716C15.532 18.2962 15.094 18.0889 14.8669 17.8816L14.884 17.8585Z"></path></svg><p class="z b y u w"><span class="pw-responses-count od lb">1</span></p></button></div></div></div></div><div class="am j w"><div><div class="bt"><div tabindex="-1" class="cq"><span data-dd-action-name="Susi presentation tracker repost_header"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" rel="noopener follow" href="/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Frepost%2Fp%2Ffb973af124f5&amp;operation=register&amp;redirect=https%3A%2F%2Fmedium.com%2Fmixpaneleng%2Fhow-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5&amp;user=Eddie+Bracho&amp;userId=fc8168dcb359&amp;source=---header_actions--fb973af124f5---------------------repost_header--------------------" data-discover="true"><span class="nn dz bx af go nr am j ns nt nm ac og oh" aria-disabled="false" aria-label="Repost"><div class="dg df e dm"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none"><path d="M3 8.72727L5.64706 6M5.64706 6L8.29412 8.72727M5.64706 6V14.1818V15.2475C5.64706 15.9633 5.92589 16.6509 6.42442 17.1646C6.94277 17.6986 7.6553 18 8.39955 18H12" stroke="currentColor" stroke-linecap="round"></path><path d="M21 15.2727L18.3529 18M18.3529 18L15.7059 15.2727M18.3529 18V9.81818V8.75249C18.3529 8.03671 18.0741 7.34908 17.5756 6.83545C17.0572 6.30139 16.3447 6 15.6004 6L12 6" stroke="currentColor" stroke-linecap="round"></path></svg></div></span></a></span></div></div></div><div class="od e"></div></div></div></div><div class="am j mr ms mt mu mv mw mx my mz na nb nc nd ne nf"><div class="gs cz da r s"></div><div class="ck cz"><div><div class="bt"><div tabindex="-1" class="cq"><span data-dd-action-name="Susi presentation tracker bookmark_footer"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" data-testid="headerBookmarkButton" rel="noopener follow" href="/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fbookmark%2Fp%2Ffb973af124f5&amp;operation=register&amp;redirect=https%3A%2F%2Fmedium.com%2Fmixpaneleng%2Fhow-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5&amp;source=---header_actions--fb973af124f5---------------------bookmark_footer--------------------" data-discover="true"><span class=""><svg width="25" height="25" viewBox="0 0 25 25" fill="none" xmlns="http://www.w3.org/2000/svg" class="w oi" aria-label="Add to list bookmark button"><path d="M18 2.5C18 2.22386 18.2239 2 18.5 2C18.7762 2 19 2.22386 19 2.5V5H21.5C21.7761 5 22 5.22386 22 5.5C22 5.77614 21.7761 6 21.5 6H19V8.5C19 8.77614 18.7762 9 18.5 9C18.2239 9 18 8.77614 18 8.5V6H15.5C15.2239 6 15 5.77614 15 5.5C15 5.22386 15.2239 5 15.5 5H18V2.5Z" fill="currentColor"></path><path d="M7 7C7 6.44772 7.44772 6 8 6H11.5C11.7761 6 12 5.77614 12 5.5C12 5.22386 11.7761 5 11.5 5H8C6.89543 5 6 5.89543 6 7V21C6 21.1905 6.10823 21.3644 6.27912 21.4486C6.45002 21.5327 6.65387 21.5125 6.80486 21.3963L12.5 17L18.1951 21.3963C18.3461 21.5125 18.55 21.5327 18.7209 21.4486C18.8918 21.3644 19 21.1905 19 21V12.5C19 12.2239 18.7761 12 18.5 12C18.2239 12 18 12.2239 18 12.5V19.9846L12.8049 15.9729C12.6251 15.8346 12.3749 15.8346 12.1951 15.9729L7 19.9846V7Z" fill="currentColor"></path></svg></span></a></span></div></div></div></div><div class="bs jp il"><div class="e ce"><div class="am ev"><div class="oj ok ol om on er fc cs"><div class="am"><span data-dd-action-name="Susi presentation tracker post_audio_button"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" rel="noopener follow" href="/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2Fplans%3Fdimension%3Dpost_audio_button%26postId%3Dfb973af124f5&amp;operation=register&amp;redirect=https%3A%2F%2Fmedium.com%2Fmixpaneleng%2Fhow-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5&amp;source=---header_actions--fb973af124f5---------------------post_audio_button--------------------" data-discover="true"><div><div class="bt"><div tabindex="-1" class="cq"><button aria-label="Listen" data-testid="audioPlayButton" class="bx dn by bz ca ab cb oo ae af ag bm op oq or os ot ou ov ow l ox oy oz pa pb pc pd n pe pf pg"><svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M3 12C3 7.02944 7.02944 3 12 3C16.9706 3 21 7.02944 21 12C21 16.9706 16.9706 21 12 21C7.02944 21 3 16.9706 3 12ZM12 2C6.47715 2 2 6.47715 2 12C2 17.5228 6.47715 22 12 22C17.5228 22 22 17.5228 22 12C22 6.47715 17.5228 2 12 2ZM15.376 12.416L10.7773 15.4818C10.4451 15.7033 10 15.4651 10 15.0657V8.93426C10 8.53491 10.4451 8.29672 10.7774 8.51823L15.376 11.584C15.6728 11.7819 15.6728 12.2181 15.376 12.416Z" fill="currentColor"></path></svg><div class="da r s"><p class="z b cr u w">Listen</p></div></button></div></div></div></a></span></div></div></div></div></div><div class="bt" aria-describedby="postFooterSocialMenu" aria-labelledby="postFooterSocialMenu" aria-haspopup="dialog"><div><div class="bt"><div tabindex="-1" class="cq"><button aria-controls="postFooterSocialMenu" aria-expanded="false" aria-label="Share Post" data-testid="headerSocialShareButton" class="bx dn by bz ca ab cb oo ae af ag bm op oq or os ot ou ov ow l ox oy oz pa pb pc pd n pe pf pg"><svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M15.2176 4.93118C15.1883 4.98289 15.1484 5.0278 15.1004 5.06293L15.1118 5.06859C15.024 5.12481 14.9269 5.15261 14.8204 5.14293C14.7004 5.13293 14.6004 5.08293 14.5204 5.01293L12.5004 2.99293V10.0629C12.5004 10.3429 12.2704 10.5629 12.0004 10.5629C11.7304 10.5629 11.5004 10.3429 11.5004 10.0629V3.02293L9.50042 5.02293C9.42474 5.09456 9.32654 5.13771 9.22259 5.14501C9.11863 5.1523 9.01537 5.12329 8.93042 5.06293H8.91042C8.82042 4.99293 8.75042 4.88293 8.75042 4.76293C8.74294 4.70553 8.74801 4.64719 8.76528 4.59195C8.78254 4.5367 8.81159 4.48586 8.85042 4.44293L11.6504 1.64293C11.7439 1.55132 11.8695 1.5 12.0004 1.5C12.1313 1.5 12.257 1.55132 12.3504 1.64293L15.1504 4.43293C15.1931 4.47602 15.2258 4.52782 15.2466 4.58479C15.2673 4.64175 15.2754 4.70252 15.2704 4.76293C15.2649 4.82212 15.2469 4.87946 15.2176 4.93118ZM15.1118 5.06859L15.1204 5.07293V5.06293C15.1175 5.06485 15.1147 5.06674 15.1118 5.06859ZM16 7.06293H17.5C18.0304 7.06293 18.5391 7.27364 18.9142 7.64871C19.2893 8.02379 19.5 8.53249 19.5 9.06293V19.0629C19.5 19.5934 19.2893 20.1021 18.9142 20.4771C18.5391 20.8522 18.0304 21.0629 17.5 21.0629H6.5C5.4 21.0629 4.5 20.1629 4.5 19.0629V9.06293C4.5 8.53249 4.71071 8.02379 5.08579 7.64871C5.46086 7.27364 5.96957 7.06293 6.5 7.06293H8C8.13188 7.06481 8.25769 7.11872 8.35 7.21293C8.44421 7.30524 8.49812 7.43104 8.5 7.56293C8.49812 7.69481 8.44421 7.82061 8.35 7.91293C8.25769 8.00713 8.13188 8.06105 8 8.06293H6.4C5.9 8.06293 5.5 8.46293 5.5 8.96293V19.1629C5.5 19.4016 5.59482 19.6305 5.7636 19.7993C5.93239 19.9681 6.16131 20.0629 6.4 20.0629H17.6C18.1 20.0629 18.5 19.6629 18.5 19.1629V8.96293C18.5 8.46293 18.1 8.06293 17.6 8.06293H16C15.8674 8.06293 15.7402 8.01025 15.6464 7.91648C15.5527 7.82271 15.5 7.69554 15.5 7.56293C15.5 7.43032 15.5527 7.30314 15.6464 7.20937C15.7402 7.11561 15.8674 7.06293 16 7.06293Z" fill="currentColor"></path></svg><div class="da r s"><p class="z b cr u w">Share</p></div></button></div></div></div></div></div></div></div></div></div></div><p id="ea28" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">We recently migrated all of our infrastructure metrics from StatsD to Prometheus and are very pleased with the results. The migration was a ton of work and we learned a lot along the way. This post aims to shed some light on why we migrated to Prometheus, as well as outline some of the technical challenges we faced during the process.</p><h2 id="da62" class="qc qd kf z qe qf qg qh fk qi qj qk fm ql qm qn qo qp qq qr qs qt qu qv qw qx cv">Background</h2><p id="cd86" class="pw-post-body-paragraph ph pi kf pj b pk qy pm pn po qz pq pr fn ra pt pu fq rb pw px ft rc pz qa qb it cv">Metrics are such a vital, commonplace component in today’s distributed system that it’s easy to forget that popular open source metrics protocols only started appearing in the last decade. StatsD came out of <a class="bx ir" href="https://www.etsy.com/codeascraft/measure-anything-measure-everything/" rel="noopener ugc nofollow" target="_blank">Etsy back in 2011</a> with Prometheus following shortly afterwards from <a class="bx ir" href="https://developers.soundcloud.com/blog/prometheus-monitoring-at-soundcloud" rel="noopener ugc nofollow" target="_blank">SoundCloud in 2012</a>. Both collect the same kinds of measurements (counters, gauges, and sample distributions), but how they’re architected differs significantly in ways that reflect the priorities of distributed systems they came from. For example, StatsD’s push-based model makes it dead-simple to collect metrics from any process that can send a UDP packet, whereas Prometheus’s decentralized pull-based model makes it easier to scale metrics collection.</p><p id="3fc2" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">Mixpanel was not slow to jump on the metrics bandwagon — we are an analytics company after all. We first began collecting metrics through StatsD in 2013 (just 2 years after our landmark <a class="bx ir" href="https://engineering.mixpanel.com/why-we-moved-off-the-cloud-907daac729a0" rel="noopener ugc nofollow">Why We Moved Off the Cloud</a> blog post 😛). For a fast-moving, multi-language tech stack running on a fixed number of machines, StatsD made a lot of sense. Scaling metrics was not even close to being a question in anyone’s mind, and client libraries were incredibly easy to implement and reason about.</p><p id="f489" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">Fast-forward several years and Mixpanel’s infrastructure had taken on a completely different shape. We moved back onto cloud (no landmark blog post this time) allowing us much more freedom to decouple and scale our performance-critical services. Our services now ran on Kubernetes with individual pods collecting tens-of-thousands of metric samples per second. Engineers also evolved much stricter expectations for the amount of dropped metric samples (none). To deal with these new requirements, we converted our StatsD sidecar connections to TCP and moved metric sample aggregation to the client process (which reduced contention induced by large flushes to StatsD connections).</p><p id="90b1" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">At some point we had outgrown traditional StatsD and accidentally engineered our own proprietary metrics architecture. So when the time came for us to change metrics vendors, we realized we had a several good reasons to move into the Prometheus ecosystem:</p><h2 id="05df" class="qc qd kf z qe qf qg qh fk qi qj qk fm ql qm qn qo qp qq qr qs qt qu qv qw qx cv">Performance</h2><p id="a037" class="pw-post-body-paragraph ph pi kf pj b pk qy pm pn po qz pq pr fn ra pt pu fq rb pw px ft rc pz qa qb it cv">A lot of the performance work we put into our internal metrics libraries mirrors closely with Prometheus’s architecture decisions. Prometheus’s pull-based model eliminates the chance for dropped metric samples, and recording a sample boils down to a single atomic operation in the client process.</p><h2 id="0a8c" class="qc qd kf z qe qf qg qh fk qi qj qk fm ql qm qn qo qp qq qr qs qt qu qv qw qx cv">Reduce Vendor Lock-in</h2><p id="4a08" class="pw-post-body-paragraph ph pi kf pj b pk qy pm pn po qz pq pr fn ra pt pu fq rb pw px ft rc pz qa qb it cv">PromQL and Grafana are open standards meaning that the work we put into migrating our collectors, dashboards and alert queries could follow us to a variety of different platforms. This would reduce vendor lock-in and give us a ton of flexibility to trial new metrics platforms.</p><h2 id="1aa4" class="qc qd kf z qe qf qg qh fk qi qj qk fm ql qm qn qo qp qq qr qs qt qu qv qw qx cv">Ecosystem</h2><p id="6dc6" class="pw-post-body-paragraph ph pi kf pj b pk qy pm pn po qz pq pr fn ra pt pu fq rb pw px ft rc pz qa qb it cv">We love metrics but we’re not an infrastructure metrics company. By investing in the Prometheus ecosystem we can inherit all the updates and improvements that the community produces (ex: exemplars, native histograms) and free our attention to focus on other things that are important to our business.</p><h2 id="f66d" class="qc qd kf z qe qf qg qh fk qi qj qk fm ql qm qn qo qp qq qr qs qt qu qv qw qx cv">Challenges</h2><p id="e91f" class="pw-post-body-paragraph ph pi kf pj b pk qy pm pn po qz pq pr fn ra pt pu fq rb pw px ft rc pz qa qb it cv">The challenges of this migration can be placed into three main categories: metrics collection, metrics reshaping, and query rewrites.</p><h2 id="d4a5" class="qc qd kf z qe qf qg qh fk qi qj qk fm ql qm qn qo qp qq qr qs qt qu qv qw qx cv">Metrics Collection</h2><p id="07b7" class="pw-post-body-paragraph ph pi kf pj b pk qy pm pn po qz pq pr fn ra pt pu fq rb pw px ft rc pz qa qb it cv">The first problem we had to deal with was how to ingest our existing metric samples into a Prometheus metrics stack. We imposed a few requirements for a solution:</p><ol class=""><li id="65b6" class="ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb rd re rf cv">Metrics must be dual emitted in StatsD and Prometheus protocols during the migration period.</li><li id="14e5" class="ph pi kf pj b pk rg pm pn po rh pq pr fn ri pt pu fq rj pw px ft rk pz qa qb rd re rf cv">Services must not be required to re-instrument application code.</li><li id="86a5" class="ph pi kf pj b pk rg pm pn po rh pq pr fn ri pt pu fq rj pw px ft rk pz qa qb rd re rf cv">No client-side performance degradation.</li><li id="1dca" class="ph pi kf pj b pk rg pm pn po rh pq pr fn ri pt pu fq rj pw px ft rk pz qa qb rd re rf cv">Access to standard Prometheus distribution metric types (summaries, histograms, native histograms).</li></ol><p id="a52f" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">There’s an existing tool called <a class="bx ir" href="https://github.com/prometheus/statsd_exporter" rel="noopener ugc nofollow" target="_blank">statsd_exporter</a> maintained by the Prometheus organization that functions great for services emitting standard StatsD protocol. It works by relaying StatsD traffic from your application to your normal StatsD server while also converting metrics to Prometheus protocol and publishing them on a <em class="rl">/metrics</em> HTTP endpoint.</p><figure class="rp rq rr rs rt ru rm rn paragraph-image"><div role="button" tabindex="0" class="rv rw dm rx cs ry"><span class="dr ds dt ac du dv dw dd dx speechify-ignore">Press enter or click to view image in full size</span><div class="rm rn ro"><picture><source srcSet="https://miro.medium.com/v2/resize:fit:640/format:webp/1*nAsJ1qOnn0c7P993BLivTQ.png 640w, https://miro.medium.com/v2/resize:fit:720/format:webp/1*nAsJ1qOnn0c7P993BLivTQ.png 720w, https://miro.medium.com/v2/resize:fit:750/format:webp/1*nAsJ1qOnn0c7P993BLivTQ.png 750w, https://miro.medium.com/v2/resize:fit:786/format:webp/1*nAsJ1qOnn0c7P993BLivTQ.png 786w, https://miro.medium.com/v2/resize:fit:828/format:webp/1*nAsJ1qOnn0c7P993BLivTQ.png 828w, https://miro.medium.com/v2/resize:fit:1100/format:webp/1*nAsJ1qOnn0c7P993BLivTQ.png 1100w, https://miro.medium.com/v2/resize:fit:1298/format:webp/1*nAsJ1qOnn0c7P993BLivTQ.png 1298w" sizes="(min-resolution: 4dppx) and (max-width: 700px) 50vw, (-webkit-min-device-pixel-ratio: 4) and (max-width: 700px) 50vw, (min-resolution: 3dppx) and (max-width: 700px) 67vw, (-webkit-min-device-pixel-ratio: 3) and (max-width: 700px) 65vw, (min-resolution: 2.5dppx) and (max-width: 700px) 80vw, (-webkit-min-device-pixel-ratio: 2.5) and (max-width: 700px) 80vw, (min-resolution: 2dppx) and (max-width: 700px) 100vw, (-webkit-min-device-pixel-ratio: 2) and (max-width: 700px) 100vw, 649px" type="image/webp"/><source data-testid="og" srcSet="https://miro.medium.com/v2/resize:fit:640/1*nAsJ1qOnn0c7P993BLivTQ.png 640w, https://miro.medium.com/v2/resize:fit:720/1*nAsJ1qOnn0c7P993BLivTQ.png 720w, https://miro.medium.com/v2/resize:fit:750/1*nAsJ1qOnn0c7P993BLivTQ.png 750w, https://miro.medium.com/v2/resize:fit:786/1*nAsJ1qOnn0c7P993BLivTQ.png 786w, https://miro.medium.com/v2/resize:fit:828/1*nAsJ1qOnn0c7P993BLivTQ.png 828w, https://miro.medium.com/v2/resize:fit:1100/1*nAsJ1qOnn0c7P993BLivTQ.png 1100w, https://miro.medium.com/v2/resize:fit:1298/1*nAsJ1qOnn0c7P993BLivTQ.png 1298w" sizes="(min-resolution: 4dppx) and (max-width: 700px) 50vw, (-webkit-min-device-pixel-ratio: 4) and (max-width: 700px) 50vw, (min-resolution: 3dppx) and (max-width: 700px) 67vw, (-webkit-min-device-pixel-ratio: 3) and (max-width: 700px) 65vw, (min-resolution: 2.5dppx) and (max-width: 700px) 80vw, (-webkit-min-device-pixel-ratio: 2.5) and (max-width: 700px) 80vw, (min-resolution: 2dppx) and (max-width: 700px) 100vw, (-webkit-min-device-pixel-ratio: 2) and (max-width: 700px) 100vw, 649px"/><img alt="" class="cs er rz c" width="649" height="167" loading="eager" role="presentation"/></picture></div></div><figcaption class="sa bv sb rm rn sc sd z b cr u w">Exporter sidecar architecture</figcaption></figure><p id="975f" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">Unfortunately <em class="rl">statsd_exporter</em> didn’t quite fit the bill for us since many of our services aggregate metric samples in-process, meaning we would not have access to Prometheus’s distribution metric types. Whatever solution we came up with would need to be embedded in the client-side process.</p><p id="e147" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">So that’s exactly what we did. All our performance-critical services are written in Go, and fortunately <strong class="pj kg"><em class="rl">statsd_exporter</em></strong> is open source and also written in Go. Our solution was to import <strong class="pj kg"><em class="rl">statsd_exporter</em></strong> into our internal Go metrics SDK and have it process metric samples directly in memory. This approach also had some additional benefits:</p><ol class=""><li id="d463" class="ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb rd re rf cv">Services could seamlessly start converting to the official <a class="bx ir" href="https://github.com/prometheus/client_golang" rel="noopener ugc nofollow" target="_blank">Prometheus Go SDK</a> at their own pace.</li><li id="4636" class="ph pi kf pj b pk rg pm pn po rh pq pr fn ri pt pu fq rj pw px ft rk pz qa qb rd re rf cv">We would have a sidecar-less collection system after the migration.</li></ol><p id="578a" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">After a bit of performance-tuning work we had solution that met all our requirements!</p><figure class="rp rq rr rs rt ru rm rn paragraph-image"><div class="rm rn se"><picture><source srcSet="https://miro.medium.com/v2/resize:fit:640/format:webp/1*uDXTt2XtbxzDnaGG3SU2fQ.png 640w, https://miro.medium.com/v2/resize:fit:720/format:webp/1*uDXTt2XtbxzDnaGG3SU2fQ.png 720w, https://miro.medium.com/v2/resize:fit:750/format:webp/1*uDXTt2XtbxzDnaGG3SU2fQ.png 750w, https://miro.medium.com/v2/resize:fit:786/format:webp/1*uDXTt2XtbxzDnaGG3SU2fQ.png 786w, https://miro.medium.com/v2/resize:fit:828/format:webp/1*uDXTt2XtbxzDnaGG3SU2fQ.png 828w, https://miro.medium.com/v2/resize:fit:1100/format:webp/1*uDXTt2XtbxzDnaGG3SU2fQ.png 1100w, https://miro.medium.com/v2/resize:fit:1306/format:webp/1*uDXTt2XtbxzDnaGG3SU2fQ.png 1306w" sizes="auto, (min-resolution: 4dppx) and (max-width: 700px) 50vw, (-webkit-min-device-pixel-ratio: 4) and (max-width: 700px) 50vw, (min-resolution: 3dppx) and (max-width: 700px) 67vw, (-webkit-min-device-pixel-ratio: 3) and (max-width: 700px) 65vw, (min-resolution: 2.5dppx) and (max-width: 700px) 80vw, (-webkit-min-device-pixel-ratio: 2.5) and (max-width: 700px) 80vw, (min-resolution: 2dppx) and (max-width: 700px) 100vw, (-webkit-min-device-pixel-ratio: 2) and (max-width: 700px) 100vw, 653px" type="image/webp"/><source data-testid="og" srcSet="https://miro.medium.com/v2/resize:fit:640/1*uDXTt2XtbxzDnaGG3SU2fQ.png 640w, https://miro.medium.com/v2/resize:fit:720/1*uDXTt2XtbxzDnaGG3SU2fQ.png 720w, https://miro.medium.com/v2/resize:fit:750/1*uDXTt2XtbxzDnaGG3SU2fQ.png 750w, https://miro.medium.com/v2/resize:fit:786/1*uDXTt2XtbxzDnaGG3SU2fQ.png 786w, https://miro.medium.com/v2/resize:fit:828/1*uDXTt2XtbxzDnaGG3SU2fQ.png 828w, https://miro.medium.com/v2/resize:fit:1100/1*uDXTt2XtbxzDnaGG3SU2fQ.png 1100w, https://miro.medium.com/v2/resize:fit:1306/1*uDXTt2XtbxzDnaGG3SU2fQ.png 1306w" sizes="auto, (min-resolution: 4dppx) and (max-width: 700px) 50vw, (-webkit-min-device-pixel-ratio: 4) and (max-width: 700px) 50vw, (min-resolution: 3dppx) and (max-width: 700px) 67vw, (-webkit-min-device-pixel-ratio: 3) and (max-width: 700px) 65vw, (min-resolution: 2.5dppx) and (max-width: 700px) 80vw, (-webkit-min-device-pixel-ratio: 2.5) and (max-width: 700px) 80vw, (min-resolution: 2dppx) and (max-width: 700px) 100vw, (-webkit-min-device-pixel-ratio: 2) and (max-width: 700px) 100vw, 653px"/><img alt="" class="cs er rz c" width="653" height="299" loading="lazy" role="presentation"/></picture></div><figcaption class="sa bv sb rm rn sc sd z b cr u w">Embedded exporter architecture</figcaption></figure><h2 id="3c50" class="qc qd kf z qe qf qg qh fk qi qj qk fm ql qm qn qo qp qq qr qs qt qu qv qw qx cv">Takeaways</h2><ul class=""><li id="781c" class="ph pi kf pj b pk qy pm pn po qz pq pr fn ra pt pu fq rb pw px ft rc pz qa qb sf re rf cv">Metrics collection can be solved ahead of time. It should be dead-simple for service owners to instrument their service for Prometheus.</li><li id="cc13" class="ph pi kf pj b pk rg pm pn po rh pq pr fn ri pt pu fq rj pw px ft rk pz qa qb sf re rf cv"><em class="rl">statsd_exporter</em> has already solved this problem, so leverage it however you can.</li><li id="5518" class="ph pi kf pj b pk rg pm pn po rh pq pr fn ri pt pu fq rj pw px ft rk pz qa qb sf re rf cv">Ensure there’s a path for services to start using native Prometheus client SDKs going forward.</li></ul><h2 id="fff7" class="qc qd kf z qe qf qg qh fk qi qj qk fm ql qm qn qo qp qq qr qs qt qu qv qw qx cv">Metrics Reshaping</h2><p id="a67b" class="pw-post-body-paragraph ph pi kf pj b pk qy pm pn po qz pq pr fn ra pt pu fq rb pw px ft rc pz qa qb it cv">The next challenge we faced was the question of how to <strong class="pj kg"><em class="rl">reshape</em></strong> our metrics. In this context, the shape of a metric refers to the name, labels, and units of a given metric. While the underlying <a class="bx ir" href="https://prometheus.io/docs/concepts/data_model/" rel="noopener ugc nofollow" target="_blank">data models</a> of modern StatsD and Prometheus are virtually identical, they have much different conventions which can have frustrating implications for query ergonomics.</p><figure class="rp rq rr rs rt ru rm rn paragraph-image"><div class="rm rn sg"><picture><source srcSet="https://miro.medium.com/v2/resize:fit:640/format:webp/1*w4N3ZMBvcU5DLbTtwoMo4w.png 640w, https://miro.medium.com/v2/resize:fit:720/format:webp/1*w4N3ZMBvcU5DLbTtwoMo4w.png 720w, https://miro.medium.com/v2/resize:fit:750/format:webp/1*w4N3ZMBvcU5DLbTtwoMo4w.png 750w, https://miro.medium.com/v2/resize:fit:786/format:webp/1*w4N3ZMBvcU5DLbTtwoMo4w.png 786w, https://miro.medium.com/v2/resize:fit:828/format:webp/1*w4N3ZMBvcU5DLbTtwoMo4w.png 828w, https://miro.medium.com/v2/resize:fit:1100/format:webp/1*w4N3ZMBvcU5DLbTtwoMo4w.png 1100w, https://miro.medium.com/v2/resize:fit:1400/format:webp/1*w4N3ZMBvcU5DLbTtwoMo4w.png 1400w" sizes="auto, (min-resolution: 4dppx) and (max-width: 700px) 50vw, (-webkit-min-device-pixel-ratio: 4) and (max-width: 700px) 50vw, (min-resolution: 3dppx) and (max-width: 700px) 67vw, (-webkit-min-device-pixel-ratio: 3) and (max-width: 700px) 65vw, (min-resolution: 2.5dppx) and (max-width: 700px) 80vw, (-webkit-min-device-pixel-ratio: 2.5) and (max-width: 700px) 80vw, (min-resolution: 2dppx) and (max-width: 700px) 100vw, (-webkit-min-device-pixel-ratio: 2) and (max-width: 700px) 100vw, 700px" type="image/webp"/><source data-testid="og" srcSet="https://miro.medium.com/v2/resize:fit:640/1*w4N3ZMBvcU5DLbTtwoMo4w.png 640w, https://miro.medium.com/v2/resize:fit:720/1*w4N3ZMBvcU5DLbTtwoMo4w.png 720w, https://miro.medium.com/v2/resize:fit:750/1*w4N3ZMBvcU5DLbTtwoMo4w.png 750w, https://miro.medium.com/v2/resize:fit:786/1*w4N3ZMBvcU5DLbTtwoMo4w.png 786w, https://miro.medium.com/v2/resize:fit:828/1*w4N3ZMBvcU5DLbTtwoMo4w.png 828w, https://miro.medium.com/v2/resize:fit:1100/1*w4N3ZMBvcU5DLbTtwoMo4w.png 1100w, https://miro.medium.com/v2/resize:fit:1400/1*w4N3ZMBvcU5DLbTtwoMo4w.png 1400w" sizes="auto, (min-resolution: 4dppx) and (max-width: 700px) 50vw, (-webkit-min-device-pixel-ratio: 4) and (max-width: 700px) 50vw, (min-resolution: 3dppx) and (max-width: 700px) 67vw, (-webkit-min-device-pixel-ratio: 3) and (max-width: 700px) 65vw, (min-resolution: 2.5dppx) and (max-width: 700px) 80vw, (-webkit-min-device-pixel-ratio: 2.5) and (max-width: 700px) 80vw, (min-resolution: 2dppx) and (max-width: 700px) 100vw, (-webkit-min-device-pixel-ratio: 2) and (max-width: 700px) 100vw, 700px"/><img alt="" class="cs er rz c" width="700" height="300" loading="lazy" role="presentation"/></picture></div></figure><p id="6b8a" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">StatsD originally did not support tags and would encode every metric dimension in the metric name. While most implementations now support tags, the dot-namespace convention is still widely used and many metrics use a combination of name prefixes and tags to encode dimensions.</p><p id="4fe0" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">Prometheus on the other hand strongly discourages encoding dimensions in the metric name. In fact, certain PromQL queries will <a class="bx ir" href="https://www.robustperception.io/whats-in-a-__name__/" rel="noopener ugc nofollow" target="_blank">completely break</a> when you try to aggregate across dimensions encoded in the metric name, such as the following success rate query:</p><pre class="rp rq rr rs rt sh si sj jf sk cn cv"><span id="9949" class="sl qd kf si b cr sm sn e so sp">sum(rate(<br/> arb_lqs_grpc_server_LQS_Execute_OK[5m]<br/>))<br/>/<br/>sum(rate(<br/> {__name__=~&quot;arb_lqs_grpc_server_LQS_Execute_.+&quot;}[5m]<br/>))<br/><br/>// Error expanding series: vector cannot contain metrics with the same labelset</span></pre><p id="8805" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">In our initial experiments we quickly realized we had some very important <a class="bx ir" rel="noopener" href="/the-mission/jeff-bezos-says-successful-people-make-these-two-types-of-decisions-c80b333e5d95" data-discover="true">type-1 decisions</a> to make with regards to metric shaping. While it is possible write a query that hacks around this limitation, we absolutely did not want to normalize it for our engineers for years to come. We needed to find a way to move dimensionality entirely into labels, again without requiring engineers to re-instrument application code.</p><p id="bf5e" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">The first piece of low-hanging fruit was to replace our RPC metrics interceptors with standard open-source Prometheus interceptors, such as the one provided by <a class="bx ir" href="https://github.com/grpc-ecosystem/go-grpc-middleware#observability" rel="noopener ugc nofollow" target="_blank">grpc-ecosystem/go-grpc-middleware</a>:</p><pre class="rp rq rr rs rt sh si sj jf sk cn cv"><span id="5ed2" class="sl qd kf si b cr sm sn e so sp">sum(rate(<br/> grpc_server_handled_total{<br/> grpc_service=&quot;LQS&quot;,<br/> grpc_method=&quot;Execute&quot;,<br/> grpc_code=&quot;OK&quot;<br/> }[5m]<br/>))<br/>/<br/>sum(rate(<br/> grpc_server_handled_total{<br/> grpc_service=&quot;LQS&quot;,<br/> grpc_method=&quot;Execute&quot;<br/> }[5m]<br/>))<br/><br/>// No error!</span></pre><p id="52fe" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">This one change ended up having a huge impact, since RPC metrics are by far the most queried metrics.</p><p id="f827" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">For other metrics, <strong class="pj kg"><em class="rl">statsd_exporter</em></strong> provides a <a class="bx ir" href="https://github.com/prometheus/statsd_exporter#metric-mapping-and-configuration" rel="noopener ugc nofollow" target="_blank">config interface for defining metric mappings</a> that uses glob-matching rules to reshape metrics on the fly. However in practice we found that engineers (who have limited time) would prefer to write a hacky query than figure out how to write a mapping rule requiring a full re-deploy of their service. It’s our hope that these remaining problematic metrics will eventually be reshaped and re-instrumented with the native Prometheus SDK.</p><h2 id="ead5" class="qc qd kf z qe qf qg qh fk qi qj qk fm ql qm qn qo qp qq qr qs qt qu qv qw qx cv">Takeaways</h2><ul class=""><li id="ff82" class="ph pi kf pj b pk qy pm pn po qz pq pr fn ra pt pu fq rb pw px ft rc pz qa qb sf re rf cv">PromQL does not handle name-encoded dimensions well.</li><li id="db90" class="ph pi kf pj b pk rg pm pn po rh pq pr fn ri pt pu fq rj pw px ft rk pz qa qb sf re rf cv">Replacing RPC interceptors is a huge win for metric reshaping and requires very little effort.</li><li id="76b0" class="ph pi kf pj b pk rg pm pn po rh pq pr fn ri pt pu fq rj pw px ft rk pz qa qb sf re rf cv">Take extra care to figure out how you want to reshape metrics <em class="rl">before</em> migrating queries.</li></ul><h2 id="ae8c" class="qc qd kf z qe qf qg qh fk qi qj qk fm ql qm qn qo qp qq qr qs qt qu qv qw qx cv">Query Rewrites</h2><p id="76d7" class="pw-post-body-paragraph ph pi kf pj b pk qy pm pn po qz pq pr fn ra pt pu fq rb pw px ft rc pz qa qb it cv">The final challenge was to rewrite all our dashboard and alert queries in PromQL. Dashboards needed to be migrated from our old vendor’s proprietary format to Grafana, while alerts needed to be migrated to our new vendor’s proprietary format.</p><h2 id="f501" class="qc qd kf z qe qf qg qh fk qi qj qk fm ql qm qn qo qp qq qr qs qt qu qv qw qx cv">Can We Automate it?</h2><p id="f62e" class="pw-post-body-paragraph ph pi kf pj b pk qy pm pn po qz pq pr fn ra pt pu fq rb pw px ft rc pz qa qb it cv">Initially we looked into tools to automatically rewrite our existing queries to PromQL, but it turns out that’s basically an unsolvable problem. There’s just too many nuances in the differences between time series query languages and their underlying data <a class="bx ir" href="https://promlabs.com/blog/2020/08/06/comparing-promql-correctness-across-vendors/" rel="noopener ugc nofollow" target="_blank">even within PromQL implementations</a>, let alone across entirely different query language specs.</p><p id="ce5c" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">While we couldn’t automatically port queries, it technically would’ve been possible to programmatically convert dashboard and alert templates such that engineers only needed to fill in the queries. Unfortunately by the time we thought of this approach it was too late into the migration to have much of an impact.</p><h2 id="91d4" class="qc qd kf z qe qf qg qh fk qi qj qk fm ql qm qn qo qp qq qr qs qt qu qv qw qx cv">Reduce the Pain</h2><p id="93fd" class="pw-post-body-paragraph ph pi kf pj b pk qy pm pn po qz pq pr fn ra pt pu fq rb pw px ft rc pz qa qb it cv">Since there was no way around the rote-labor of manually rewriting queries, our job became to make that process as painless as possible.</p><p id="7a00" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">The work we put into the first two categories was a great start; Prometheus instrumentation for a service could be enabled with a single feature flag and our metrics were already reshaped and ready to be queried. Additionally, we set up multiple office hours per week where we could help engineers with PromQL questions or issues. We made sure to document everything to help bring all our engineers up-to-speed with PromQL as quickly as possible.</p><figure class="rp rq rr rs rt ru rm rn paragraph-image"><div role="button" tabindex="0" class="rv rw dm rx cs ry"><span class="dr ds dt ac du dv dw dd dx speechify-ignore">Press enter or click to view image in full size</span><div class="rm rn sq"><picture><source srcSet="https://miro.medium.com/v2/resize:fit:640/format:webp/1*Y6irDD2KZqyqIvDc-433qQ.png 640w, https://miro.medium.com/v2/resize:fit:720/format:webp/1*Y6irDD2KZqyqIvDc-433qQ.png 720w, https://miro.medium.com/v2/resize:fit:750/format:webp/1*Y6irDD2KZqyqIvDc-433qQ.png 750w, https://miro.medium.com/v2/resize:fit:786/format:webp/1*Y6irDD2KZqyqIvDc-433qQ.png 786w, https://miro.medium.com/v2/resize:fit:828/format:webp/1*Y6irDD2KZqyqIvDc-433qQ.png 828w, https://miro.medium.com/v2/resize:fit:1100/format:webp/1*Y6irDD2KZqyqIvDc-433qQ.png 1100w, https://miro.medium.com/v2/resize:fit:1400/format:webp/1*Y6irDD2KZqyqIvDc-433qQ.png 1400w" sizes="auto, (min-resolution: 4dppx) and (max-width: 700px) 50vw, (-webkit-min-device-pixel-ratio: 4) and (max-width: 700px) 50vw, (min-resolution: 3dppx) and (max-width: 700px) 67vw, (-webkit-min-device-pixel-ratio: 3) and (max-width: 700px) 65vw, (min-resolution: 2.5dppx) and (max-width: 700px) 80vw, (-webkit-min-device-pixel-ratio: 2.5) and (max-width: 700px) 80vw, (min-resolution: 2dppx) and (max-width: 700px) 100vw, (-webkit-min-device-pixel-ratio: 2) and (max-width: 700px) 100vw, 700px" type="image/webp"/><source data-testid="og" srcSet="https://miro.medium.com/v2/resize:fit:640/1*Y6irDD2KZqyqIvDc-433qQ.png 640w, https://miro.medium.com/v2/resize:fit:720/1*Y6irDD2KZqyqIvDc-433qQ.png 720w, https://miro.medium.com/v2/resize:fit:750/1*Y6irDD2KZqyqIvDc-433qQ.png 750w, https://miro.medium.com/v2/resize:fit:786/1*Y6irDD2KZqyqIvDc-433qQ.png 786w, https://miro.medium.com/v2/resize:fit:828/1*Y6irDD2KZqyqIvDc-433qQ.png 828w, https://miro.medium.com/v2/resize:fit:1100/1*Y6irDD2KZqyqIvDc-433qQ.png 1100w, https://miro.medium.com/v2/resize:fit:1400/1*Y6irDD2KZqyqIvDc-433qQ.png 1400w" sizes="auto, (min-resolution: 4dppx) and (max-width: 700px) 50vw, (-webkit-min-device-pixel-ratio: 4) and (max-width: 700px) 50vw, (min-resolution: 3dppx) and (max-width: 700px) 67vw, (-webkit-min-device-pixel-ratio: 3) and (max-width: 700px) 65vw, (min-resolution: 2.5dppx) and (max-width: 700px) 80vw, (-webkit-min-device-pixel-ratio: 2.5) and (max-width: 700px) 80vw, (min-resolution: 2dppx) and (max-width: 700px) 100vw, (-webkit-min-device-pixel-ratio: 2) and (max-width: 700px) 100vw, 700px"/><img alt="" class="cs er rz c" width="700" height="481" loading="lazy" role="presentation"/></picture></div></div></figure><p id="dcc9" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">In total there were ~300 dashboards and ~600 alerts to migrate totaling to roughly 4,000 query rewrites. Between 20 engineers we were able to complete this phase of the migration in just over 1 month!</p><h2 id="893e" class="qc qd kf z qe qf qg qh fk qi qj qk fm ql qm qn qo qp qq qr qs qt qu qv qw qx cv">Takeaways</h2><ul class=""><li id="47e4" class="ph pi kf pj b pk qy pm pn po qz pq pr fn ra pt pu fq rb pw px ft rc pz qa qb sf re rf cv">Make sure you fully understand the amount of work before committing to a migration</li><li id="0d27" class="ph pi kf pj b pk rg pm pn po rh pq pr fn ri pt pu fq rj pw px ft rk pz qa qb sf re rf cv">Automate conversion of dashboard and alert templates if possible.</li><li id="429c" class="ph pi kf pj b pk rg pm pn po rh pq pr fn ri pt pu fq rj pw px ft rk pz qa qb sf re rf cv">Become a PromQL expert beforehand and schedule time to impart your knowledge to everyone else.</li></ul><h2 id="fe24" class="qc qd kf z qe qf qg qh fk qi qj qk fm ql qm qn qo qp qq qr qs qt qu qv qw qx cv">Final Thoughts</h2><p id="ba54" class="pw-post-body-paragraph ph pi kf pj b pk qy pm pn po qz pq pr fn ra pt pu fq rb pw px ft rc pz qa qb it cv">I’m incredibly proud of how efficiently our engineering team was able to execute this migration. I’m also excited to see how the Prometheus ecosystem will evolve over time, and I’m glad to say that Mixpanel will be along for the ride!</p><p id="e7e8" class="pw-post-body-paragraph ph pi kf pj b pk pl pm pn po pp pq pr fn ps pt pu fq pv pw px ft py pz qa qb it cv">Do you love metrics as much as we do? If so, the DevInfra team at Mixpanel is hiring! Apply <a class="bx ir" href="https://mixpanel.com/jobs" rel="noopener ugc nofollow" target="_blank">here</a> and mention this blog post in your application!</p></div></div></div></div></section></div></div></article></div><div class="am ev"><div class="fc cs iz ja jb jc"><div class="sr ss am jo do"><div class="am"><div class="jp jq bp jr dd"><div class="js"><div class="am"><div class="am" aria-haspopup="dialog"><a class="bx x by cf ca ab cb st ae af ag ah ai aj ak am j ju jv" rel="noopener follow" href="/tag/prometheus?source=post_page---footer_tags--fb973af124f5-----------------------------------------" data-discover="true"><span class="z b cr u cv">Prometheus</span></a></div></div></div><button aria-label="Follow Prometheus" class="w dn by cf ca ab cb su ae af ag bm am j jx jy jv jz dp"><svg width="12" height="12" viewBox="0 0 12 12" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true"><path d="M5.96219 1V6M5.96219 11V6M5.96219 6H11M5.96219 6H1" stroke="currentColor" stroke-linecap="round"></path></svg></button></div></div><div class="am"><div class="jp jq bp jr dd"><div class="js"><div class="am"><div class="am" aria-haspopup="dialog"><a class="bx x by cf ca ab cb st ae af ag ah ai aj ak am j ju jv" rel="noopener follow" href="/tag/metrics?source=post_page---footer_tags--fb973af124f5-----------------------------------------" data-discover="true"><span class="z b cr u cv">Metrics</span></a></div></div></div><button aria-label="Follow Metrics" class="w dn by cf ca ab cb su ae af ag bm am j jx jy jv jz dp"><svg width="12" height="12" viewBox="0 0 12 12" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true"><path d="M5.96219 1V6M5.96219 11V6M5.96219 6H11M5.96219 6H1" stroke="currentColor" stroke-linecap="round"></path></svg></button></div></div></div></div></div><div class="e"></div><footer class="sv sw sx sy sz am j ta tb"><div class="e ce"><div class="am ev"><div class="fc cs iz ja jb jc"><div class="am lc gg"><div class="am j nh"><div class="tc e"><span class="e td te tf tg s"><div class="am j nh ni"><div class="pw-multi-vote-icon dm nj nk nl nm"><span data-dd-action-name="Susi presentation tracker clap_footer"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" data-testid="footerClapButton" rel="noopener follow" href="/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fvote%2Fmixpaneleng%2Ffb973af124f5&amp;operation=register&amp;redirect=https%3A%2F%2Fmedium.com%2Fmixpaneleng%2Fhow-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5&amp;user=Eddie+Bracho&amp;userId=fc8168dcb359&amp;source=---footer_actions--fb973af124f5---------------------clap_footer--------------------" data-discover="true"><div><div class="bt"><div tabindex="-1" class="cq"><div class="nn af dn no np nq ac nr ns nt nm" role="presentation"><svg width="24" height="24" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-label="clap"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.3702 0.827583L12 3.2822L12.6298 0.827583H11.3702Z"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M13.9159 3.95255L15.4386 1.84138L14.2552 1.45158L13.9159 3.95255Z"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M8.58869 1.84055L10.1114 3.95172L9.77379 1.45076L8.58869 1.84055Z"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M18.523 18.9192C17.6623 19.7799 16.7727 20.1664 15.9029 20.25C16.0419 20.1341 16.1776 20.01 16.3101 19.8776C18.6976 17.4891 19.1702 14.9269 17.7087 12.2547L16.7975 10.6516L16.0072 8.9799C15.7465 8.42046 15.8127 7.99921 16.2099 7.69218C16.2866 7.63312 16.3749 7.59082 16.469 7.56801C16.5631 7.5452 16.6609 7.54239 16.7561 7.55977C17.0392 7.60611 17.3023 7.79149 17.4836 8.06046L19.8472 12.2174C20.8229 13.8411 20.9884 16.4538 18.523 18.9192ZM7.52441 18.4822L3.36992 14.3278C3.25591 14.2118 3.1786 14.0648 3.14765 13.9051C3.1167 13.7454 3.13348 13.5802 3.19591 13.43C3.25833 13.2798 3.36363 13.1513 3.49864 13.0606C3.63365 12.9699 3.79239 12.921 3.95503 12.92C4.16606 12.92 4.37875 13.0011 4.54013 13.1617L6.69765 15.3192C6.76738 15.3812 6.85815 15.4142 6.9514 15.4114C7.04465 15.4087 7.13333 15.3704 7.1993 15.3044C7.26526 15.2385 7.30353 15.1498 7.30627 15.0566C7.30902 14.9633 7.27603 14.8725 7.21406 14.8028L5.05655 12.6445L3.60827 11.1962C3.4532 11.0414 3.36595 10.8314 3.36572 10.6124C3.36549 10.3933 3.45229 10.1831 3.60703 10.028C3.76177 9.87297 3.97177 9.78572 4.19084 9.78549C4.40991 9.78526 4.6201 9.87206 4.77517 10.0268L8.21296 13.4662C8.2468 13.5003 8.28703 13.5273 8.33134 13.5457C8.37566 13.5641 8.42317 13.5736 8.47117 13.5736C8.51916 13.5736 8.56668 13.5641 8.61099 13.5457C8.6553 13.5273 8.69553 13.5003 8.72937 13.4662C8.7634 13.4324 8.79041 13.3922 8.80884 13.3479C8.82726 13.3036 8.83675 13.256 8.83675 13.208C8.83675 13.16 8.82726 13.1125 8.80884 13.0682C8.79041 13.0239 8.7634 12.9837 8.72937 12.9498L5.29323 9.51287L4.3233 8.54377C4.16876 8.38892 4.08197 8.17909 4.08197 7.96032C4.08197 7.74155 4.16876 7.53171 4.3233 7.37687C4.4802 7.22625 4.68926 7.14214 4.90675 7.14214C5.12424 7.14214 5.33331 7.22625 5.4902 7.37687L6.46013 8.34514L9.8971 11.7813C9.93077 11.8156 9.97094 11.8429 10.0153 11.8615C10.0596 11.8801 10.1072 11.8897 10.1553 11.8897C10.2034 11.8897 10.251 11.8801 10.2953 11.8615C10.3397 11.8429 10.3798 11.8156 10.4135 11.7813C10.4818 11.7127 10.5202 11.6199 10.5202 11.5231C10.5202 11.4263 10.4818 11.3334 10.4135 11.2649L6.97655 7.82956C6.89975 7.75312 6.83891 7.66217 6.79756 7.56201C6.75622 7.46185 6.7352 7.35447 6.73572 7.24611C6.73507 7.13769 6.75595 7.03021 6.79715 6.92991C6.83835 6.82962 6.89904 6.7385 6.97572 6.66183C7.13084 6.50719 7.34095 6.42035 7.55999 6.42035C7.77903 6.42035 7.98914 6.50719 8.14427 6.66183L13.9308 12.4491C13.9922 12.5076 14.0721 12.5429 14.1567 12.549C14.2413 12.5552 14.3254 12.5318 14.3947 12.4829C14.464 12.434 14.5142 12.3626 14.5367 12.2809C14.5593 12.1991 14.5528 12.112 14.5183 12.0345L13.4011 9.67094C13.1404 9.11149 13.2066 8.69025 13.6047 8.38239C13.6817 8.32396 13.77 8.28204 13.864 8.25926C13.958 8.23648 14.0556 8.23331 14.1509 8.24997C14.4339 8.29632 14.6963 8.4817 14.8783 8.75149L17.0714 12.6114C18.3732 14.9915 17.9545 17.2003 15.7936 19.3611C14.6383 20.5173 13.1917 20.9882 11.6036 20.7283C10.1859 20.4916 8.73765 19.6955 7.52524 18.4822H7.52441ZM10.7503 5.9708L12.8706 8.09108C12.4593 8.59342 12.4047 9.26128 12.7423 9.98128L12.9616 10.4464L9.43861 6.92335C9.37757 6.81002 9.34438 6.68378 9.34179 6.55508C9.34179 6.33411 9.42786 6.12721 9.58344 5.9708C9.74082 5.82112 9.9497 5.73766 10.1669 5.73766C10.3841 5.73766 10.593 5.82112 10.7503 5.9708ZM18.1051 7.67563C17.7948 7.21466 17.3586 6.91756 16.8753 6.83894C16.6812 6.80632 16.4825 6.81388 16.2915 6.86114C16.1005 6.9084 15.9212 6.99436 15.7647 7.1137C15.4535 7.3537 15.2599 7.65659 15.1754 7.99508C14.9266 7.75301 14.611 7.59102 14.2692 7.52997C13.9911 7.4859 13.7061 7.52267 13.4483 7.6359L11.2668 5.45439C10.9747 5.16339 10.5792 5.00001 10.1669 5.00001C9.7546 5.00001 9.35911 5.16339 9.06703 5.45439C8.87539 5.64887 8.73887 5.89078 8.67144 6.15535C8.5271 6.00929 8.35536 5.89315 8.16606 5.81358C7.97675 5.73401 7.77361 5.69257 7.56826 5.69165C7.36292 5.69073 7.15941 5.73034 6.9694 5.8082C6.77938 5.88607 6.60661 6.00066 6.46096 6.14542C6.25382 6.35175 6.10947 6.6126 6.04468 6.8977C5.42068 6.2737 4.39613 6.2737 3.80772 6.86128C3.51632 7.15315 3.35266 7.54872 3.35266 7.96114C3.35266 8.37357 3.51632 8.76914 3.80772 9.06101C3.56937 9.16032 3.30703 9.29935 3.09268 9.5137C2.80164 9.80592 2.63823 10.2015 2.63823 10.614C2.63823 11.0264 2.80164 11.422 3.09268 11.7142L3.6091 12.229C3.33829 12.291 3.08902 12.4243 2.88721 12.6152C2.6854 12.8061 2.53842 13.0476 2.46154 13.3146C2.38467 13.5815 2.38071 13.8642 2.45008 14.1332C2.51945 14.4022 2.65961 14.6477 2.85599 14.8442L7.00965 18.9987C8.32965 20.3178 9.91861 21.1876 11.4852 21.4475C11.806 21.5011 12.1307 21.5282 12.456 21.5286C13.3051 21.5286 14.1087 21.33 14.8494 20.9493C15.0803 20.9824 15.3087 21.0031 15.5346 21.0031C16.8008 21.0031 17.9917 20.4834 19.0403 19.4356C21.8028 16.6731 21.5917 13.7021 20.4786 11.85L18.1051 7.67645V7.67563Z"></path></svg></div></div></div></div></a></span></div><div class="pw-multi-vote-count e nu nv nw nx ny nz oa"><p class="z b y u w"><span class="ob">--</span></p></div></div></span><span class="e ck th ga ti tj"><div class="am j nh ni"><div class="pw-multi-vote-icon dm nj nk nl nm"><span data-dd-action-name="Susi presentation tracker clap_footer"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" data-testid="footerClapButton" rel="noopener follow" href="/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fvote%2Fmixpaneleng%2Ffb973af124f5&amp;operation=register&amp;redirect=https%3A%2F%2Fmedium.com%2Fmixpaneleng%2Fhow-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5&amp;user=Eddie+Bracho&amp;userId=fc8168dcb359&amp;source=---footer_actions--fb973af124f5---------------------clap_footer--------------------" data-discover="true"><div><div class="bt"><div tabindex="-1" class="cq"><div class="nn af dn no np nq ac nr ns nt nm" role="presentation"><svg width="24" height="24" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" aria-label="clap"><path fill-rule="evenodd" clip-rule="evenodd" d="M11.3702 0.827583L12 3.2822L12.6298 0.827583H11.3702Z"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M13.9159 3.95255L15.4386 1.84138L14.2552 1.45158L13.9159 3.95255Z"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M8.58869 1.84055L10.1114 3.95172L9.77379 1.45076L8.58869 1.84055Z"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M18.523 18.9192C17.6623 19.7799 16.7727 20.1664 15.9029 20.25C16.0419 20.1341 16.1776 20.01 16.3101 19.8776C18.6976 17.4891 19.1702 14.9269 17.7087 12.2547L16.7975 10.6516L16.0072 8.9799C15.7465 8.42046 15.8127 7.99921 16.2099 7.69218C16.2866 7.63312 16.3749 7.59082 16.469 7.56801C16.5631 7.5452 16.6609 7.54239 16.7561 7.55977C17.0392 7.60611 17.3023 7.79149 17.4836 8.06046L19.8472 12.2174C20.8229 13.8411 20.9884 16.4538 18.523 18.9192ZM7.52441 18.4822L3.36992 14.3278C3.25591 14.2118 3.1786 14.0648 3.14765 13.9051C3.1167 13.7454 3.13348 13.5802 3.19591 13.43C3.25833 13.2798 3.36363 13.1513 3.49864 13.0606C3.63365 12.9699 3.79239 12.921 3.95503 12.92C4.16606 12.92 4.37875 13.0011 4.54013 13.1617L6.69765 15.3192C6.76738 15.3812 6.85815 15.4142 6.9514 15.4114C7.04465 15.4087 7.13333 15.3704 7.1993 15.3044C7.26526 15.2385 7.30353 15.1498 7.30627 15.0566C7.30902 14.9633 7.27603 14.8725 7.21406 14.8028L5.05655 12.6445L3.60827 11.1962C3.4532 11.0414 3.36595 10.8314 3.36572 10.6124C3.36549 10.3933 3.45229 10.1831 3.60703 10.028C3.76177 9.87297 3.97177 9.78572 4.19084 9.78549C4.40991 9.78526 4.6201 9.87206 4.77517 10.0268L8.21296 13.4662C8.2468 13.5003 8.28703 13.5273 8.33134 13.5457C8.37566 13.5641 8.42317 13.5736 8.47117 13.5736C8.51916 13.5736 8.56668 13.5641 8.61099 13.5457C8.6553 13.5273 8.69553 13.5003 8.72937 13.4662C8.7634 13.4324 8.79041 13.3922 8.80884 13.3479C8.82726 13.3036 8.83675 13.256 8.83675 13.208C8.83675 13.16 8.82726 13.1125 8.80884 13.0682C8.79041 13.0239 8.7634 12.9837 8.72937 12.9498L5.29323 9.51287L4.3233 8.54377C4.16876 8.38892 4.08197 8.17909 4.08197 7.96032C4.08197 7.74155 4.16876 7.53171 4.3233 7.37687C4.4802 7.22625 4.68926 7.14214 4.90675 7.14214C5.12424 7.14214 5.33331 7.22625 5.4902 7.37687L6.46013 8.34514L9.8971 11.7813C9.93077 11.8156 9.97094 11.8429 10.0153 11.8615C10.0596 11.8801 10.1072 11.8897 10.1553 11.8897C10.2034 11.8897 10.251 11.8801 10.2953 11.8615C10.3397 11.8429 10.3798 11.8156 10.4135 11.7813C10.4818 11.7127 10.5202 11.6199 10.5202 11.5231C10.5202 11.4263 10.4818 11.3334 10.4135 11.2649L6.97655 7.82956C6.89975 7.75312 6.83891 7.66217 6.79756 7.56201C6.75622 7.46185 6.7352 7.35447 6.73572 7.24611C6.73507 7.13769 6.75595 7.03021 6.79715 6.92991C6.83835 6.82962 6.89904 6.7385 6.97572 6.66183C7.13084 6.50719 7.34095 6.42035 7.55999 6.42035C7.77903 6.42035 7.98914 6.50719 8.14427 6.66183L13.9308 12.4491C13.9922 12.5076 14.0721 12.5429 14.1567 12.549C14.2413 12.5552 14.3254 12.5318 14.3947 12.4829C14.464 12.434 14.5142 12.3626 14.5367 12.2809C14.5593 12.1991 14.5528 12.112 14.5183 12.0345L13.4011 9.67094C13.1404 9.11149 13.2066 8.69025 13.6047 8.38239C13.6817 8.32396 13.77 8.28204 13.864 8.25926C13.958 8.23648 14.0556 8.23331 14.1509 8.24997C14.4339 8.29632 14.6963 8.4817 14.8783 8.75149L17.0714 12.6114C18.3732 14.9915 17.9545 17.2003 15.7936 19.3611C14.6383 20.5173 13.1917 20.9882 11.6036 20.7283C10.1859 20.4916 8.73765 19.6955 7.52524 18.4822H7.52441ZM10.7503 5.9708L12.8706 8.09108C12.4593 8.59342 12.4047 9.26128 12.7423 9.98128L12.9616 10.4464L9.43861 6.92335C9.37757 6.81002 9.34438 6.68378 9.34179 6.55508C9.34179 6.33411 9.42786 6.12721 9.58344 5.9708C9.74082 5.82112 9.9497 5.73766 10.1669 5.73766C10.3841 5.73766 10.593 5.82112 10.7503 5.9708ZM18.1051 7.67563C17.7948 7.21466 17.3586 6.91756 16.8753 6.83894C16.6812 6.80632 16.4825 6.81388 16.2915 6.86114C16.1005 6.9084 15.9212 6.99436 15.7647 7.1137C15.4535 7.3537 15.2599 7.65659 15.1754 7.99508C14.9266 7.75301 14.611 7.59102 14.2692 7.52997C13.9911 7.4859 13.7061 7.52267 13.4483 7.6359L11.2668 5.45439C10.9747 5.16339 10.5792 5.00001 10.1669 5.00001C9.7546 5.00001 9.35911 5.16339 9.06703 5.45439C8.87539 5.64887 8.73887 5.89078 8.67144 6.15535C8.5271 6.00929 8.35536 5.89315 8.16606 5.81358C7.97675 5.73401 7.77361 5.69257 7.56826 5.69165C7.36292 5.69073 7.15941 5.73034 6.9694 5.8082C6.77938 5.88607 6.60661 6.00066 6.46096 6.14542C6.25382 6.35175 6.10947 6.6126 6.04468 6.8977C5.42068 6.2737 4.39613 6.2737 3.80772 6.86128C3.51632 7.15315 3.35266 7.54872 3.35266 7.96114C3.35266 8.37357 3.51632 8.76914 3.80772 9.06101C3.56937 9.16032 3.30703 9.29935 3.09268 9.5137C2.80164 9.80592 2.63823 10.2015 2.63823 10.614C2.63823 11.0264 2.80164 11.422 3.09268 11.7142L3.6091 12.229C3.33829 12.291 3.08902 12.4243 2.88721 12.6152C2.6854 12.8061 2.53842 13.0476 2.46154 13.3146C2.38467 13.5815 2.38071 13.8642 2.45008 14.1332C2.51945 14.4022 2.65961 14.6477 2.85599 14.8442L7.00965 18.9987C8.32965 20.3178 9.91861 21.1876 11.4852 21.4475C11.806 21.5011 12.1307 21.5282 12.456 21.5286C13.3051 21.5286 14.1087 21.33 14.8494 20.9493C15.0803 20.9824 15.3087 21.0031 15.5346 21.0031C16.8008 21.0031 17.9917 20.4834 19.0403 19.4356C21.8028 16.6731 21.5917 13.7021 20.4786 11.85L18.1051 7.67645V7.67563Z"></path></svg></div></div></div></div></a></span></div><div class="pw-multi-vote-count e nu nv nw nx ny nz oa"><p class="z b y u cv"><span class="ob">--</span></p></div></div></span></div><div class="tk am"><div><div class="bt"><div tabindex="-1" class="cq"><button class="af nn go oe am j dn np of" aria-label="responses"><svg width="24" height="24" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" class="lb"><path d="M18.0064 16.8029C19.5393 15.347 20.24 13.4782 20.24 11.4818C20.24 7.35659 16.7088 4 12.1914 4C7.67395 4 4 7.35659 4 11.4818C4 15.6077 7.67395 18.9643 12.1914 18.9643C13.0081 18.9643 13.8134 18.8527 14.5839 18.6374C14.815 18.8368 15.064 19.0281 15.3284 19.1956C16.3876 19.8892 17.5312 20.24 18.7266 20.24C18.9505 20.232 19.1273 20.1284 19.2132 19.953C19.2554 19.8711 19.2738 19.7795 19.2664 19.688C19.2589 19.5966 19.2259 19.5089 19.1711 19.4347C18.6763 18.765 18.326 18.0714 18.1313 17.3777C18.0754 17.1816 18.0337 16.9817 18.0064 16.7798V16.8029ZM14.884 17.8585L14.8167 17.6353L14.502 17.7309C13.7529 17.9559 12.9738 18.0698 12.1906 18.069C8.16787 18.069 4.89943 15.1142 4.89943 11.4818C4.89943 7.84932 8.16787 4.89376 12.1906 4.89376C16.2051 4.89376 19.3033 7.85171 19.3033 11.4874C19.3033 13.2813 18.695 14.9556 17.2757 16.2073L17.081 16.3747V16.6299C17.081 16.6857 17.0811 16.7814 17.0973 16.9249C17.1216 17.1561 17.1784 17.4032 17.2514 17.6584C17.4055 18.2165 17.6488 18.7746 17.9732 19.3167C17.2189 19.2051 16.489 18.9261 15.8077 18.4716C15.532 18.2962 15.094 18.0889 14.8669 17.8816L14.884 17.8585Z"></path></svg><p class="z b cr u w"><span class="pw-responses-count od lb">1</span></p></button></div></div></div></div><div class="am j w"><div><div class="bt"><div tabindex="-1" class="cq"><span data-dd-action-name="Susi presentation tracker repost_footer"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" rel="noopener follow" href="/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Frepost%2Fp%2Ffb973af124f5&amp;operation=register&amp;redirect=https%3A%2F%2Fmedium.com%2Fmixpaneleng%2Fhow-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5&amp;user=Eddie+Bracho&amp;userId=fc8168dcb359&amp;source=---footer_actions--fb973af124f5---------------------repost_footer--------------------" data-discover="true"><span class="nn dz bx af go nr am j ns nt nm ac og oh" aria-disabled="false" aria-label="Repost"><div class="dg df e dm"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none"><path d="M3 8.72727L5.64706 6M5.64706 6L8.29412 8.72727M5.64706 6V14.1818V15.2475C5.64706 15.9633 5.92589 16.6509 6.42442 17.1646C6.94277 17.6986 7.6553 18 8.39955 18H12" stroke="currentColor" stroke-linecap="round"></path><path d="M21 15.2727L18.3529 18M18.3529 18L15.7059 15.2727M18.3529 18V9.81818V8.75249C18.3529 8.03671 18.0741 7.34908 17.5756 6.83545C17.0572 6.30139 16.3447 6 15.6004 6L12 6" stroke="currentColor" stroke-linecap="round"></path></svg></div></span></a></span></div></div></div><div class="od e"></div></div></div><div class="am j"><div class="ck"><div class="tl e ej"><div><div class="bt"><div tabindex="-1" class="cq"><span data-dd-action-name="Susi presentation tracker bookmark_footer"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" data-testid="footerBookmarkButton" rel="noopener follow" href="/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fbookmark%2Fp%2Ffb973af124f5&amp;operation=register&amp;redirect=https%3A%2F%2Fmedium.com%2Fmixpaneleng%2Fhow-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5&amp;source=---footer_actions--fb973af124f5---------------------bookmark_footer--------------------" data-discover="true"><span class=""><svg width="25" height="25" viewBox="0 0 25 25" fill="none" xmlns="http://www.w3.org/2000/svg" class="w oi" aria-label="Add to list bookmark button"><path d="M18 2.5C18 2.22386 18.2239 2 18.5 2C18.7762 2 19 2.22386 19 2.5V5H21.5C21.7761 5 22 5.22386 22 5.5C22 5.77614 21.7761 6 21.5 6H19V8.5C19 8.77614 18.7762 9 18.5 9C18.2239 9 18 8.77614 18 8.5V6H15.5C15.2239 6 15 5.77614 15 5.5C15 5.22386 15.2239 5 15.5 5H18V2.5Z" fill="currentColor"></path><path d="M7 7C7 6.44772 7.44772 6 8 6H11.5C11.7761 6 12 5.77614 12 5.5C12 5.22386 11.7761 5 11.5 5H8C6.89543 5 6 5.89543 6 7V21C6 21.1905 6.10823 21.3644 6.27912 21.4486C6.45002 21.5327 6.65387 21.5125 6.80486 21.3963L12.5 17L18.1951 21.3963C18.3461 21.5125 18.55 21.5327 18.7209 21.4486C18.8918 21.3644 19 21.1905 19 21V12.5C19 12.2239 18.7761 12 18.5 12C18.2239 12 18 12.2239 18 12.5V19.9846L12.8049 15.9729C12.6251 15.8346 12.3749 15.8346 12.1951 15.9729L7 19.9846V7Z" fill="currentColor"></path></svg></span></a></span></div></div></div></div></div><div class="tl e ej"><div class="bt" aria-describedby="postFooterSocialMenu" aria-labelledby="postFooterSocialMenu" aria-haspopup="dialog"><div><div class="bt"><div tabindex="-1" class="cq"><button aria-controls="postFooterSocialMenu" aria-expanded="false" aria-label="Share Post" data-testid="footerSocialShareButton" class="bx dn by bz ca ab cb oo ae af ag bm op oq or os"><svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M15.2176 4.93118C15.1883 4.98289 15.1484 5.0278 15.1004 5.06293L15.1118 5.06859C15.024 5.12481 14.9269 5.15261 14.8204 5.14293C14.7004 5.13293 14.6004 5.08293 14.5204 5.01293L12.5004 2.99293V10.0629C12.5004 10.3429 12.2704 10.5629 12.0004 10.5629C11.7304 10.5629 11.5004 10.3429 11.5004 10.0629V3.02293L9.50042 5.02293C9.42474 5.09456 9.32654 5.13771 9.22259 5.14501C9.11863 5.1523 9.01537 5.12329 8.93042 5.06293H8.91042C8.82042 4.99293 8.75042 4.88293 8.75042 4.76293C8.74294 4.70553 8.74801 4.64719 8.76528 4.59195C8.78254 4.5367 8.81159 4.48586 8.85042 4.44293L11.6504 1.64293C11.7439 1.55132 11.8695 1.5 12.0004 1.5C12.1313 1.5 12.257 1.55132 12.3504 1.64293L15.1504 4.43293C15.1931 4.47602 15.2258 4.52782 15.2466 4.58479C15.2673 4.64175 15.2754 4.70252 15.2704 4.76293C15.2649 4.82212 15.2469 4.87946 15.2176 4.93118ZM15.1118 5.06859L15.1204 5.07293V5.06293C15.1175 5.06485 15.1147 5.06674 15.1118 5.06859ZM16 7.06293H17.5C18.0304 7.06293 18.5391 7.27364 18.9142 7.64871C19.2893 8.02379 19.5 8.53249 19.5 9.06293V19.0629C19.5 19.5934 19.2893 20.1021 18.9142 20.4771C18.5391 20.8522 18.0304 21.0629 17.5 21.0629H6.5C5.4 21.0629 4.5 20.1629 4.5 19.0629V9.06293C4.5 8.53249 4.71071 8.02379 5.08579 7.64871C5.46086 7.27364 5.96957 7.06293 6.5 7.06293H8C8.13188 7.06481 8.25769 7.11872 8.35 7.21293C8.44421 7.30524 8.49812 7.43104 8.5 7.56293C8.49812 7.69481 8.44421 7.82061 8.35 7.91293C8.25769 8.00713 8.13188 8.06105 8 8.06293H6.4C5.9 8.06293 5.5 8.46293 5.5 8.96293V19.1629C5.5 19.4016 5.59482 19.6305 5.7636 19.7993C5.93239 19.9681 6.16131 20.0629 6.4 20.0629H17.6C18.1 20.0629 18.5 19.6629 18.5 19.1629V8.96293C18.5 8.46293 18.1 8.06293 17.6 8.06293H16C15.8674 8.06293 15.7402 8.01025 15.6464 7.91648C15.5527 7.82271 15.5 7.69554 15.5 7.56293C15.5 7.43032 15.5527 7.30314 15.6464 7.20937C15.7402 7.11561 15.8674 7.06293 16 7.06293Z" fill="currentColor"></path></svg></button></div></div></div></div></div></div></div></div></div></div></footer><div class="tm e"><div class="am ev"><div class="fc cs iz ja jb jc"><div class="am gh tn"><div class="am lm tp lk tq li tr ts tt tu tv"><div class="tw tx ty tz ua ub uc ud ue uf am lc"><div class="ck cz"><a href="https://medium.com/mixpaneleng?source=post_page---post_publication_info--fb973af124f5-----------------------------------------" rel="noopener follow"><div class="dm"><img alt="Mixpanel Engineering" class="ei in e uh ug" src="https://miro.medium.com/v2/resize:fill:96:96/1*wdBmRrZ2IldG8fNVhER_xg.png" width="48" height="48" loading="lazy"/><div class="in e ug uh dr g ed iq"></div></div></a></div><div class="da r s"><a href="https://medium.com/mixpaneleng?source=post_page---post_publication_info--fb973af124f5-----------------------------------------" rel="noopener follow"><div class="dm"><img alt="Mixpanel Engineering" class="ei in e uj ui" src="https://miro.medium.com/v2/resize:fill:128:128/1*wdBmRrZ2IldG8fNVhER_xg.png" width="64" height="64" loading="lazy"/><div class="in e ui uj dr g ed iq"></div></div></a></div><div class="da r s uk ej"></div></div><div class="fc am gh el"><div class="ul um un uo up e"><a class="bx x by ca ab cb ac ae af ag ah ai aj ak" href="https://medium.com/mixpaneleng?source=post_page---post_publication_info--fb973af124f5-----------------------------------------" rel="noopener follow"><h2 class="z fi uq ur us ut uu uv fn fo fp fq fr fs ft fu fv cv">Published in <!-- -->Mixpanel Engineering</h2></a><div class="uw am lo"><div class="e ej"><span class="pw-follower-count z b cr u w"><a class="bx x by bz ca ab cb ac ae af ag ah ai lx" rel="noopener follow" href="/mixpaneleng/followers?source=post_page---post_publication_info--fb973af124f5-----------------------------------------" data-discover="true">341 followers</a></span></div><div class="z b cr u w am to"><span class="fw e" aria-hidden="true"><span class="z b cr u w">·</span></span><a class="bx x by bz ca ab cb ac ae af ag ah ai lx" rel="noopener follow" href="/mixpaneleng/why-we-left-medium-10cddd141c70?source=post_page---post_publication_info--fb973af124f5-----------------------------------------" data-discover="true">Last published <span>May 16, 2025</span></a></div></div><div class="ux e"><p class="z b cr u cv">Blog is now hosted at <a class="bx x by bz ca ab cb ac ae af ag ah ai ir is it" href="https://mixpanel.substack.com" rel="noopener ugc nofollow">https://mixpanel.substack.com</a></p></div></div></div><div class="ck cz"><div class="e ej"></div></div></div><div class="am lm tp lk tq li tr ts tt tu tv"><div class="tw tx ty tz ua ub uc ud ue uf am lc"><div class="ck cz"><a tabindex="0" rel="noopener follow" href="/@eddie.bracho?source=post_page---post_author_info--fb973af124f5-----------------------------------------" data-discover="true"><div class="e dm"><img alt="Eddie Bracho" class="e bs ee ug uh ei" src="https://miro.medium.com/v2/resize:fill:96:96/1*FgW6VTtMYvR9jCKWKNcecQ.jpeg" width="48" height="48" loading="lazy"/><div class="ed ee e ug uh dr g cf iq"></div></div></a></div><div class="da r s"><a tabindex="0" rel="noopener follow" href="/@eddie.bracho?source=post_page---post_author_info--fb973af124f5-----------------------------------------" data-discover="true"><div class="e dm"><img alt="Eddie Bracho" class="e bs ee ui uj ei" src="https://miro.medium.com/v2/resize:fill:128:128/1*FgW6VTtMYvR9jCKWKNcecQ.jpeg" width="64" height="64" loading="lazy"/><div class="ed ee e ui uj dr g cf iq"></div></div></a></div><div class="da r s uk ej"><div class="am do"><!--$--><!--/$--></div></div></div><div class="fc am gh el"><div class="ul um un uo up e"><a class="bx x by ca ab cb ac ae af ag ah ai aj ak" rel="noopener follow" href="/@eddie.bracho?source=post_page---post_author_info--fb973af124f5-----------------------------------------" data-discover="true"><h2 class="z fi uq ur us ut uu uv fn fo fp fq fr fs ft fu fv cv">Written by <!-- -->Eddie Bracho</h2></a><div class="uw am lo"><div class="e ej"><span class="pw-follower-count z b cr u w"><a class="bx x by bz ca ab cb ac ae af ag ah ai lx" rel="noopener follow" href="/@eddie.bracho/followers?source=post_page---post_author_info--fb973af124f5-----------------------------------------" data-discover="true">4 followers</a></span></div><div class="z b cr u w am to"><span class="fw e" aria-hidden="true"><span class="z b cr u w">·</span></span><a class="bx x by bz ca ab cb ac ae af ag ah ai lx" rel="noopener follow" href="/@eddie.bracho/following?source=post_page---post_author_info--fb973af124f5-----------------------------------------" data-discover="true">1 following</a></div></div></div></div><div class="ck cz"><div class="e ej"><div class="am do"><!--$--><!--/$--></div></div></div></div></div></div></div></div><div><div class="am ev"><div class="fc cs iz ja jb jc"><div class="uy uz"><div class="am nh"><div class="va ef eg e ei"></div><div class="v vb e"><div class="vc vd ve e ei"></div><div class="vd vf e ei"></div></div></div><div class="vg e"><div class="vc vh cs e ei"></div><div class="vc vh cs e ei"></div><div class="vh vi e ei"></div></div></div></div></div></div><div><div class="vj vk vl vm vn e"><div class="vo cs k vp vq iw ix iy"></div><div class="am ev"><div class="fc cs iz ja jb jc"><div class="vc vr vs e ei"></div><div class="vr vt e ei"></div><div class="vu am im"><div class="vv cm e ei"></div><div class="vv cm e ei"></div></div><div class="vw vr uh e ei"></div></div></div></div></div><div class="vx vy vz wa wb e"><div><div class="am ev"><div class="fc cs iz ja jb jc"><div class="jq am nh jo wc wd we wf wg wh wi wj wk wl wm wn wo wp wq"><div class="wr ws wt up wu wv ww uo wx wy wz xa xb xc xd xe xf xg xh xi xj"><div class="vw xk cs e ei"></div><div class="xl ef xm e ei"></div><div class="vc xn xo e ei"></div><div class="vc xn xp e ei"></div><div class="xq xn xr e ei"></div></div><div class="wr ws wt up wu wv ww uo wx wy wz xa xb xc xd xe xf xg xh xi xj"><div class="vw xk cs e ei"></div><div class="xl ef xm e ei"></div><div class="vc xn xo e ei"></div><div class="vc xn xp e ei"></div><div class="xq xn xr e ei"></div></div></div></div></div></div><div><div class="am ev"><div class="fc cs iz ja jb jc"><div class="jq am nh jo wc wd we wf wg wh wi wj wk wl wm wn wo wp wq"><div class="wr ws wt up wu wv ww uo wx wy wz xa xb xc xd xe xf xg xh xi xj"><div class="vw xk cs e ei"></div><div class="xl ef xm e ei"></div><div class="vc xn xo e ei"></div><div class="vc xn xp e ei"></div><div class="xq xn xr e ei"></div></div><div class="wr ws wt up wu wv ww uo wx wy wz xa xb xc xd xe xf xg xh xi xj"><div class="vw xk cs e ei"></div><div class="xl ef xm e ei"></div><div class="vc xn xo e ei"></div><div class="vc xn xp e ei"></div><div class="xq xn xr e ei"></div></div></div></div></div></div><div class="ck cz da"><div class="vo cs xs xt"></div><div class="am ev"><div class="fc cs iz ja jb jc"><div class="xu am nh jo"><div class="xv xw e"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" href="https://help.medium.com/hc/en-us?source=post_page-----fb973af124f5-----------------------------------------" rel="noopener follow"><p class="z b y u w">Help</p></a></div><div class="xv xw e"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" href="https://status.medium.com/?source=post_page-----fb973af124f5-----------------------------------------" rel="noopener follow"><p class="z b y u w">Status</p></a></div><div class="xv xw e"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" rel="noopener follow" href="/about?autoplay=1&amp;source=post_page-----fb973af124f5-----------------------------------------" data-discover="true"><p class="z b y u w">About</p></a></div><div class="xv xw e"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" rel="noopener follow" href="/jobs-at-medium/work-at-medium-959d1a85284e?source=post_page-----fb973af124f5-----------------------------------------" data-discover="true"><p class="z b y u w">Careers</p></a></div><div class="xv xw e"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" href="mailto:pressinquiries@medium.com" rel="noopener follow"><p class="z b y u w">Press</p></a></div><div class="xv xw e"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" href="https://blog.medium.com/?source=post_page-----fb973af124f5-----------------------------------------" rel="noopener follow"><p class="z b y u w">Blog</p></a></div><div class="xv xw e"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" href="https://medium.com/store" rel="noopener follow" target="_blank"><p class="z b y u w">Store</p></a></div><div class="xv xw e"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" href="https://policy.medium.com/medium-privacy-policy-f03bf92035c9?source=post_page-----fb973af124f5-----------------------------------------" rel="noopener follow"><p class="z b y u w">Privacy</p></a></div><div class="xv xw e"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" href="https://policy.medium.com/medium-rules-30e5502c4eb4?source=post_page-----fb973af124f5-----------------------------------------" rel="noopener follow"><p class="z b y u w">Rules</p></a></div><div class="xv xw e"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" href="https://policy.medium.com/medium-terms-of-service-9db0094a1e0f?source=post_page-----fb973af124f5-----------------------------------------" rel="noopener follow"><p class="z b y u w">Terms</p></a></div><div class="xv e"><a class="bx x by bz ca ab cb ac ae af ag ah ai aj ak" href="https://speechify.com/medium?source=post_page-----fb973af124f5-----------------------------------------" rel="noopener follow"><p class="z b y u w">Text to speech</p></a></div></div></div></div></div></div></div></div></div></div><script>((storageKey2, restoreKey) => {
if (!window.history.state || !window.history.state.key) {
let key = Math.random().toString(32).slice(2);
window.history.replaceState({ key }, "");
}
try {
let positions = JSON.parse(sessionStorage.getItem(storageKey2) || "{}");
let storedY = positions[restoreKey || window.history.state.key];
if (typeof storedY === "number") {
window.scrollTo(0, storedY);
}
} catch (error) {
console.error(error);
sessionStorage.removeItem(storageKey2);
}
})("react-router-scroll-positions", null)</script><script>window.__reactRouterContext = {"basename":"/","future":{"unstable_optimizeDeps":true,"v8_passThroughRequests":false,"v8_trailingSlashAwareDataRequests":false,"unstable_previewServerPrerendering":false,"v8_middleware":true,"v8_splitRouteModules":false,"v8_viteEnvironmentApi":false},"routeDiscovery":{"mode":"initial"},"ssr":true,"isSpaMode":false};window.__reactRouterContext.stream = new ReadableStream({start(controller){window.__reactRouterContext.streamController = controller;}}).pipeThrough(new TextEncoderStream());</script><script type="module" async="">import "https://cdn-client.medium.com/lite/static/manifest-3cf8583e.js";
import * as route0 from "https://cdn-client.medium.com/lite/assets/entry-fBNTF2ig9x.js";
import * as route1 from "https://cdn-client.medium.com/lite/assets/entry-DJjt4aY773.js";
window.__reactRouterRouteModules = {"root":route0,"routes/pages/entity-routes/ShowEntityPost":route1};
import("https://cdn-client.medium.com/lite/assets/entry-C9P2t20qJW.js");</script><!--$--><script>window.__reactRouterContext.streamController.enqueue("[{\"_1\":2,\"_3\":-5,\"_4\":-5},\"loaderData\",{\"_5\":6},\"actionData\",\"errors\",\"routes/pages/entity-routes/ShowEntityPost\",{}]\n");</script><!--$--><script>window.__reactRouterContext.streamController.close();</script><!--/$--><!--/$--><script>window.__APOLLO_STATE__ = {"ROOT_QUERY":{"__typename":"Query","viewer":null,"postResult({\"id\":\"fb973af124f5\"})":{"__ref":"Post:fb973af124f5"}},"User:5e2bfa799359":{"__typename":"User","id":"5e2bfa799359","customDomainState":null,"hasSubdomain":false,"username":"neal.kaviratna"},"Collection:b5de3443ad1f":{"__typename":"Collection","id":"b5de3443ad1f","domain":null,"slug":"mixpaneleng","googleAnalyticsId":null,"name":"Mixpanel Engineering","avatar":{"__typename":"ImageMetadata","id":"1*wdBmRrZ2IldG8fNVhER_xg.png"},"description":"Blog is now hosted at https:\u002F\u002Fmixpanel.substack.com","latestPostsConnection({\"paging\":{\"limit\":1}})":{"__typename":"PostConnection","posts":[{"__ref":"Post:10cddd141c70"}]},"subscriberCount":341,"viewerEdge":{"__ref":"CollectionViewerEdge:collectionId:b5de3443ad1f-viewerId:lo_5aa66d57da88"},"twitterUsername":"mixpanel","facebookPageId":null},"Post:10cddd141c70":{"__typename":"Post","id":"10cddd141c70","firstPublishedAt":1747430207625,"creator":{"__ref":"User:5e2bfa799359"},"collection":{"__ref":"Collection:b5de3443ad1f"},"isSeries":false,"mediumUrl":"https:\u002F\u002Fmedium.com\u002Fmixpaneleng\u002Fwhy-we-left-medium-10cddd141c70","sequence":null,"uniqueSlug":"why-we-left-medium-10cddd141c70"},"LinkedAccounts:fc8168dcb359":{"__typename":"LinkedAccounts","mastodon":null,"id":"fc8168dcb359"},"User:fc8168dcb359":{"__typename":"User","id":"fc8168dcb359","linkedAccounts":{"__ref":"LinkedAccounts:fc8168dcb359"},"isSuspended":false,"name":"Eddie Bracho","customDomainState":null,"hasSubdomain":false,"username":"eddie.bracho","bio":"","verifications":{"__typename":"VerifiedInfo","isBookAuthor":false},"imageId":"1*FgW6VTtMYvR9jCKWKNcecQ.jpeg","socialStats":{"__typename":"SocialStats","followerCount":4,"followingCount":1,"collectionFollowingCount":0},"newsletterV3":null,"allowNotes":true,"membership":null,"viewerEdge":{"__ref":"UserViewerEdge:userId:fc8168dcb359-viewerId:lo_5aa66d57da88"},"twitterScreenName":""},"Paragraph:9fe7eb1faed4_0":{"__typename":"Paragraph","id":"9fe7eb1faed4_0","name":"a54e","type":"H3","href":null,"layout":null,"metadata":null,"text":"How We Migrated from StatsD to Prometheus in One Month","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_1":{"__typename":"Paragraph","id":"9fe7eb1faed4_1","name":"ea28","type":"P","href":null,"layout":null,"metadata":null,"text":"We recently migrated all of our infrastructure metrics from StatsD to Prometheus and are very pleased with the results. The migration was a ton of work and we learned a lot along the way. This post aims to shed some light on why we migrated to Prometheus, as well as outline some of the technical challenges we faced during the process.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_2":{"__typename":"Paragraph","id":"9fe7eb1faed4_2","name":"da62","type":"H3","href":null,"layout":null,"metadata":null,"text":"Background","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_3":{"__typename":"Paragraph","id":"9fe7eb1faed4_3","name":"cd86","type":"P","href":null,"layout":null,"metadata":null,"text":"Metrics are such a vital, commonplace component in today’s distributed system that it’s easy to forget that popular open source metrics protocols only started appearing in the last decade. StatsD came out of Etsy back in 2011 with Prometheus following shortly afterwards from SoundCloud in 2012. Both collect the same kinds of measurements (counters, gauges, and sample distributions), but how they’re architected differs significantly in ways that reflect the priorities of distributed systems they came from. For example, StatsD’s push-based model makes it dead-simple to collect metrics from any process that can send a UDP packet, whereas Prometheus’s decentralized pull-based model makes it easier to scale metrics collection.","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"A","start":208,"end":225,"href":"https:\u002F\u002Fwww.etsy.com\u002Fcodeascraft\u002Fmeasure-anything-measure-everything\u002F","anchorType":"LINK","userId":null,"linkMetadata":null},{"__typename":"Markup","type":"A","start":276,"end":294,"href":"https:\u002F\u002Fdevelopers.soundcloud.com\u002Fblog\u002Fprometheus-monitoring-at-soundcloud","anchorType":"LINK","userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_4":{"__typename":"Paragraph","id":"9fe7eb1faed4_4","name":"3fc2","type":"P","href":null,"layout":null,"metadata":null,"text":"Mixpanel was not slow to jump on the metrics bandwagon — we are an analytics company after all. We first began collecting metrics through StatsD in 2013 (just 2 years after our landmark Why We Moved Off the Cloud blog post 😛). For a fast-moving, multi-language tech stack running on a fixed number of machines, StatsD made a lot of sense. Scaling metrics was not even close to being a question in anyone’s mind, and client libraries were incredibly easy to implement and reason about.","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"A","start":186,"end":212,"href":"https:\u002F\u002Fengineering.mixpanel.com\u002Fwhy-we-moved-off-the-cloud-907daac729a0","anchorType":"LINK","userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_5":{"__typename":"Paragraph","id":"9fe7eb1faed4_5","name":"f489","type":"P","href":null,"layout":null,"metadata":null,"text":"Fast-forward several years and Mixpanel’s infrastructure had taken on a completely different shape. We moved back onto cloud (no landmark blog post this time) allowing us much more freedom to decouple and scale our performance-critical services. Our services now ran on Kubernetes with individual pods collecting tens-of-thousands of metric samples per second. Engineers also evolved much stricter expectations for the amount of dropped metric samples (none). To deal with these new requirements, we converted our StatsD sidecar connections to TCP and moved metric sample aggregation to the client process (which reduced contention induced by large flushes to StatsD connections).","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_6":{"__typename":"Paragraph","id":"9fe7eb1faed4_6","name":"90b1","type":"P","href":null,"layout":null,"metadata":null,"text":"At some point we had outgrown traditional StatsD and accidentally engineered our own proprietary metrics architecture. So when the time came for us to change metrics vendors, we realized we had a several good reasons to move into the Prometheus ecosystem:","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_7":{"__typename":"Paragraph","id":"9fe7eb1faed4_7","name":"05df","type":"H3","href":null,"layout":null,"metadata":null,"text":"Performance","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_8":{"__typename":"Paragraph","id":"9fe7eb1faed4_8","name":"a037","type":"P","href":null,"layout":null,"metadata":null,"text":"A lot of the performance work we put into our internal metrics libraries mirrors closely with Prometheus’s architecture decisions. Prometheus’s pull-based model eliminates the chance for dropped metric samples, and recording a sample boils down to a single atomic operation in the client process.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_9":{"__typename":"Paragraph","id":"9fe7eb1faed4_9","name":"0a8c","type":"H3","href":null,"layout":null,"metadata":null,"text":"Reduce Vendor Lock-in","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_10":{"__typename":"Paragraph","id":"9fe7eb1faed4_10","name":"4a08","type":"P","href":null,"layout":null,"metadata":null,"text":"PromQL and Grafana are open standards meaning that the work we put into migrating our collectors, dashboards and alert queries could follow us to a variety of different platforms. This would reduce vendor lock-in and give us a ton of flexibility to trial new metrics platforms.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_11":{"__typename":"Paragraph","id":"9fe7eb1faed4_11","name":"1aa4","type":"H3","href":null,"layout":null,"metadata":null,"text":"Ecosystem","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_12":{"__typename":"Paragraph","id":"9fe7eb1faed4_12","name":"6dc6","type":"P","href":null,"layout":null,"metadata":null,"text":"We love metrics but we’re not an infrastructure metrics company. By investing in the Prometheus ecosystem we can inherit all the updates and improvements that the community produces (ex: exemplars, native histograms) and free our attention to focus on other things that are important to our business.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_13":{"__typename":"Paragraph","id":"9fe7eb1faed4_13","name":"f66d","type":"H3","href":null,"layout":null,"metadata":null,"text":"Challenges","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_14":{"__typename":"Paragraph","id":"9fe7eb1faed4_14","name":"e91f","type":"P","href":null,"layout":null,"metadata":null,"text":"The challenges of this migration can be placed into three main categories: metrics collection, metrics reshaping, and query rewrites.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_15":{"__typename":"Paragraph","id":"9fe7eb1faed4_15","name":"d4a5","type":"H3","href":null,"layout":null,"metadata":null,"text":"Metrics Collection","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_16":{"__typename":"Paragraph","id":"9fe7eb1faed4_16","name":"07b7","type":"P","href":null,"layout":null,"metadata":null,"text":"The first problem we had to deal with was how to ingest our existing metric samples into a Prometheus metrics stack. We imposed a few requirements for a solution:","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_17":{"__typename":"Paragraph","id":"9fe7eb1faed4_17","name":"65b6","type":"OLI","href":null,"layout":null,"metadata":null,"text":"Metrics must be dual emitted in StatsD and Prometheus protocols during the migration period.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_18":{"__typename":"Paragraph","id":"9fe7eb1faed4_18","name":"14e5","type":"OLI","href":null,"layout":null,"metadata":null,"text":"Services must not be required to re-instrument application code.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_19":{"__typename":"Paragraph","id":"9fe7eb1faed4_19","name":"86a5","type":"OLI","href":null,"layout":null,"metadata":null,"text":"No client-side performance degradation.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_20":{"__typename":"Paragraph","id":"9fe7eb1faed4_20","name":"1dca","type":"OLI","href":null,"layout":null,"metadata":null,"text":"Access to standard Prometheus distribution metric types (summaries, histograms, native histograms).","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_21":{"__typename":"Paragraph","id":"9fe7eb1faed4_21","name":"a52f","type":"P","href":null,"layout":null,"metadata":null,"text":"There’s an existing tool called statsd_exporter maintained by the Prometheus organization that functions great for services emitting standard StatsD protocol. It works by relaying StatsD traffic from your application to your normal StatsD server while also converting metrics to Prometheus protocol and publishing them on a \u002Fmetrics HTTP endpoint.","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"A","start":32,"end":47,"href":"https:\u002F\u002Fgithub.com\u002Fprometheus\u002Fstatsd_exporter","anchorType":"LINK","userId":null,"linkMetadata":null},{"__typename":"Markup","type":"EM","start":324,"end":332,"href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_22":{"__typename":"Paragraph","id":"9fe7eb1faed4_22","name":"3188","type":"IMG","href":null,"layout":"INSET_CENTER","metadata":{"__typename":"ImageMetadata","id":"1*nAsJ1qOnn0c7P993BLivTQ.png","originalHeight":166,"originalWidth":649,"focusPercentX":null,"focusPercentY":null,"alt":null},"text":"Exporter sidecar architecture","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_23":{"__typename":"Paragraph","id":"9fe7eb1faed4_23","name":"975f","type":"P","href":null,"layout":null,"metadata":null,"text":"Unfortunately statsd_exporter didn’t quite fit the bill for us since many of our services aggregate metric samples in-process, meaning we would not have access to Prometheus’s distribution metric types. Whatever solution we came up with would need to be embedded in the client-side process.","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"EM","start":14,"end":29,"href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_24":{"__typename":"Paragraph","id":"9fe7eb1faed4_24","name":"e147","type":"P","href":null,"layout":null,"metadata":null,"text":"So that’s exactly what we did. All our performance-critical services are written in Go, and fortunately statsd_exporter is open source and also written in Go. Our solution was to import statsd_exporter into our internal Go metrics SDK and have it process metric samples directly in memory. This approach also had some additional benefits:","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"STRONG","start":104,"end":119,"href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","type":"STRONG","start":186,"end":201,"href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","type":"EM","start":104,"end":119,"href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","type":"EM","start":186,"end":201,"href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_25":{"__typename":"Paragraph","id":"9fe7eb1faed4_25","name":"d463","type":"OLI","href":null,"layout":null,"metadata":null,"text":"Services could seamlessly start converting to the official Prometheus Go SDK at their own pace.","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"A","start":59,"end":76,"href":"https:\u002F\u002Fgithub.com\u002Fprometheus\u002Fclient_golang","anchorType":"LINK","userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_26":{"__typename":"Paragraph","id":"9fe7eb1faed4_26","name":"4636","type":"OLI","href":null,"layout":null,"metadata":null,"text":"We would have a sidecar-less collection system after the migration.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_27":{"__typename":"Paragraph","id":"9fe7eb1faed4_27","name":"578a","type":"P","href":null,"layout":null,"metadata":null,"text":"After a bit of performance-tuning work we had solution that met all our requirements!","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_28":{"__typename":"Paragraph","id":"9fe7eb1faed4_28","name":"1f54","type":"IMG","href":null,"layout":"INSET_CENTER","metadata":{"__typename":"ImageMetadata","id":"1*uDXTt2XtbxzDnaGG3SU2fQ.png","originalHeight":299,"originalWidth":653,"focusPercentX":null,"focusPercentY":null,"alt":null},"text":"Embedded exporter architecture","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_29":{"__typename":"Paragraph","id":"9fe7eb1faed4_29","name":"3c50","type":"H3","href":null,"layout":null,"metadata":null,"text":"Takeaways","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_30":{"__typename":"Paragraph","id":"9fe7eb1faed4_30","name":"781c","type":"ULI","href":null,"layout":null,"metadata":null,"text":"Metrics collection can be solved ahead of time. It should be dead-simple for service owners to instrument their service for Prometheus.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_31":{"__typename":"Paragraph","id":"9fe7eb1faed4_31","name":"cc13","type":"ULI","href":null,"layout":null,"metadata":null,"text":"statsd_exporter has already solved this problem, so leverage it however you can.","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"EM","start":0,"end":15,"href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_32":{"__typename":"Paragraph","id":"9fe7eb1faed4_32","name":"5518","type":"ULI","href":null,"layout":null,"metadata":null,"text":"Ensure there’s a path for services to start using native Prometheus client SDKs going forward.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_33":{"__typename":"Paragraph","id":"9fe7eb1faed4_33","name":"fff7","type":"H3","href":null,"layout":null,"metadata":null,"text":"Metrics Reshaping","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_34":{"__typename":"Paragraph","id":"9fe7eb1faed4_34","name":"a67b","type":"P","href":null,"layout":null,"metadata":null,"text":"The next challenge we faced was the question of how to reshape our metrics. In this context, the shape of a metric refers to the name, labels, and units of a given metric. While the underlying data models of modern StatsD and Prometheus are virtually identical, they have much different conventions which can have frustrating implications for query ergonomics.","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"A","start":193,"end":204,"href":"https:\u002F\u002Fprometheus.io\u002Fdocs\u002Fconcepts\u002Fdata_model\u002F","anchorType":"LINK","userId":null,"linkMetadata":null},{"__typename":"Markup","type":"STRONG","start":55,"end":62,"href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","type":"EM","start":55,"end":62,"href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_35":{"__typename":"Paragraph","id":"9fe7eb1faed4_35","name":"e962","type":"IMG","href":null,"layout":"INSET_CENTER","metadata":{"__typename":"ImageMetadata","id":"1*w4N3ZMBvcU5DLbTtwoMo4w.png","originalHeight":300,"originalWidth":700,"focusPercentX":null,"focusPercentY":null,"alt":null},"text":"","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_36":{"__typename":"Paragraph","id":"9fe7eb1faed4_36","name":"6b8a","type":"P","href":null,"layout":null,"metadata":null,"text":"StatsD originally did not support tags and would encode every metric dimension in the metric name. While most implementations now support tags, the dot-namespace convention is still widely used and many metrics use a combination of name prefixes and tags to encode dimensions.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_37":{"__typename":"Paragraph","id":"9fe7eb1faed4_37","name":"4fe0","type":"P","href":null,"layout":null,"metadata":null,"text":"Prometheus on the other hand strongly discourages encoding dimensions in the metric name. In fact, certain PromQL queries will completely break when you try to aggregate across dimensions encoded in the metric name, such as the following success rate query:","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"A","start":127,"end":143,"href":"https:\u002F\u002Fwww.robustperception.io\u002Fwhats-in-a-__name__\u002F","anchorType":"LINK","userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_38":{"__typename":"Paragraph","id":"9fe7eb1faed4_38","name":"9949","type":"PRE","href":null,"layout":null,"metadata":null,"text":"sum(rate(\n arb_lqs_grpc_server_LQS_Execute_OK[5m]\n))\n\u002F\nsum(rate(\n {__name__=~\"arb_lqs_grpc_server_LQS_Execute_.+\"}[5m]\n))\n\n\u002F\u002F Error expanding series: vector cannot contain metrics with the same labelset","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":{"__typename":"CodeBlockMetadata","mode":"DISABLED","lang":null},"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_39":{"__typename":"Paragraph","id":"9fe7eb1faed4_39","name":"8805","type":"P","href":null,"layout":null,"metadata":null,"text":"In our initial experiments we quickly realized we had some very important type-1 decisions to make with regards to metric shaping. While it is possible write a query that hacks around this limitation, we absolutely did not want to normalize it for our engineers for years to come. We needed to find a way to move dimensionality entirely into labels, again without requiring engineers to re-instrument application code.","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"A","start":74,"end":90,"href":"https:\u002F\u002Fmedium.com\u002Fthe-mission\u002Fjeff-bezos-says-successful-people-make-these-two-types-of-decisions-c80b333e5d95","anchorType":"LINK","userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_40":{"__typename":"Paragraph","id":"9fe7eb1faed4_40","name":"bf5e","type":"P","href":null,"layout":null,"metadata":null,"text":"The first piece of low-hanging fruit was to replace our RPC metrics interceptors with standard open-source Prometheus interceptors, such as the one provided by grpc-ecosystem\u002Fgo-grpc-middleware:","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"A","start":160,"end":193,"href":"https:\u002F\u002Fgithub.com\u002Fgrpc-ecosystem\u002Fgo-grpc-middleware#observability","anchorType":"LINK","userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_41":{"__typename":"Paragraph","id":"9fe7eb1faed4_41","name":"5ed2","type":"PRE","href":null,"layout":null,"metadata":null,"text":"sum(rate(\n grpc_server_handled_total{\n grpc_service=\"LQS\",\n grpc_method=\"Execute\",\n grpc_code=\"OK\"\n }[5m]\n))\n\u002F\nsum(rate(\n grpc_server_handled_total{\n grpc_service=\"LQS\",\n grpc_method=\"Execute\"\n }[5m]\n))\n\n\u002F\u002F No error!","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":{"__typename":"CodeBlockMetadata","mode":"DISABLED","lang":null},"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_42":{"__typename":"Paragraph","id":"9fe7eb1faed4_42","name":"52fe","type":"P","href":null,"layout":null,"metadata":null,"text":"This one change ended up having a huge impact, since RPC metrics are by far the most queried metrics.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_43":{"__typename":"Paragraph","id":"9fe7eb1faed4_43","name":"f827","type":"P","href":null,"layout":null,"metadata":null,"text":"For other metrics, statsd_exporter provides a config interface for defining metric mappings that uses glob-matching rules to reshape metrics on the fly. However in practice we found that engineers (who have limited time) would prefer to write a hacky query than figure out how to write a mapping rule requiring a full re-deploy of their service. It’s our hope that these remaining problematic metrics will eventually be reshaped and re-instrumented with the native Prometheus SDK.","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"A","start":46,"end":91,"href":"https:\u002F\u002Fgithub.com\u002Fprometheus\u002Fstatsd_exporter#metric-mapping-and-configuration","anchorType":"LINK","userId":null,"linkMetadata":null},{"__typename":"Markup","type":"STRONG","start":19,"end":34,"href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","type":"EM","start":19,"end":34,"href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_44":{"__typename":"Paragraph","id":"9fe7eb1faed4_44","name":"ead5","type":"H3","href":null,"layout":null,"metadata":null,"text":"Takeaways","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_45":{"__typename":"Paragraph","id":"9fe7eb1faed4_45","name":"ff82","type":"ULI","href":null,"layout":null,"metadata":null,"text":"PromQL does not handle name-encoded dimensions well.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_46":{"__typename":"Paragraph","id":"9fe7eb1faed4_46","name":"db90","type":"ULI","href":null,"layout":null,"metadata":null,"text":"Replacing RPC interceptors is a huge win for metric reshaping and requires very little effort.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_47":{"__typename":"Paragraph","id":"9fe7eb1faed4_47","name":"76b0","type":"ULI","href":null,"layout":null,"metadata":null,"text":"Take extra care to figure out how you want to reshape metrics before migrating queries.","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"EM","start":62,"end":68,"href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_48":{"__typename":"Paragraph","id":"9fe7eb1faed4_48","name":"ae8c","type":"H3","href":null,"layout":null,"metadata":null,"text":"Query Rewrites","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_49":{"__typename":"Paragraph","id":"9fe7eb1faed4_49","name":"76d7","type":"P","href":null,"layout":null,"metadata":null,"text":"The final challenge was to rewrite all our dashboard and alert queries in PromQL. Dashboards needed to be migrated from our old vendor’s proprietary format to Grafana, while alerts needed to be migrated to our new vendor’s proprietary format.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_50":{"__typename":"Paragraph","id":"9fe7eb1faed4_50","name":"f501","type":"H3","href":null,"layout":null,"metadata":null,"text":"Can We Automate it?","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_51":{"__typename":"Paragraph","id":"9fe7eb1faed4_51","name":"f62e","type":"P","href":null,"layout":null,"metadata":null,"text":"Initially we looked into tools to automatically rewrite our existing queries to PromQL, but it turns out that’s basically an unsolvable problem. There’s just too many nuances in the differences between time series query languages and their underlying data even within PromQL implementations, let alone across entirely different query language specs.","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"A","start":256,"end":290,"href":"https:\u002F\u002Fpromlabs.com\u002Fblog\u002F2020\u002F08\u002F06\u002Fcomparing-promql-correctness-across-vendors\u002F","anchorType":"LINK","userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_52":{"__typename":"Paragraph","id":"9fe7eb1faed4_52","name":"ce5c","type":"P","href":null,"layout":null,"metadata":null,"text":"While we couldn’t automatically port queries, it technically would’ve been possible to programmatically convert dashboard and alert templates such that engineers only needed to fill in the queries. Unfortunately by the time we thought of this approach it was too late into the migration to have much of an impact.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_53":{"__typename":"Paragraph","id":"9fe7eb1faed4_53","name":"91d4","type":"H3","href":null,"layout":null,"metadata":null,"text":"Reduce the Pain","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_54":{"__typename":"Paragraph","id":"9fe7eb1faed4_54","name":"93fd","type":"P","href":null,"layout":null,"metadata":null,"text":"Since there was no way around the rote-labor of manually rewriting queries, our job became to make that process as painless as possible.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_55":{"__typename":"Paragraph","id":"9fe7eb1faed4_55","name":"7a00","type":"P","href":null,"layout":null,"metadata":null,"text":"The work we put into the first two categories was a great start; Prometheus instrumentation for a service could be enabled with a single feature flag and our metrics were already reshaped and ready to be queried. Additionally, we set up multiple office hours per week where we could help engineers with PromQL questions or issues. We made sure to document everything to help bring all our engineers up-to-speed with PromQL as quickly as possible.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_56":{"__typename":"Paragraph","id":"9fe7eb1faed4_56","name":"06ce","type":"IMG","href":null,"layout":"INSET_CENTER","metadata":{"__typename":"ImageMetadata","id":"1*Y6irDD2KZqyqIvDc-433qQ.png","originalHeight":637,"originalWidth":928,"focusPercentX":null,"focusPercentY":null,"alt":null},"text":"","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_57":{"__typename":"Paragraph","id":"9fe7eb1faed4_57","name":"dcc9","type":"P","href":null,"layout":null,"metadata":null,"text":"In total there were ~300 dashboards and ~600 alerts to migrate totaling to roughly 4,000 query rewrites. Between 20 engineers we were able to complete this phase of the migration in just over 1 month!","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_58":{"__typename":"Paragraph","id":"9fe7eb1faed4_58","name":"893e","type":"H3","href":null,"layout":null,"metadata":null,"text":"Takeaways","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_59":{"__typename":"Paragraph","id":"9fe7eb1faed4_59","name":"47e4","type":"ULI","href":null,"layout":null,"metadata":null,"text":"Make sure you fully understand the amount of work before committing to a migration","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_60":{"__typename":"Paragraph","id":"9fe7eb1faed4_60","name":"0d27","type":"ULI","href":null,"layout":null,"metadata":null,"text":"Automate conversion of dashboard and alert templates if possible.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_61":{"__typename":"Paragraph","id":"9fe7eb1faed4_61","name":"429c","type":"ULI","href":null,"layout":null,"metadata":null,"text":"Become a PromQL expert beforehand and schedule time to impart your knowledge to everyone else.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_62":{"__typename":"Paragraph","id":"9fe7eb1faed4_62","name":"fe24","type":"H3","href":null,"layout":null,"metadata":null,"text":"Final Thoughts","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_63":{"__typename":"Paragraph","id":"9fe7eb1faed4_63","name":"ba54","type":"P","href":null,"layout":null,"metadata":null,"text":"I’m incredibly proud of how efficiently our engineering team was able to execute this migration. I’m also excited to see how the Prometheus ecosystem will evolve over time, and I’m glad to say that Mixpanel will be along for the ride!","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:9fe7eb1faed4_64":{"__typename":"Paragraph","id":"9fe7eb1faed4_64","name":"e7e8","type":"P","href":null,"layout":null,"metadata":null,"text":"Do you love metrics as much as we do? If so, the DevInfra team at Mixpanel is hiring! Apply here and mention this blog post in your application!","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"A","start":92,"end":96,"href":"https:\u002F\u002Fmixpanel.com\u002Fjobs","anchorType":"LINK","userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"CollectionViewerEdge:collectionId:b5de3443ad1f-viewerId:lo_5aa66d57da88":{"__typename":"CollectionViewerEdge","id":"collectionId:b5de3443ad1f-viewerId:lo_5aa66d57da88","isEditor":false,"isMuting":false},"UserViewerEdge:userId:fc8168dcb359-viewerId:lo_5aa66d57da88":{"__typename":"UserViewerEdge","id":"userId:fc8168dcb359-viewerId:lo_5aa66d57da88","isMuting":false},"TagViewerEdge:tagSlug:prometheus-viewerId:lo_5aa66d57da88":{"__typename":"TagViewerEdge","id":"tagSlug:prometheus-viewerId:lo_5aa66d57da88","isMuting":false,"isFollowing":false},"Tag:prometheus":{"__typename":"Tag","viewerEdge":{"__ref":"TagViewerEdge:tagSlug:prometheus-viewerId:lo_5aa66d57da88"},"id":"prometheus","displayTitle":"Prometheus","normalizedTagSlug":"prometheus"},"TagViewerEdge:tagSlug:metrics-viewerId:lo_5aa66d57da88":{"__typename":"TagViewerEdge","id":"tagSlug:metrics-viewerId:lo_5aa66d57da88","isMuting":false,"isFollowing":false},"Tag:metrics":{"__typename":"Tag","viewerEdge":{"__ref":"TagViewerEdge:tagSlug:metrics-viewerId:lo_5aa66d57da88"},"id":"metrics","displayTitle":"Metrics","normalizedTagSlug":"metrics"},"PostViewerEdge:postId:fb973af124f5-viewerId:lo_5aa66d57da88":{"__typename":"PostViewerEdge","hasReposted":false,"id":"postId:fb973af124f5-viewerId:lo_5aa66d57da88","repostNote":null},"Post:fb973af124f5":{"__typename":"Post","id":"fb973af124f5","isPublished":true,"allowResponses":true,"isLimitedState":false,"collection":{"__ref":"Collection:b5de3443ad1f"},"content({\"postMeteringOptions\":{\"referrer\":\"\"}})":{"__typename":"PostContent","isLockedPreviewOnly":false,"bodyModel":{"__typename":"RichText","sections":[{"__typename":"Section","name":"0960","startIndex":0,"textLayout":null,"imageLayout":null,"backgroundImage":null,"videoLayout":null,"backgroundVideo":null}],"paragraphs":[{"__ref":"Paragraph:9fe7eb1faed4_0"},{"__ref":"Paragraph:9fe7eb1faed4_1"},{"__ref":"Paragraph:9fe7eb1faed4_2"},{"__ref":"Paragraph:9fe7eb1faed4_3"},{"__ref":"Paragraph:9fe7eb1faed4_4"},{"__ref":"Paragraph:9fe7eb1faed4_5"},{"__ref":"Paragraph:9fe7eb1faed4_6"},{"__ref":"Paragraph:9fe7eb1faed4_7"},{"__ref":"Paragraph:9fe7eb1faed4_8"},{"__ref":"Paragraph:9fe7eb1faed4_9"},{"__ref":"Paragraph:9fe7eb1faed4_10"},{"__ref":"Paragraph:9fe7eb1faed4_11"},{"__ref":"Paragraph:9fe7eb1faed4_12"},{"__ref":"Paragraph:9fe7eb1faed4_13"},{"__ref":"Paragraph:9fe7eb1faed4_14"},{"__ref":"Paragraph:9fe7eb1faed4_15"},{"__ref":"Paragraph:9fe7eb1faed4_16"},{"__ref":"Paragraph:9fe7eb1faed4_17"},{"__ref":"Paragraph:9fe7eb1faed4_18"},{"__ref":"Paragraph:9fe7eb1faed4_19"},{"__ref":"Paragraph:9fe7eb1faed4_20"},{"__ref":"Paragraph:9fe7eb1faed4_21"},{"__ref":"Paragraph:9fe7eb1faed4_22"},{"__ref":"Paragraph:9fe7eb1faed4_23"},{"__ref":"Paragraph:9fe7eb1faed4_24"},{"__ref":"Paragraph:9fe7eb1faed4_25"},{"__ref":"Paragraph:9fe7eb1faed4_26"},{"__ref":"Paragraph:9fe7eb1faed4_27"},{"__ref":"Paragraph:9fe7eb1faed4_28"},{"__ref":"Paragraph:9fe7eb1faed4_29"},{"__ref":"Paragraph:9fe7eb1faed4_30"},{"__ref":"Paragraph:9fe7eb1faed4_31"},{"__ref":"Paragraph:9fe7eb1faed4_32"},{"__ref":"Paragraph:9fe7eb1faed4_33"},{"__ref":"Paragraph:9fe7eb1faed4_34"},{"__ref":"Paragraph:9fe7eb1faed4_35"},{"__ref":"Paragraph:9fe7eb1faed4_36"},{"__ref":"Paragraph:9fe7eb1faed4_37"},{"__ref":"Paragraph:9fe7eb1faed4_38"},{"__ref":"Paragraph:9fe7eb1faed4_39"},{"__ref":"Paragraph:9fe7eb1faed4_40"},{"__ref":"Paragraph:9fe7eb1faed4_41"},{"__ref":"Paragraph:9fe7eb1faed4_42"},{"__ref":"Paragraph:9fe7eb1faed4_43"},{"__ref":"Paragraph:9fe7eb1faed4_44"},{"__ref":"Paragraph:9fe7eb1faed4_45"},{"__ref":"Paragraph:9fe7eb1faed4_46"},{"__ref":"Paragraph:9fe7eb1faed4_47"},{"__ref":"Paragraph:9fe7eb1faed4_48"},{"__ref":"Paragraph:9fe7eb1faed4_49"},{"__ref":"Paragraph:9fe7eb1faed4_50"},{"__ref":"Paragraph:9fe7eb1faed4_51"},{"__ref":"Paragraph:9fe7eb1faed4_52"},{"__ref":"Paragraph:9fe7eb1faed4_53"},{"__ref":"Paragraph:9fe7eb1faed4_54"},{"__ref":"Paragraph:9fe7eb1faed4_55"},{"__ref":"Paragraph:9fe7eb1faed4_56"},{"__ref":"Paragraph:9fe7eb1faed4_57"},{"__ref":"Paragraph:9fe7eb1faed4_58"},{"__ref":"Paragraph:9fe7eb1faed4_59"},{"__ref":"Paragraph:9fe7eb1faed4_60"},{"__ref":"Paragraph:9fe7eb1faed4_61"},{"__ref":"Paragraph:9fe7eb1faed4_62"},{"__ref":"Paragraph:9fe7eb1faed4_63"},{"__ref":"Paragraph:9fe7eb1faed4_64"}]},"validatedShareKey":"","shareKeyCreator":null},"creator":{"__ref":"User:fc8168dcb359"},"inResponseToEntityType":null,"isLocked":false,"isMarkedPaywallOnly":false,"lockedSource":"LOCKED_POST_SOURCE_NONE","mediumUrl":"https:\u002F\u002Fmedium.com\u002Fmixpaneleng\u002Fhow-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5","primaryTopic":null,"topics":[{"__typename":"Topic","slug":"programming"}],"responsesLocked":false,"visibility":"PUBLIC","latestPublishedVersion":"9fe7eb1faed4","postResponses":{"__typename":"PostResponses","count":1},"responseDistribution":"NOT_DISTRIBUTED","clapCount":5,"title":"How We Migrated from StatsD to Prometheus in One Month","isSeries":false,"sequence":null,"uniqueSlug":"how-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5","tags":[{"__ref":"Tag:prometheus"},{"__ref":"Tag:metrics"}],"inResponseToPostResult":null,"inResponseToCatalogResult":null,"viewerEdge":{"__ref":"PostViewerEdge:postId:fb973af124f5-viewerId:lo_5aa66d57da88"},"extendedPreviewContent":{"__typename":"PreviewContent","subtitle":"","isFullContent":false},"previewImage":{"__typename":"ImageMetadata","id":"1*uDXTt2XtbxzDnaGG3SU2fQ.png","focusPercentX":null,"focusPercentY":null,"alt":null,"originalHeight":299,"originalWidth":653},"firstPublishedAt":1695952446689,"latestPublishedAt":1747253485049,"pinnedAt":0,"socialTitle":"","socialDek":"","canonicalUrl":"https:\u002F\u002Fmedium.com\u002Fmixpaneleng\u002Fhow-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5","metaDescription":"","readingTime":7.122641509433962,"previewContent":{"__typename":"PreviewContent","subtitle":""},"isShortform":false,"seoMetaTags":{"__typename":"SEOMetaTags","jsonLd":"{\"@context\":\"https:\u002F\u002Fschema.org\",\"@id\":\"https:\u002F\u002Fmedium.com\u002Fmixpaneleng\u002Fhow-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5\",\"@type\":\"SocialMediaPosting\",\"image\":[\"https:\u002F\u002Fmiro.medium.com\u002F1*uDXTt2XtbxzDnaGG3SU2fQ.png\"],\"url\":\"https:\u002F\u002Fmedium.com\u002Fmixpaneleng\u002Fhow-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5\",\"dateCreated\":\"2023-09-29T01:54:06Z\",\"datePublished\":\"2023-09-29T01:54:06Z\",\"dateModified\":\"2025-05-14T20:11:25Z\",\"headline\":\"How We Migrated from StatsD to Prometheus in One Month\",\"name\":\"How We Migrated from StatsD to Prometheus in One Month\",\"description\":\"How We Migrated from StatsD to Prometheus in One Month We recently migrated all of our infrastructure metrics from StatsD to Prometheus and are very pleased with the results. The migration was a ton …\",\"identifier\":\"fb973af124f5\",\"author\":{\"@context\":\"https:\u002F\u002Fschema.org\",\"@id\":\"https:\u002F\u002Fmedium.com\u002F@eddie.bracho\",\"@type\":\"Person\",\"identifier\":\"eddie.bracho\",\"name\":\"Eddie Bracho\",\"url\":\"https:\u002F\u002Fmedium.com\u002F@eddie.bracho\"},\"creator\":{\"@context\":\"https:\u002F\u002Fschema.org\",\"@id\":\"https:\u002F\u002Fmedium.com\u002F@eddie.bracho\",\"@type\":\"Person\",\"identifier\":\"eddie.bracho\",\"name\":\"Eddie Bracho\",\"url\":\"https:\u002F\u002Fmedium.com\u002F@eddie.bracho\"},\"publisher\":{\"@context\":\"https:\u002F\u002Fschema.org\",\"@type\":\"Organization\",\"@id\":\"https:\u002F\u002Fmedium.com\u002Fmixpaneleng\",\"name\":\"Mixpanel Engineering\",\"description\":\"Blog is now hosted at https:\u002F\u002Fmixpanel.substack.com\",\"url\":\"https:\u002F\u002Fmedium.com\u002Fmixpaneleng\",\"sameAs\":[\"https:\u002F\u002Ftwitter.com\u002Fmixpanel\"],\"logo\":{\"@type\":\"ImageObject\",\"width\":280,\"height\":280,\"url\":\"https:\u002F\u002Fmiro.medium.com\u002Fv2\u002Fresize:fit:280\u002F1%2AwdBmRrZ2IldG8fNVhER_xg.png\"}},\"mainEntityOfPage\":\"https:\u002F\u002Fmedium.com\u002Fmixpaneleng\u002Fhow-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5\",\"isAccessibleForFree\":true}"},"seoDescription":"How We Migrated from StatsD to Prometheus in One Month We recently migrated all of our infrastructure metrics from StatsD to Prometheus and are very pleased with the results. The migration was a ton …","shortformType":"SHORTFORM_TYPE_LINK","seoTitle":"","isSuspended":false,"license":"ALL_RIGHTS_RESERVED","isFeaturedInPublishedPublication":false,"isNewsletter":false,"statusForCollection":"APPROVED","pendingCollection":null,"detectedLanguage":"en","wordCount":1702}};</script><script>window.__GRAPHQL_URI__ = "\u002F_\u002Fgraphql";</script><script>window.__PRELOADED_STATE__ = {"algolia":{"queries":{}},"cache":{"experimentGroupSet":true,"reason":"","group":"enabled","tags":["group-edgeCachePosts","group-enableDarkModeV0","group-enableStoryPageWriterBio","group-enableUserCustomFooter","post-fb973af124f5","user-fc8168dcb359","collection-b5de3443ad1f"],"serverVariantState":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","middlewareEnabled":true,"cacheStatus":"DYNAMIC","shouldUseCache":true,"vary":[],"enableDarkModeV0":false,"storyPageWriterBioEnabled":"control","expandedTopicPagesEnabled":"experiment","exposureBatteryX3Enabled":"control","enableUserCustomFooter":false},"client":{"hydrated":false,"isUs":false,"isNativeMedium":false,"isSafariMobile":false,"isSafari":false,"isFirefox":false,"routingEntity":{"type":"DEFAULT","explicit":false},"viewerIsBot":false,"clientScore":44},"debug":{"requestId":"d5ea9fd4-5a04-4dd6-bc10-a4cbcd73d698","requestTag":"","hybridDevServices":[]},"multiVote":{"clapsPerEntity":{}},"navigation":{"hideGoogleOneTap":false,"hasRenderedAlternateUserBanner":null,"currentLocation":"https:\u002F\u002Fmedium.com\u002Fmixpaneleng\u002Fhow-we-migrated-from-statsd-to-prometheus-in-one-month-fb973af124f5","host":"medium.com","hostname":"medium.com","referrer":"","hasSetReferrer":false,"susiModal":{"step":null,"operation":"register","rememberMe":true},"postRead":false,"paymentFunnel":{},"partnerProgram":{"selectedCountryCode":null},"staticRouterContext":{"route":{"name":"ShowPostUnderCollection"},"statusCode":200},"toastQueue":[],"currentToast":null,"queryString":"","currentHash":""},"config":{"nodeEnv":"production","version":"main-20260911-235047-8654e8124b","target":"production","productName":"Medium","publicUrl":"https:\u002F\u002Fcdn-client.medium.com\u002Flite","authDomain":"medium.com","authGoogleClientId":"216296035834-k1k6qe060s2tp2a2jam4ljdcms00sttg.apps.googleusercontent.com","favicon":"production","iosAppId":"828256236","glyphUrl":"https:\u002F\u002Fglyph.medium.com","branchKey":"key_live_ofxXr2qTrrU9NqURK8ZwEhknBxiI6KBm","algolia":{"appId":"MQ57UUUQZ2","apiKeySearch":"394474ced050e3911ae2249ecc774921","indexPrefix":"medium_","host":"-dsn.algolia.net"},"recaptchaKey":"6Lfc37IUAAAAAKGGtC6rLS13R1Hrw_BqADfS1LRk","recaptcha3Key":"6Lf8R9wUAAAAABMI_85Wb8melS7Zj6ziuf99Yot5","recaptchaEnterpriseKeyId":"6Le-uGgpAAAAAPprRaokM8AKthQ9KNGdoxaGUvVp","datadog":{"applicationId":"6702d87d-a7e0-42fe-bbcb-95b469547ea0","clientToken":"pub853ea8d17ad6821d9f8f11861d23dfed","rumToken":"pubf9cc52896502b9413b68ba36fc0c7162","context":{"deployment":{"target":"production","tag":"main-20260911-235047-8654e8124b","commit":"8654e8124b64da937b7a19894acb51fdc2d0b94d"}},"datacenter":"us"},"googleAdsCode":"AW-17106321204","googleAnalyticsCode":"G-7JY7T788PK","googlePay":{"apiVersion":"2","apiVersionMinor":"0","merchantId":"BCR2DN6TV7EMTGBM","merchantName":"Medium","instanceMerchantId":"13685562959212738550"},"applePay":{"version":3},"signInWallCustomDomainCollectionIds":["3a8144eabfe3","336d898217ee","61061eb0c96b","138adf9c44c","819cc2aaeee0"],"mediumMastodonDomainName":"me.dm","mediumOwnedAndOperatedCollectionIds":["8a9336e5bb4","b7e45b22fec3","193b68bd4fba","8d6b8a439e32","54c98c43354d","3f6ecf56618","d944778ce714","92d2092dc598","ae2a65f35510","1285ba81cada","544c7006046e","fc8964313712","40187e704f1c","88d9857e584e","7b6769f2748b","bcc38c8f6edf","cef6983b292","cb8577c9149e","444d13b52878","713d7dbc99b0","ef8e90590e66","191186aaafa0","55760f21cdc5","9dc80918cc93","bdc4052bbdba","8ccfed20cbb2"],"tierOneDomains":["medium.com","thebolditalic.com","arcdigital.media","towardsdatascience.com","uxdesign.cc","codeburst.io","psiloveyou.xyz","writingcooperative.com","entrepreneurshandbook.co","prototypr.io","betterhumans.coach.me","theascent.pub"],"topicsToFollow":["d61cf867d93f","8a146bc21b28","1eca0103fff3","4d562ee63426","aef1078a3ef5","e15e46793f8d","6158eb913466","55f1c20aba7a","3d18b94f6858","4861fee224fd","63c6f1f93ee","1d98b3a9a871","decb52b64abf","ae5d4995e225","830cded25262"],"topicToTagMappings":{"accessibility":"accessibility","addiction":"addiction","android-development":"android-development","art":"art","artificial-intelligence":"artificial-intelligence","astrology":"astrology","basic-income":"basic-income","beauty":"beauty","biotech":"biotech","blockchain":"blockchain","books":"books","business":"business","cannabis":"cannabis","cities":"cities","climate-change":"climate-change","comics":"comics","coronavirus":"coronavirus","creativity":"creativity","cryptocurrency":"cryptocurrency","culture":"culture","cybersecurity":"cybersecurity","data-science":"data-science","design":"design","digital-life":"digital-life","disability":"disability","economy":"economy","education":"education","equality":"equality","family":"family","feminism":"feminism","fiction":"fiction","film":"film","fitness":"fitness","food":"food","freelancing":"freelancing","future":"future","gadgets":"gadgets","gaming":"gaming","gun-control":"gun-control","health":"health","history":"history","humor":"humor","immigration":"immigration","ios-development":"ios-development","javascript":"javascript","justice":"justice","language":"language","leadership":"leadership","lgbtqia":"lgbtqia","lifestyle":"lifestyle","machine-learning":"machine-learning","makers":"makers","marketing":"marketing","math":"math","media":"media","mental-health":"mental-health","mindfulness":"mindfulness","money":"money","music":"music","neuroscience":"neuroscience","nonfiction":"nonfiction","outdoors":"outdoors","parenting":"parenting","pets":"pets","philosophy":"philosophy","photography":"photography","podcasts":"podcast","poetry":"poetry","politics":"politics","privacy":"privacy","product-management":"product-management","productivity":"productivity","programming":"programming","psychedelics":"psychedelics","psychology":"psychology","race":"race","relationships":"relationships","religion":"religion","remote-work":"remote-work","san-francisco":"san-francisco","science":"science","self":"self","self-driving-cars":"self-driving-cars","sexuality":"sexuality","social-media":"social-media","society":"society","software-engineering":"software-engineering","space":"space","spirituality":"spirituality","sports":"sports","startups":"startup","style":"style","technology":"technology","transportation":"transportation","travel":"travel","true-crime":"true-crime","tv":"tv","ux":"ux","venture-capital":"venture-capital","visual-design":"visual-design","work":"work","world":"world","writing":"writing"},"defaultImages":{"avatar":{"imageId":"1*dmbNkD5D-u45r44go_cf0g.png","height":150,"width":150},"orgLogo":{"imageId":"7*V1_7XP4snlmqrc_0Njontw.png","height":110,"width":500},"postLogo":{"imageId":"167cff2a3d17ac1e64d0762539978f2d54c0058886e8b3c8a03a725a83012ec0","height":630,"width":1200},"postPreviewImage":{"imageId":"bc1f8416df0cad099e43cda2872716e5864f18a73bda2a7547ea082aca9b5632","height":630,"width":1200}},"embeddedPostIds":{"coronavirus":"cd3010f9d81f"},"sharedCdcMessaging":{"COVID_APPLICABLE_TAG_SLUGS":[],"COVID_APPLICABLE_TOPIC_NAMES":[],"COVID_APPLICABLE_TOPIC_NAMES_FOR_TOPIC_PAGE":[],"COVID_MESSAGES":{"tierA":{"text":"For more information on the novel coronavirus and Covid-19, visit cdc.gov.","markups":[{"start":66,"end":73,"href":"https:\u002F\u002Fwww.cdc.gov\u002Fcoronavirus\u002F2019-nCoV"}]},"tierB":{"text":"Anyone can publish on Medium per our Policies, but we don’t fact-check every story. For more info about the coronavirus, see cdc.gov.","markups":[{"start":37,"end":45,"href":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Fcategories\u002F201931128-Policies-Safety"},{"start":125,"end":132,"href":"https:\u002F\u002Fwww.cdc.gov\u002Fcoronavirus\u002F2019-nCoV"}]},"paywall":{"text":"This article has been made free for everyone, thanks to Medium Members. For more information on the novel coronavirus and Covid-19, visit cdc.gov.","markups":[{"start":56,"end":70,"href":"https:\u002F\u002Fmedium.com\u002Fmembership"},{"start":138,"end":145,"href":"https:\u002F\u002Fwww.cdc.gov\u002Fcoronavirus\u002F2019-nCoV"}]},"unbound":{"text":"This article is free for everyone, thanks to Medium Members. For more information on the novel coronavirus and Covid-19, visit cdc.gov.","markups":[{"start":45,"end":59,"href":"https:\u002F\u002Fmedium.com\u002Fmembership"},{"start":127,"end":134,"href":"https:\u002F\u002Fwww.cdc.gov\u002Fcoronavirus\u002F2019-nCoV"}]}},"COVID_BANNER_POST_ID_OVERRIDE_WHITELIST":["3b31a67bff4a"]},"sharedVoteMessaging":{"TAGS":["politics","election-2020","government","us-politics","election","2020-presidential-race","trump","donald-trump","democrats","republicans","congress","republican-party","democratic-party","biden","joe-biden","maga"],"TOPICS":["politics","election"],"MESSAGE":{"text":"Find out more about the U.S. election results here.","markups":[{"start":46,"end":50,"href":"https:\u002F\u002Fcookpolitical.com\u002F2020-national-popular-vote-tracker"}]},"EXCLUDE_POSTS":["397ef29e3ca5"]},"embedPostRules":[],"recircOptions":{"v1":{"limit":3},"v2":{"limit":8}},"braintreeClientKey":"production_zjkj96jm_m56f8fqpf7ngnrd4","braintree":{"enabled":true,"merchantId":"m56f8fqpf7ngnrd4","merchantAccountId":{"usd":"AMediumCorporation_instant","eur":"amediumcorporation_EUR","cad":"amediumcorporation_CAD"},"publicKey":"ds2nn34bg2z7j5gd","braintreeEnvironment":"production","dashboardUrl":"https:\u002F\u002Fwww.braintreegateway.com\u002Fmerchants","gracePeriodDurationInDays":14,"mediumMembershipPlanId":{"monthly":"ce105f8c57a3","monthlyV2":"e8a5e126-792b-4ee6-8fba-d574c1b02fc5","monthlyWithTrial":"d5ee3dbe3db8","monthlyPremium":"fa741a9b47a2","yearly":"a40ad4a43185","yearlyV2":"3815d7d6-b8ca-4224-9b8c-182f9047866e","yearlyStaff":"d74fb811198a","yearlyWithTrial":"b3bc7350e5c7","yearlyPremium":"e21bd2c12166","monthlyOneYearFree":"e6c0637a-2bad-4171-ab4f-3c268633d83c","monthly25PercentOffFirstYear":"235ecc62-0cdb-49ae-9378-726cd21c504b","monthly20PercentOffFirstYear":"ba518864-9c13-4a99-91ca-411bf0cac756","monthly15PercentOffFirstYear":"594c029b-9f89-43d5-88f8-8173af4e070e","monthly10PercentOffFirstYear":"c6c7bc9a-40f2-4b51-8126-e28511d5bdb0","monthlyForStudents":"629ebe51-da7d-41fd-8293-34cd2f2030a8","yearlyOneYearFree":"78ba7be9-0d9f-4ece-aa3e-b54b826f2bf1","yearly25PercentOffFirstYear":"2dbb010d-bb8f-4eeb-ad5c-a08509f42d34","yearly20PercentOffFirstYear":"47565488-435b-47f8-bf93-40d5fbe0ebc8","yearly15PercentOffFirstYear":"8259809b-0881-47d9-acf7-6c001c7f720f","yearly10PercentOffFirstYear":"9dd694fb-96e1-472c-8d9e-3c868d5c1506","yearlyForStudents":"e29345ef-ab1c-4234-95c5-70e50fe6bc23","monthlyCad":"p52orjkaceei","yearlyCad":"h4q9g2up9ktt"},"braintreeDiscountId":{"oneMonthFree":"MONTHS_FREE_01","threeMonthsFree":"MONTHS_FREE_03","sixMonthsFree":"MONTHS_FREE_06","fiftyPercentOffOneYear":"FIFTY_PERCENT_OFF_ONE_YEAR"},"3DSecureVersion":"2","defaultCurrency":"usd","providerPlanIdCurrency":{"4ycw":"usd","rz3b":"usd","3kqm":"usd","jzw6":"usd","c2q2":"usd","nnsw":"usd","q8qw":"usd","d9y6":"usd","fx7w":"cad","nwf2":"cad"}},"paypalClientId":"AXj1G4fotC2GE8KzWX9mSxCH1wmPE3nJglf4Z2ig_amnhvlMVX87otaq58niAg9iuLktVNF_1WCMnN7v","paypal":{"host":"https:\u002F\u002Fapi.paypal.com:443","clientMode":"production","serverMode":"live","webhookId":"4G466076A0294510S","monthlyPlan":{"planId":"P-9WR0658853113943TMU5FDQA","name":"Medium Membership (Monthly) with setup fee","description":"Unlimited access to the best and brightest stories on Medium. Membership billed monthly."},"yearlyPlan":{"planId":"P-7N8963881P8875835MU5JOPQ","name":"Medium Membership (Annual) with setup fee","description":"Unlimited access to the best and brightest stories on Medium. Membership billed annually."},"oneYearGift":{"name":"Medium Membership (1 Year, Digital Gift Code)","description":"Unlimited access to the best and brightest stories on Medium. Gift codes can be redeemed at medium.com\u002Fredeem.","price":"50.00","currency":"USD","sku":"membership-gift-1-yr"},"oldMonthlyPlan":{"planId":"P-96U02458LM656772MJZUVH2Y","name":"Medium Membership (Monthly)","description":"Unlimited access to the best and brightest stories on Medium. Membership billed monthly."},"oldYearlyPlan":{"planId":"P-59P80963JF186412JJZU3SMI","name":"Medium Membership (Annual)","description":"Unlimited access to the best and brightest stories on Medium. Membership billed annually."},"monthlyPlanWithTrial":{"planId":"P-66C21969LR178604GJPVKUKY","name":"Medium Membership (Monthly) with setup fee","description":"Unlimited access to the best and brightest stories on Medium. Membership billed monthly."},"yearlyPlanWithTrial":{"planId":"P-6XW32684EX226940VKCT2MFA","name":"Medium Membership (Annual) with setup fee","description":"Unlimited access to the best and brightest stories on Medium. Membership billed annually."},"oldMonthlyPlanNoSetupFee":{"planId":"P-4N046520HR188054PCJC7LJI","name":"Medium Membership (Monthly)","description":"Unlimited access to the best and brightest stories on Medium. Membership billed monthly."},"oldYearlyPlanNoSetupFee":{"planId":"P-7A4913502Y5181304CJEJMXQ","name":"Medium Membership (Annual)","description":"Unlimited access to the best and brightest stories on Medium. Membership billed annually."},"sdkUrl":"https:\u002F\u002Fwww.paypal.com\u002Fsdk\u002Fjs"},"stripePublishableKey":"pk_live_7FReX44VnNIInZwrIIx6ghjl","log":{"json":true,"level":"info"},"imageUploadMaxSizeMb":25,"staffPicks":{"title":"Staff Picks","catalogId":"c7bc6e1ee00f"}},"session":{"xsrf":""}};</script><script>window.__BUILD_ID__ = "main-20260911-235047-8654e8124b";</script><script>window.__MIDDLEWARE_STATE__={"session":{"xsrf":""},"cache":{"cacheStatus":"BYPASS"}}</script><script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" data-cf-beacon='{"version":"2024.11.0","token":"0b5f665943484354a59c39c6833f7078","spa":2}' crossorigin="anonymous"></script>
<script>window.__CF$cv$params={r:'a39cd41d2dbfe172',t:'MTc4OTE5NDI2MA==',u:'01a09449d2867ed195ba0f9e7301845f',ut:'xP35d4mcaUU8OKoUGW7nG01u0eMWY9QWI6ciiFRVbA0-1789194261-1.2.1.1-ALDC00OYMUCvJ58F7DUdhdTq0JVCT7XzKMKYO4ZPHPVzQxBD3eQ6qeGWJ1JYy1_BX9EOzNhyiCtigaUDYu9I_3dCYF_MDQ9xhpEjWbfzg7Q',i:60};(function(){if(!document.body)return;var s=document.createElement('script');s.src='/cdn-cgi/challenge-platform/scripts/precursor/main.js';document.head.appendChild(s);})();</script></body></html>