Files
nexus/sreweekly/articles/82/07-high-availability-prometheus-alerting-and-notification.html
2026-09-12 17:23:01 +08:00

436 lines
42 KiB
HTML
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html data-semplice="4.2.2">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0">
<title>High Availability Prometheus Alerting and Notification &#8211; Robust Perception | Prometheus Monitoring Experts</title>
<meta name="robots" content="max-image-preview:large">
<link rel="dns-prefetch" href="//s.w.org">
<link rel="stylesheet" id="wp-block-library-css" href="/wp-includes/css/dist/block-library/style.min.css?ver=5.9.3" type="text/css" media="all">
<style id="global-styles-inline-css" type="text/css">body{--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgba(6,147,227,1) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgba(252,185,0,1) 0%,rgba(255,105,0,1) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgba(255,105,0,1) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--duotone--dark-grayscale: url('#wp-duotone-dark-grayscale');--wp--preset--duotone--grayscale: url('#wp-duotone-grayscale');--wp--preset--duotone--purple-yellow: url('#wp-duotone-purple-yellow');--wp--preset--duotone--blue-red: url('#wp-duotone-blue-red');--wp--preset--duotone--midnight: url('#wp-duotone-midnight');--wp--preset--duotone--magenta-yellow: url('#wp-duotone-magenta-yellow');--wp--preset--duotone--purple-green: url('#wp-duotone-purple-green');--wp--preset--duotone--blue-orange: url('#wp-duotone-blue-orange');--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}</style>
<link rel="stylesheet" id="ppress-frontend-css" href="/wp-content/plugins/wp-user-avatar/assets/css/frontend.min.css?ver=3.1.19" type="text/css" media="all">
<link rel="stylesheet" id="ppress-flatpickr-css" href="/wp-content/plugins/wp-user-avatar/assets/flatpickr/flatpickr.min.css?ver=3.1.19" type="text/css" media="all">
<link rel="stylesheet" id="ppress-select2-css" href="/wp-content/plugins/wp-user-avatar/assets/select2/select2.min.css?ver=5.9.3" type="text/css" media="all">
<link rel="stylesheet" id="semplice-stylesheet-css" href="/wp-content/themes/semplice4/style.css?ver=4.2.2" type="text/css" media="all">
<link rel="stylesheet" id="semplice-frontend-stylesheet-css" href="/wp-content/themes/semplice4/assets/css/frontend.min.css?ver=4.2.2" type="text/css" media="all">
<link rel="stylesheet" id="mediaelement-css" href="/wp-includes/js/mediaelement/mediaelementplayer-legacy.min.css?ver=4.2.16" type="text/css" media="all">
<script type="text/javascript" src="/wp-includes/js/jquery/jquery.min.js?ver=3.6.0" id="jquery-core-js"></script>
<script type="text/javascript" src="/wp-includes/js/jquery/jquery-migrate.min.js?ver=3.3.2" id="jquery-migrate-js"></script>
<script type="text/javascript" src="/wp-content/plugins/wp-user-avatar/assets/flatpickr/flatpickr.min.js?ver=5.9.3" id="ppress-flatpickr-js"></script>
<script type="text/javascript" src="/wp-content/plugins/wp-user-avatar/assets/select2/select2.min.js?ver=5.9.3" id="ppress-select2-js"></script>
<link rel="https://api.w.org/" href="/wp-json/">
<link rel="alternate" type="application/json" href="/wp-json/wp/v2/posts/3014">
<link rel="EditURI" type="application/rsd+xml" title="RSD" href="/xmlrpc.php?rsd">
<link rel="wlwmanifest" type="application/wlwmanifest+xml" href="/wp-includes/wlwmanifest.xml">
<meta name="generator" content="WordPress 5.9.3">
<link rel="canonical" href="/high-availability-prometheus-alerting-and-notification">
<link rel="shortlink" href="/?p=3014">
<link rel="alternate" type="application/json+oembed" href="/wp-json/oembed/1.0/embed?url=http%3A%2F%2F%2Fhigh-availability-prometheus-alerting-and-notification">
<link rel="alternate" type="text/xml+oembed" href="/wp-json/oembed/1.0/embed?url=http%3A%2F%2F%2Fhigh-availability-prometheus-alerting-and-notification&#038;format=xml">
<link href="https://fonts.googleapis.com/css?family=Inconsolata" rel="stylesheet">
<style type="text/css" id="semplice-webfonts-css">.font_21cb0s4ud, [data-font="font_21cb0s4ud"], [data-font="font_21cb0s4ud"] li a, #content-holder p, #content-holder li {font-family: "inconsolata", sans-serif;font-weight: 400;font-style: normal;}.font_7g535ae2g, [data-font="font_7g535ae2g"], [data-font="font_7g535ae2g"] li a, #content-holder h1, #content-holder h2, #content-holder h3, #content-holder h4, #content-holder h5, #content-holder h6 {font-family: "inconsolata-bold", sans-serif;font-weight: 800;font-style: normal;}</style>
<style type="text/css" id="semplice-custom-css">.container {
max-width: 1230px;
}
@media screen and (min-width: 992px) and (max-width: 1169px) { #content-holder h1 { font-size: 2.15rem; line-height: 2.76rem; }#content-holder h2 { font-size: 1.84rem; line-height: 2.45rem; }#content-holder h3 { font-size: 1.49rem; line-height: 1.92rem; }#content-holder h4 { font-size: 1.28rem; line-height: 1.92rem; }#content-holder h5 { font-size: 1.11rem; line-height: 1.78rem; }#content-holder h6 { font-size: 1rem; line-height: 1.67rem; }}@media screen and (min-width: 768px) and (max-width: 991px) { #content-holder h1 { font-size: 2.01rem; line-height: 2.58rem; }#content-holder h2 { font-size: 1.76rem; line-height: 2.35rem; }#content-holder h3 { font-size: 1.43rem; line-height: 1.84rem; }#content-holder h4 { font-size: 1.23rem; line-height: 1.84rem; }#content-holder h5 { font-size: 1.11rem; line-height: 1.78rem; }#content-holder h6 { font-size: 1rem; line-height: 1.67rem; }}@media screen and (min-width: 544px) and (max-width: 767px) { #content-holder h1 { font-size: 1.91rem; line-height: 2.46rem; }#content-holder h2 { font-size: 1.68rem; line-height: 2.24rem; }#content-holder h3 { font-size: 1.4rem; line-height: 1.8rem; }#content-holder h4 { font-size: 1.2rem; line-height: 1.8rem; }#content-holder h5 { font-size: 1.11rem; line-height: 1.78rem; }#content-holder h6 { font-size: 1rem; line-height: 1.67rem; }}@media screen and (max-width: 543px) { #content-holder h1 { font-size: 1.82rem; line-height: 2.34rem; }#content-holder h2 { font-size: 1.6rem; line-height: 2.13rem; }#content-holder h3 { font-size: 1.37rem; line-height: 1.76rem; }#content-holder h4 { font-size: 1.17rem; line-height: 1.76rem; }#content-holder h5 { font-size: 1.11rem; line-height: 1.78rem; }#content-holder h6 { font-size: 1rem; line-height: 1.67rem; }}
#comments {
display: none !important;
}
pre, code {
background: #31302F;
color: white;
}
code {
white-space: nowrap;
}
.posts table td, .posts table th {
line-height: inherit;
}
.post table {
display: block;
overflow: auto;
}
.is-frontend div.posts {
padding-top: 1rem;
}
.post-divider {
margin-top: 3rem;
margin-bottom: 3rem;
}
.project-panel {
background: #f5f5f5;
padding: 2.5rem 0rem;
}
.pp-thumbs {
margin-bottom: -1.666666666666667rem;
}
#content-holder .panel-label, .projectpanel-preview .panel-label {
color: #000000;
font-size: 1.777777777777778rem;
text-transform: none;
padding-left: 0rem;
padding-bottom: 1.666666666666667rem;
text-align: left;
line-height: 1;
}
.project-panel .pp-title {
padding: 0.5555555555555556rem 0rem 1.666666666666667rem 0rem;
}
.project-panel .pp-title a {
color: #000000;
font-size: 0.7222222222222222rem;
text-transform: none;
}
.project-panel .pp-title span {
color: #999999;
font-size: 0.7222222222222222rem;
text-transform: none;
}</style>
<style type="text/css" id="3014-post-css">.post .row { justify-content: center; }
#comments form#commentform input, #comments form#commentform #submit, #comments form#commentform textarea,
#comments form#commentform input:hover, #comments form#commentform #submit:hover, #comments form#commentform textarea:hover,
#comments form#commentform input:focus, #comments form#commentform #submit:focus, #comments form#commentform textarea:focus {
}#content-3014 #section_d861b0e30 {padding-top: 3.5555555555555554rem;padding-bottom: 3.5555555555555554rem;background-color: #262626;}#content-3014 #content_e266ccdba {padding-top: 0rem;padding-bottom: 0rem;}#content-3014 #content_e266ccdba .is-content {box-shadow: 0rem 0rem 0rem 0rem rgba(0,0,0,1);}#content-3014 #column_2079c0b8c {padding-top: 1.7777777777777777rem;padding-bottom: 1.7777777777777777rem;}
.nav_4ndms1087 { background-color: rgba(255, 255, 255, 1);; }.nav_4ndms1087 { height: 4.444444444444445rem; }.is-frontend #content-3014 .sections { margin-top: 4.444444444444445rem; }.nav_4ndms1087 { padding-top: 0rem; }.nav_4ndms1087 { padding-bottom: 0rem; }.nav_4ndms1087 .navbar-inner .logo { margin-top: 0rem; }.nav_4ndms1087 .logo img, .nav_4ndms1087 .logo svg { width: 14.222222222222221rem; }.nav_4ndms1087 .navbar-inner .logo { align-items: center; }.nav_4ndms1087 .navbar-inner .hamburger a.menu-icon span { background-color: #000000; }.nav_4ndms1087 .navbar-inner .hamburger a.menu-icon { width: 24; }.nav_4ndms1087 .navbar-inner .hamburger a.menu-icon span { height: 2px; }.nav_4ndms1087 .navbar-inner .hamburger a.open-menu span::before { transform: translateY(-6px); }.nav_4ndms1087 .navbar-inner .hamburger a.open-menu span::after { transform: translateY(6px); }.nav_4ndms1087 .navbar-inner .hamburger a.open-menu:hover span::before { transform: translateY(-8px); }.nav_4ndms1087 .navbar-inner .hamburger a.open-menu:hover span::after { transform: translateY(8px); }.nav_4ndms1087 .navbar-inner .hamburger a.menu-icon { height: 14px; }.nav_4ndms1087 .navbar-inner .hamburger a.menu-icon span { margin-top: 7px; }.nav_4ndms1087 .navbar-inner nav ul li a span { font-size: 1.0555555555555556rem; }.nav_4ndms1087 .navbar-inner nav ul li a span { color: #262626; }.nav_4ndms1087 .navbar-inner nav ul li a:hover span, .navbar-inner nav ul li.current-menu-item a span, .navbar-inner nav ul li.current_page_item a span { color: #0081a5; }.nav_4ndms1087 .navbar-inner nav ul li.current-menu-item a span { color: #0081a5; }.nav_4ndms1087 .navbar-inner nav ul li.current_page_item a span { color: #0081a5; }[data-post-type="project"] .navbar-inner nav ul li.portfolio-grid a span, [data-post-type="post"] .navbar-inner nav ul li.blog-overview a span { color: #0081a5; }#overlay-menu { background-color: rgba(245, 245, 245, 1); }#overlay-menu .overlay-menu-inner nav { text-align: center; }#overlay-menu .overlay-menu-inner nav ul li a { padding-top: 1.1111111111111rem; }#overlay-menu .overlay-menu-inner nav ul li a { padding-bottom: 1.1111111111111rem; }#overlay-menu .overlay-menu-inner nav ul li a span { letter-spacing: 0.022222222222222223rem; }@media screen and (min-width: 992px) and (max-width: 1169.9px) { .nav_4ndms1087 .navbar-inner .hamburger a.menu-icon { height: 14px; }.nav_4ndms1087 .navbar-inner .hamburger a.menu-icon span { margin-top: 7px; }}@media screen and (min-width: 768px) and (max-width: 991.9px) { .nav_4ndms1087 .navbar-inner .hamburger a.menu-icon { height: 14px; }.nav_4ndms1087 .navbar-inner .hamburger a.menu-icon span { margin-top: 7px; }}@media screen and (min-width: 544px) and (max-width: 767.9px) { .nav_4ndms1087 .navbar-inner .hamburger a.menu-icon { height: 14px; }.nav_4ndms1087 .navbar-inner .hamburger a.menu-icon span { margin-top: 7px; }}@media screen and (max-width: 543.9px) { .nav_4ndms1087 .logo img, .nav_4ndms1087 .logo svg { width: 13.555555555555555rem; }.nav_4ndms1087 .navbar-inner .hamburger a.menu-icon { height: 14px; }.nav_4ndms1087 .navbar-inner .hamburger a.menu-icon span { margin-top: 7px; }}</style>
<link rel="icon" href="/wp-content/uploads/2015/07/cropped-robust-icon-32x32.png" sizes="32x32">
<link rel="icon" href="/wp-content/uploads/2015/07/cropped-robust-icon-192x192.png" sizes="192x192">
<link rel="apple-touch-icon" href="/wp-content/uploads/2015/07/cropped-robust-icon-180x180.png">
<meta name="msapplication-TileImage" content="/wp-content/uploads/2015/07/cropped-robust-icon-270x270.png">
<style type="text/css" id="wp-custom-css">.grecaptcha-badge{
visibility: collapse !important;
}</style>
<style>html{margin-top:0px!important;}#wpadminbar{top:auto!important;bottom:0;}}</style>
</head>
<body class="post-template-default single single-post postid-3014 single-format-standard is-frontend static-mode mejs-semplice-ui" data-post-type="post">
<div id="content-holder" data-active-post="3014">
<header class="nav_4ndms1087 semplice-navbar active-navbar non-sticky-nav menu-type-text" data-cover-transparent="disabled" data-bg-overlay-visibility="visible" data-mobile-fallback="enabled">
<div class="container" data-nav="logo-left-menu-right">
<div class="navbar-inner menu-type-text" data-xl-width="12" data-navbar-type="container">
<div class="logo navbar-left"><a href="/" title="Robust Perception | Prometheus Monitoring Experts"><img src="/wp-content/uploads/2016/12/robust-perception-final_3.png" alt="logo"></a></div>
<nav class="standard navbar-right" data-font="font_7g535ae2g"><ul class="menu">
<li class="menu-item menu-item-type-post_type menu-item-object-page current_page_parent menu-item-3964"><a href="/blog"><span>Blog</span></a></li>
<li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-3965"><a href="https://robustperception.teachable.com/"><span>Training</span></a></li>
<li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-3977"><a target="_blank" rel="noopener" href="https://www.amazon.com/Prometheus-Infrastructure-Application-Performance-Monitoring-ebook/dp/B07FCV2VVG/"><span>Book</span></a></li>
</ul></nav>
<div class="hamburger navbar-right semplice-menu"><a class="open-menu menu-icon"><span></span></a></div>
</div>
</div>
</header>
<div id="overlay-menu">
<div class="overlay-menu-inner" data-xl-width="12">
<nav class="overlay-nav" data-justify="center" data-align="align-middle" data-font="font_7g535ae2g">
<ul class="container">
<li class="menu-item menu-item-type-post_type menu-item-object-page current_page_parent menu-item-3964"><a href="/blog"><span>Blog</span></a></li>
<li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-3965"><a href="https://robustperception.teachable.com/"><span>Training</span></a></li>
<li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-3977"><a target="_blank" rel="noopener" href="https://www.amazon.com/Prometheus-Infrastructure-Application-Performance-Monitoring-ebook/dp/B07FCV2VVG/"><span>Book</span></a></li>
</ul>
</nav>
</div>
</div>
<div id="content-3014" class="content-container active-content ">
<div class="sections">
<section class="content-block" data-column-mode-xs="single" data-column-mode-sm="single" style="padding-top: 3.5rem; padding-bottom: 3.5rem; background-color: rgb(125, 178, 207); margin-bottom: 3rem;">
<div class="is-content" style="height: 100%;" id="mce_0" spellcheck="false">
<h1 style="text-align: center;" data-mce-style="text-align: center;">
<span style="color: rgb(255, 255, 255);" data-mce-style="color: #ffffff;">Reliable Insights</span>
</h1>
<p style="text-align: center;" data-mce-style="text-align: center;">
<span style="color: rgb(33, 33, 33);" data-mce-style="color: #212121;">A blog on monitoring, scale and operational Sanity</span>
</p>
</div>
</section>
<div class="posts">
<section class="post is-content post-3014 post type-post status-publish format-standard hentry category-posts tag-alertmanager tag-prometheus tag-relabelling tag-reliability">
<div class="container">
<div class="row">
<div class="column" data-xl-width="8" data-md-width="11" data-sm-width="12" data-xs-width="12">
<div class="post-heading">
<p><a href="/high-availability-prometheus-alerting-and-notification">July 17, 2017</a><span></span></p>
<h2><a href="/high-availability-prometheus-alerting-and-notification">High Availability Prometheus Alerting and Notification</a></h2>
</div>
<div class="post-content">
<p>Prometheus is <a href="https://www.robustperception.io/prometheus-and-alertmanager-architecture/">architected</a> for <a href="https://www.robustperception.io/monitoring-without-consensus/">reliability</a> of alerting, how do you set it up?</p>
<div id="more-3014"></div>
<p>For a setup that can gracefully handle any machine failing, we'll need to run two Prometheus servers and two Alertmanagers. First we'll run the Alertmanagers on different machines, and setup a mesh between them:</p>
<pre># On a machine named "am-1":
wget https://github.com/prometheus/alertmanager/releases/download/v0.15.3/alertmanager-0.15.3.linux-amd64.tar.gz
tar -xzf alertmanager-*.linux-amd64.tar.gz
cd alertmanager-*
./alertmanager --cluster.peer=am-2:9094
# On a machine named "am-2":
wget https://github.com/prometheus/alertmanager/releases/download/v0.15.3/alertmanager-0.15.3.linux-amd64.tar.gz
tar -xzf alertmanager-*.linux-amd64.tar.gz
cd alertmanager-*
./alertmanager --cluster.peer=am-1:9094</pre>
<p>To verify that the Alertmanager mesh is working correctly, create a silence in one Alertmanager. If it shows up in the other, then all is well.</p>
<p>&nbsp;</p>
<p>Next we configure the Prometheus servers to talk to the Alertmanager:</p>
<pre># On a machine named "prom-1":
wget https://github.com/prometheus/prometheus/releases/download/v2.5.0/prometheus-2.0.0.linux-amd64.tar.gz
tar -xzf prometheus-*.linux-amd64.tar.gz
cd prometheus-*
cat &gt; prometheus.yml &lt;&lt; EOF
global:
external_labels:
dc: europe1
alerting:
alert_relabel_configs:
- source_labels: [dc]
regex: (.+)\d+
target_label: dc
alertmanagers:
- static_configs:
- targets: ['am-1:9093', 'am-2:9093']
# The rest of your Prometheus config goes here as usual.
EOF
./prometheus
# On a machine named "prom-2":
wget https://github.com/prometheus/prometheus/releases/download/v2.5.0/prometheus-2.0.0.linux-amd64.tar.gz
tar -xzf prometheus-*.linux-amd64.tar.gz
cd prometheus-*
cat &gt; prometheus.yml &lt;&lt; EOF
global:
external_labels:
dc: europe2 # Note that this is different only by the trailing number.
alerting:
alert_relabel_configs:
- source_labels: [dc]
regex: (.+)\d+
target_label: dc
alertmanagers:
- static_configs:
- targets: ['am-1:9093', 'am-2:9093']
# The rest of your Prometheus config goes here as usual.
EOF
./prometheus</pre>
<p>The key point here is that both Prometheus servers talk to both Alertmanagers.</p>
<p>In addition the two Prometheus servers have slightly different external labels, so their data does not conflict if remote storage is in use. We then use alert relabelling to ensure they still send identically labelled alerts, which the Alertmanager will automatically de-duplicate.</p>
<p>&nbsp;</p>
<p>As long as one Prometheus and one Alertmanager are working and can talk to each other, alerts and notifications will get through!</p>
<p>&nbsp;</p>
<p><em>Looking for expert advice on architecting Prometheus? Contact us.</em></p>
<div class="post-meta">
<div class="category-meta">
<span>Published by Brian Brazil in</span> <a href="/category/posts">Posts</a>
</div>
<div class="tags-meta">
<span>Tags: </span> <a href="/tag/alertmanager">alertmanager</a>, <a href="/tag/prometheus">prometheus</a>, <a href="/tag/relabelling">relabelling</a>, <a href="/tag/reliability">reliability</a>
</div>
</div>
<div id="share-holder">
<div class="is-content share-box">
<div class="share-icons-wrapper">
<p class="visible regular">Share on</p>
<div class="semplice-share-icons first">
<div class="share-icon icon-facebook">
<a href="//facebook.com/sharer.php?u=/high-availability-prometheus-alerting-and-notification" onclick="window.open(this.href,this.title,'width=500,height=500,top=200px,left=200px'); return false;" rel="nofollow" target="_blank"><svg version="1.1" id="Ebene_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewbox="0 0 16 16" enable-background="new 0 0 16 16" xml:space="preserve">
<path d="M4.025,5.291H5.68V4.541V3.805V3.683c0-0.708,0.018-1.802,0.533-2.479C6.755,0.487,7.5,0,8.781,0
c2.087,0,2.966,0.297,2.966,0.297l-0.414,2.451c0,0-0.689-0.199-1.333-0.199c-0.643,0-1.219,0.23-1.219,0.873v0.26v0.858v0.751
h2.638l-0.184,2.393H8.781V16H5.68V7.684H4.025V5.291"></path>
</svg>
</a>
</div>
</div>
<div class="semplice-share-icons">
<div class="share-icon icon-twitter">
<a href="//twitter.com/share?url=/high-availability-prometheus-alerting-and-notification" onclick="window.open(this.href,this.title,'width=500,height=500,top=200px,left=200px'); return false;" rel="nofollow" target="_blank"><svg version="1.1" id="Ebene_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewbox="0 0 16 16" enable-background="new 0 0 16 16" xml:space="preserve">
<path d="M16,3.536c-0.589,0.261-1.221,0.438-1.885,0.517c0.678-0.406,1.198-1.05,1.443-1.816c-0.634,0.376-1.337,0.649-2.085,0.797
c-0.599-0.638-1.452-1.037-2.396-1.037c-1.813,0-3.283,1.47-3.283,3.282c0,0.257,0.029,0.508,0.085,0.748
c-2.728-0.137-5.147-1.444-6.766-3.43c-0.283,0.485-0.444,1.049-0.444,1.65c0,1.139,0.579,2.144,1.46,2.732
C1.592,6.963,1.086,6.816,0.643,6.57c0,0.014,0,0.027,0,0.041c0,1.59,1.132,2.917,2.633,3.219C3,9.905,2.71,9.945,2.411,9.945
c-0.212,0-0.417-0.021-0.618-0.059c0.418,1.304,1.63,2.253,3.066,2.28c-1.123,0.88-2.539,1.405-4.077,1.405
c-0.265,0-0.526-0.016-0.783-0.046C1.453,14.456,3.178,15,5.032,15c6.038,0,9.34-5.002,9.34-9.34c0-0.142-0.003-0.284-0.01-0.425
C15.003,4.773,15.56,4.195,16,3.536z"></path>
</svg>
</a>
</div>
</div>
<div class="semplice-share-icons">
<div class="share-icon icon-linkedin">
<a href="//linkedin.com/shareArticle?mini=true&url=/high-availability-prometheus-alerting-and-notification&title=High%20Availability%20Prometheus%20Alerting%20and%20Notification" onclick="window.open(this.href,this.title,'width=500,height=500,top=200px,left=200px'); return false;" rel="nofollow" target="_blank"><svg version="1.1" id="Ebene_2" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewbox="0 0 16 16" enable-background="new 0 0 16 16" xml:space="preserve">
<g>
<g>
<path d="M14.815,0H1.181C0.529,0,0,0.517,0,1.153v13.693C0,15.483,0.529,16,1.181,16h13.634C15.467,16,16,15.483,16,14.845V1.153
C16,0.517,15.467,0,14.815,0z M4.745,13.634H2.371V5.999h2.374V13.634z M3.558,4.955c-0.761,0-1.376-0.617-1.376-1.376
c0-0.759,0.615-1.376,1.376-1.376c0.759,0,1.375,0.617,1.375,1.376C4.934,4.338,4.317,4.955,3.558,4.955z M13.633,13.634h-2.37
V9.921c0-0.886-0.017-2.025-1.234-2.025c-1.235,0-1.423,0.965-1.423,1.961v3.777H6.234V5.999H8.51v1.043h0.032
c0.317-0.6,1.091-1.233,2.246-1.233c2.401,0,2.845,1.581,2.845,3.638V13.634z"></path>
</g>
</g>
</svg>
</a>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</section>
<section id="comments" class="comments-area">
<div class="container">
<div class="row">
<div class="column" data-xl-width="8" data-md-width="11" data-sm-width="12" data-xs-width="12">
<p class="comments-title">Comments</p>
<p class="no-comments">No comments.</p>
</div>
</div>
</div>
</section>
<section class="blog-pagination">
<div class="container">
<div class="row">
<div class="column" data-xl-width="8" data-md-width="11" data-sm-width="12" data-xs-width="12">
</div>
</div>
</div>
</section>
</div>
<section id="section_d861b0e30" class="content-block" data-column-mode-sm="single" data-column-mode-xs="single" data-valign="center" data-justify="center">
<div class="container">
<div id="row_440d9a329" class="row">
<div id="column_62a534dee" class="column" data-xl-width="1">
<div class="content-wrapper">
<div id="content_e266ccdba" class="column-content" data-module="image">
<div class="ce-image" data-align="center"><a href="https://twitter.com/RobustPerceiver" target="_blank"><img loading="lazy" class="is-content" src="/wp-content/uploads/2018/07/twitter.png" width="80" height="81" alt="twitter" caption="" data-width="original" data-scaling="no"></a></div>
</div>
</div>
</div>
<div id="column_0ef698c3f" class="column" data-xl-width="1">
<div class="content-wrapper">
<div id="content_f019265e3" class="column-content" data-module="image">
<div class="ce-image" data-align="center"><a href="https://www.youtube.com/channel/UCtiFWOeRSTP3M6QUnTEKwpw" target="_blank"><img loading="lazy" class="is-content" src="/wp-content/uploads/2018/07/youtube.png" width="81" height="80" alt="youtube" caption="" data-width="original" data-scaling="no"></a></div>
</div>
</div>
</div>
<div id="column_d00c653b1" class="column" data-xl-width="1">
<div class="content-wrapper">
<div id="content_97c8a2ac0" class="column-content" data-module="image">
<div class="ce-image" data-align="center"><a href="https://www.linkedin.com/company/10177555" target="_blank"><img loading="lazy" class="is-content" src="/wp-content/uploads/2018/07/linkedin.png" width="80" height="80" alt="linkedin" caption="" data-width="original" data-scaling="no"></a></div>
</div>
</div>
</div>
</div>
<div id="row_a6c6c82a9" class="row"><div id="column_2079c0b8c" class="column" data-xl-width="12">
<div class="content-wrapper">
<div id="content_15153b52c" class="column-content" data-module="paragraph">
<div class="is-content"><div data-content-for="xl"><p style="text-align: center; line-height: 2.77778rem;" data-mce-style="text-align: center; line-height: 2.77778rem;"><span style="color: rgb(255, 255, 255); letter-spacing: 0.0222222rem;" data-mce-style="color: #ffffff; letter-spacing: 0.0222222rem;"><span class="font_7g535ae2g"><a style="color: #ffffff;" href="/blog" data-mce-href="../blog" data-mce-style="color: #ffffff;">Blog</a></span>&nbsp; &nbsp;|&nbsp; &nbsp;<a style="color: #ffffff;" href="https://robustperception.teachable.com/" target="_blank" rel="noopener" data-mce-href="https://robustperception.teachable.com/" data-mce-style="color: #ffffff;"><span class="font_7g535ae2g">Training</span></a>&nbsp; &nbsp;|&nbsp; &nbsp;<span class="font_7g535ae2g"><a style="color: #ffffff;" href="https://www.amazon.com/Prometheus-Infrastructure-Application-Performance-Monitoring-ebook/dp/B07FCV2VVG/" data-mce-href="https://www.amazon.com/Prometheus-Infrastructure-Application-Performance-Monitoring-ebook/dp/B07FCV2VVG/" data-mce-style="color: #ffffff;" target="_blank" rel="noopener">Book</a></span>&nbsp; &nbsp;|&nbsp; &nbsp;<span class="font_7g535ae2g"><a style="color: #ffffff;" href="/privacy" data-mce-href="../privacy" data-mce-style="color: #ffffff;" data-mce-selected="1">Privacy</a></span><span class="font_7g535ae2g"></span></span></p></div></div>
</div>
</div>
</div></div>
</div>
</section>
</div>
</div>
</div>
<div class="pswp" tabindex="-1" role="dialog" aria-hidden="true">
<div class="pswp__bg"></div>
<div class="pswp__scroll-wrap">
<div class="pswp__container">
<div class="pswp__item"></div>
<div class="pswp__item"></div>
<div class="pswp__item"></div>
</div>
<div class="pswp__ui pswp__ui--hidden">
<div class="pswp__top-bar">
<div class="pswp__counter"></div>
<button class="pswp__button pswp__button--close" title="Close (Esc)"></button>
<button class="pswp__button pswp__button--share" title="Share"></button>
<button class="pswp__button pswp__button--fs" title="Toggle fullscreen"></button>
<button class="pswp__button pswp__button--zoom" title="Zoom in/out"></button>
<div class="pswp__preloader">
<div class="pswp__preloader__icn">
<div class="pswp__preloader__cut">
<div class="pswp__preloader__donut"></div>
</div>
</div>
</div>
</div>
<div class="pswp__share-modal pswp__share-modal--hidden pswp__single-tap">
<div class="pswp__share-tooltip"></div>
</div>
<button class="pswp__button pswp__button--arrow--left" title="Previous (arrow left)">
</button>
<button class="pswp__button pswp__button--arrow--right" title="Next (arrow right)">
</button>
<div class="pswp__caption">
<div class="pswp__caption__center"></div>
</div>
</div>
</div>
</div> <div class="back-to-top">
<a class="semplice-event" data-event-type="helper" data-event="scrollToTop"><svg version="1.1" id="Ebene_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" width="53px" height="20px" viewbox="0 0 53 20" enable-background="new 0 0 53 20" xml:space="preserve">
<g id="Ebene_3">
</g>
<g>
<polygon points="43.886,16.221 42.697,17.687 26.5,4.731 10.303,17.688 9.114,16.221 26.5,2.312 "></polygon>
</g>
</svg>
</a>
</div>
<script type="text/javascript" id="ppress-frontend-script-js-extra">
/* <![CDATA[ */
var pp_ajax_form = {"ajaxurl":"\/wp-admin\/admin-ajax.php","confirm_delete":"Are you sure?","deleting_text":"Deleting...","deleting_error":"An error occurred. Please try again.","nonce":"5058643fc8","disable_ajax_form":"false"};
/* ]]> */
</script>
<script type="text/javascript" src="/wp-content/plugins/wp-user-avatar/assets/js/frontend.min.js?ver=3.1.19" id="ppress-frontend-script-js"></script>
<script type="text/javascript" id="mediaelement-core-js-before">
var mejsL10n = {"language":"en","strings":{"mejs.download-file":"Download File","mejs.install-flash":"You are using a browser that does not have Flash player enabled or installed. Please turn on your Flash player plugin or download the latest version from https:\/\/get.adobe.com\/flashplayer\/","mejs.fullscreen":"Fullscreen","mejs.play":"Play","mejs.pause":"Pause","mejs.time-slider":"Time Slider","mejs.time-help-text":"Use Left\/Right Arrow keys to advance one second, Up\/Down arrows to advance ten seconds.","mejs.live-broadcast":"Live Broadcast","mejs.volume-help-text":"Use Up\/Down Arrow keys to increase or decrease volume.","mejs.unmute":"Unmute","mejs.mute":"Mute","mejs.volume-slider":"Volume Slider","mejs.video-player":"Video Player","mejs.audio-player":"Audio Player","mejs.captions-subtitles":"Captions\/Subtitles","mejs.captions-chapters":"Chapters","mejs.none":"None","mejs.afrikaans":"Afrikaans","mejs.albanian":"Albanian","mejs.arabic":"Arabic","mejs.belarusian":"Belarusian","mejs.bulgarian":"Bulgarian","mejs.catalan":"Catalan","mejs.chinese":"Chinese","mejs.chinese-simplified":"Chinese (Simplified)","mejs.chinese-traditional":"Chinese (Traditional)","mejs.croatian":"Croatian","mejs.czech":"Czech","mejs.danish":"Danish","mejs.dutch":"Dutch","mejs.english":"English","mejs.estonian":"Estonian","mejs.filipino":"Filipino","mejs.finnish":"Finnish","mejs.french":"French","mejs.galician":"Galician","mejs.german":"German","mejs.greek":"Greek","mejs.haitian-creole":"Haitian Creole","mejs.hebrew":"Hebrew","mejs.hindi":"Hindi","mejs.hungarian":"Hungarian","mejs.icelandic":"Icelandic","mejs.indonesian":"Indonesian","mejs.irish":"Irish","mejs.italian":"Italian","mejs.japanese":"Japanese","mejs.korean":"Korean","mejs.latvian":"Latvian","mejs.lithuanian":"Lithuanian","mejs.macedonian":"Macedonian","mejs.malay":"Malay","mejs.maltese":"Maltese","mejs.norwegian":"Norwegian","mejs.persian":"Persian","mejs.polish":"Polish","mejs.portuguese":"Portuguese","mejs.romanian":"Romanian","mejs.russian":"Russian","mejs.serbian":"Serbian","mejs.slovak":"Slovak","mejs.slovenian":"Slovenian","mejs.spanish":"Spanish","mejs.swahili":"Swahili","mejs.swedish":"Swedish","mejs.tagalog":"Tagalog","mejs.thai":"Thai","mejs.turkish":"Turkish","mejs.ukrainian":"Ukrainian","mejs.vietnamese":"Vietnamese","mejs.welsh":"Welsh","mejs.yiddish":"Yiddish"}};
</script>
<script type="text/javascript" src="/wp-includes/js/mediaelement/mediaelement-and-player.min.js?ver=4.2.16" id="mediaelement-core-js"></script>
<script type="text/javascript" src="/wp-includes/js/mediaelement/mediaelement-migrate.min.js?ver=5.9.3" id="mediaelement-migrate-js"></script>
<script type="text/javascript" id="mediaelement-js-extra">
/* <![CDATA[ */
var _wpmejsSettings = {"pluginPath":"\/wp-includes\/js\/mediaelement\/","classPrefix":"mejs-","stretching":"responsive"};
/* ]]> */
</script>
<script type="text/javascript" id="semplice-frontend-js-js-extra">
/* <![CDATA[ */
var semplice = {"default_api_url":"\/wp-json","semplice_api_url":"\/wp-json\/semplice\/v1\/frontend","template_dir":"\/wp-content\/themes\/semplice4","category_base":"\/category\/","tag_base":"\/tag\/","nonce":"d1a583233f","frontend_mode":"static","static_transitions":"disabled","site_name":"Robust Perception | Prometheus Monitoring Experts","base_url":"","frontpage_id":"3967","blog_home":"\/blog","blog_navbar":"nav_4ndms1087","sr_status":"disabled","blog_sr_status":"disabled","is_preview":"","password_form":"\n<div class=\"post-password-form\">\n\t<div class=\"inner\">\n\t\t<form action=\"\/wp-login.php?action=postpass\" method=\"post\">\n\t\t\t<div class=\"password-lock\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"35\" height=\"52\" viewBox=\"0 0 35 52\">\r\n <path id=\"Form_1\" data-name=\"Form 1\" d=\"M31.3,25.028H27.056a0.755,0.755,0,0,1-.752-0.757V14.654a8.8,8.8,0,1,0-17.608,0v9.616a0.755,0.755,0,0,1-.752.757H3.7a0.755,0.755,0,0,1-.752-0.757V14.654a14.556,14.556,0,1,1,29.111,0v9.616A0.755,0.755,0,0,1,31.3,25.028Zm-3.495-1.514h2.743V14.654a13.051,13.051,0,1,0-26.1,0v8.859H7.192V14.654a10.309,10.309,0,1,1,20.617,0v8.859Zm4.43,28.475H2.761A2.77,2.77,0,0,1,0,49.213V25.28a1.763,1.763,0,0,1,1.755-1.766H33.242A1.763,1.763,0,0,1,35,25.28V49.213A2.77,2.77,0,0,1,32.239,51.988ZM1.758,25.028a0.252,0.252,0,0,0-.251.252V49.213a1.259,1.259,0,0,0,1.254,1.262H32.239a1.259,1.259,0,0,0,1.254-1.262V25.28a0.252,0.252,0,0,0-.251-0.252H1.758ZM20.849,43h-6.7a0.75,0.75,0,0,1-.61-0.314,0.763,0.763,0,0,1-.1-0.682l1.471-4.44a4.1,4.1,0,1,1,5.184,0L21.563,42a0.763,0.763,0,0,1-.1.682A0.75,0.75,0,0,1,20.849,43ZM15.2,41.487H19.8l-1.319-3.979a0.76,0.76,0,0,1,.33-0.891,2.6,2.6,0,1,0-2.633,0,0.76,0.76,0,0,1,.33.891Z\"\/>\r\n<\/svg>\r\n<\/div>\n\t\t\t<p>This content is protected. <br \/><span>To view, please enter the password.<\/span><\/p>\n\t\t\t<div class=\"input-fields\">\n\t\t\t\t<input name=\"post_password\" class=\"post-password-input\" type=\"password\" size=\"20\" maxlength=\"20\" placeholder=\"Enter password\" \/><input type=\"submit\" name=\"Submit\" value=\"Submit\" \/>\t\t\t<\/div>\n\t\t<\/form>\n\t<\/div>\n<\/div>\n\n","gallery":{"prev":"<svg version=\"1.1\" id=\"Ebene_1\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" x=\"0px\" y=\"0px\"\n\twidth=\"18px\" height=\"40px\" viewBox=\"0 0 18 40\" enable-background=\"new 0 0 18 40\" xml:space=\"preserve\">\n<g id=\"Ebene_2\">\n\t<g>\n\t\t<polygon points=\"16.3,40 0.3,20 16.3,0 17.7,1 2.5,20 17.7,39 \t\t\"\/>\n\t<\/g>\n<\/g>\n<\/svg>\n","next":"<svg version=\"1.1\" id=\"Ebene_1\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" x=\"0px\" y=\"0px\"\n\twidth=\"18px\" height=\"40px\" viewBox=\"0 0 18 40\" enable-background=\"new 0 0 18 40\" xml:space=\"preserve\">\n<g id=\"Ebene_2\">\n\t<g>\n\t\t<polygon points=\"0.3,39 15.5,20 0.3,1 1.7,0 17.7,20 1.7,40 \t\t\"\/>\n\t<\/g>\n<\/g>\n<\/svg>\n"}};
/* ]]> */
</script>
<script type="text/javascript" src="/wp-content/themes/semplice4/assets/js/frontend.min.js?ver=4.2.2" id="semplice-frontend-js-js"></script>
</body>
</html>