Files
nexus/sreweekly/articles/504/08-querying-and-ingest-issues-in-eu.html
2026-09-12 17:23:01 +08:00

1933 lines
104 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<!-- force IE browsers in compatibility mode to use their most aggressive rendering engine -->
<meta charset="utf-8">
<title>Honeycomb Status - Querying and Ingest issues in EU</title>
<meta name="description" content="Honeycomb&#39;s Status Page - Querying and Ingest issues in EU.">
<!-- Mobile viewport optimization -->
<meta name="HandheldFriendly" content="True">
<meta name="MobileOptimized" content="320">
<meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0">
<!-- Time this page was rendered - http://purl.org/dc/terms/issued -->
<meta name="issued" content="1789195983">
<!-- Mobile IE allows us to activate ClearType technology for smoothing fonts for easy reading -->
<meta http-equiv="cleartype" content="on">
<!-- Atlassian Sans & Mono Fonts -->
<link
rel="preconnect"
href="https://ds-cdn.prod-east.frontend.public.atl-paas.net" />
<link
rel="preload"
href="https://ds-cdn.prod-east.frontend.public.atl-paas.net/assets/fonts/atlassian-sans/v3/AtlassianSans-latin.woff2"
as="font" type="font/woff2" crossorigin />
<link
rel="preload"
href="https://ds-cdn.prod-east.frontend.public.atl-paas.net/assets/font-rules/v5/atlassian-fonts.css"
as="style" crossorigin />
<style>
/* Dynamic Font Stack based on Feature Flag */
/* Using Atlassian Sans & Mono */
:root {
--font-stack-a: "Atlassian Sans", "Helvetica Neue", Helvetica, Arial, Sans-Serif;
--font-stack-b: "Atlassian Mono", "SFMono-Medium", "SF Mono", "Segoe UI Mono", "Roboto Mono", "Ubuntu Mono", Menlo, Consolas, Courier, monospace;
}
</style>
<link rel="shortcut icon" type="image/x-icon" href="//dka575ofm4ao0.cloudfront.net/pages-favicon_logos/original/52154/2021-HC-Favicon.jpg" />
<link rel="shortcut icon" href='//dka575ofm4ao0.cloudfront.net/pages-favicon_logos/original/52154/2021-HC-Favicon.jpg'>
<link rel="alternate" type="application/atom+xml" href="https://status.honeycomb.io/history.atom" title="Honeycomb Status History - Atom Feed">
<link rel="alternate" type="application/rss+xml" href="https://status.honeycomb.io/history.rss" title="Honeycomb Status History - RSS Feed">
<!-- Canonical Link to ensure that only the custom domain is indexed when present -->
<link rel="canonical" href="https://status.honeycomb.io/incidents/pjzh0mtqw3vt">
<meta name="_globalsign-domain-verification" content="y_VzfckMy4iePo5oDJNivyYIjh8LffYa4jzUndm_bZ"/>
<link rel="alternate" type="application/atom+xml" title="ATOM" href="https://status.honeycomb.io/history.atom" />
<!-- Le styles -->
<link rel="stylesheet" media="screen" href="https://dka575ofm4ao0.cloudfront.net/packs/0.6d587e2c8ae0df459ecd.css" />
<link rel="stylesheet" media="all" href="https://dka575ofm4ao0.cloudfront.net/assets/status/status_manifest-3975cac43f498cd81d991d1abc986307c014f9c6ec29a9abfeb3b0882674850a.css" />
<script src="https://dka575ofm4ao0.cloudfront.net/assets/jquery-3.5.1.min-729e416557a365062a8a20f0562f18aa171da57298005d392312670c706c68de.js"></script>
<script>
window.pageColorData = {"blue":"#0298EC","border":"#D0D3D4","body_background":"#FFFFFF","font":"#25303E","graph":"#AE73C1","green":"#64BA00","light_font":"#AAAAAA","link":"#0278CD","orange":"#F96E10","red":"#D93E0E","yellow":"#FFB000","no_data":"#D0D3D4"};
</script>
<style>
/* BODY BACKGROUND */ /* BODY BACKGROUND */ /* BODY BACKGROUND */ /* BODY BACKGROUND */ /* BODY BACKGROUND */
body,
.layout-content.status.status-api .section .example-container .example-opener .color-secondary,
.grouped-items-selector,
.layout-content.status.status-full-history .history-nav a.current,
div[id^="subscribe-modal"] .modal-footer,
div[id^="subscribe-modal"],
div[id^="updates-dropdown"] .updates-dropdown-section,
#uptime-tooltip .tooltip-box {
background-color:#FFFFFF;
}
#uptime-tooltip .pointer-container .pointer-smaller {
border-bottom-color:#FFFFFF;
}
/* PRIMARY FONT COLOR */ /* PRIMARY FONT COLOR */ /* PRIMARY FONT COLOR */ /* PRIMARY FONT COLOR */
body.status,
.color-primary,
.color-primary:hover,
.layout-content.status-index .status-day .update-title.impact-none a,
.layout-content.status-index .status-day .update-title.impact-none a:hover,
.layout-content.status-index .timeframes-container .timeframe.active,
.layout-content.status-full-history .month .incident-container .impact-none,
.layout-content.status.status-index .incidents-list .incident-title.impact-none a,
.incident-history .impact-none,
.layout-content.status .grouped-items-selector.inline .grouped-item.active,
.layout-content.status.status-full-history .history-nav a.current,
.layout-content.status.status-full-history .history-nav a:not(.current):hover,
div[id^="subscribe-modal"] .modal-header .close,
.grouped-item-label,
#uptime-tooltip .tooltip-box .tooltip-content .related-events .related-event a.related-event-link {
color:#25303E;
}
.layout-content.status.status-index .components-statuses .component-container .name {
color:#25303E;
color:rgba(37,48,62,.8);
}
/* SECONDARY FONT COLOR */ /* SECONDARY FONT COLOR */ /* SECONDARY FONT COLOR */ /* SECONDARY FONT COLOR */
small,
.layout-content.status .table-row .date,
.color-secondary,
.layout-content.status .grouped-items-selector.inline .grouped-item,
.layout-content.status.status-full-history .history-footer .pagination a.disabled,
.layout-content.status.status-full-history .history-nav a,
#uptime-tooltip .tooltip-box .tooltip-content .related-events #related-event-header {
color:#AAAAAA;
}
/* BORDER COLOR */ /* BORDER COLOR */ /* BORDER COLOR */ /* BORDER COLOR */ /* BORDER COLOR */ /* BORDER COLOR */
body.status .layout-content.status .border-color,
hr,
.tooltip-base,
.markdown-display table,
div[id^="subscribe-modal"],
#uptime-tooltip .tooltip-box {
border-color:#D0D3D4;
}
div[id^="subscribe-modal"] .modal-footer,
.markdown-display table td {
border-top-color:#D0D3D4;
}
.markdown-display table td + td, .markdown-display table th + th {
border-left-color:#D0D3D4;
}
div[id^="subscribe-modal"] .modal-header,
#uptime-tooltip .pointer-container .pointer-larger {
border-bottom-color:#D0D3D4;
}
#uptime-tooltip .tooltip-box .outage-field {
/*
Generate the background-color for the outage-field from the css_body_background_color and css_border_color.
For the default background (#ffffff) and default css_border_color (#e0e0e0), use the luminosity of the default background with a magic number to arrive at
the original outage-field background color (#f4f5f7). I used the formula Target Color = Color * alpha + Background * (1 - alpha) to find the magic number of ~0.08.
For darker css_body_background_color, luminosity values are lower so alpha trends toward becoming transparent (thus outage-field background becomes same as css_body_background_color).
*/
background-color: rgba(208,211,212,0.31);
/*
outage-field border-color alpha is inverse to the luminosity of css_body_background_color.
That is to say, with a default white background this border is transparent, but on a black background, it's opaque css_border_color.
*/
border-color: rgba(208,211,212,0.0);
}
/* CSS REDS */ /* CSS REDS */ /* CSS REDS */ /* CSS REDS */ /* CSS REDS */ /* CSS REDS */ /* CSS REDS */
.layout-content.status.status-index .status-day .update-title.impact-critical a,
.layout-content.status.status-index .status-day .update-title.impact-critical a:hover,
.layout-content.status.status-index .page-status.status-critical,
.layout-content.status.status-index .unresolved-incident.impact-critical .incident-title,
.flat-button.background-red {
background-color:#D93E0E;
}
.layout-content.status-index .components-statuses .component-container.status-red:after,
.layout-content.status-full-history .month .incident-container .impact-critical,
.layout-content.status-incident .incident-name.impact-critical,
.layout-content.status.status-index .incidents-list .incident-title.impact-critical a,
.status-red .icon-indicator,
.incident-history .impact-critical,
.components-container .component-inner-container.status-red .component-status,
.components-container .component-inner-container.status-red .icon-indicator {
color:#D93E0E;
}
.layout-content.status.status-index .unresolved-incident.impact-critical .updates {
border-color:#D93E0E;
}
/* CSS ORANGES */ /* CSS ORANGES */ /* CSS ORANGES */ /* CSS ORANGES */ /* CSS ORANGES */ /* CSS ORANGES */
.layout-content.status.status-index .status-day .update-title.impact-major a,
.layout-content.status.status-index .status-day .update-title.impact-major a:hover,
.layout-content.status.status-index .page-status.status-major,
.layout-content.status.status-index .unresolved-incident.impact-major .incident-title {
background-color:#F96E10;
}
.layout-content.status-index .components-statuses .component-container.status-orange:after,
.layout-content.status-full-history .month .incident-container .impact-major,
.layout-content.status-incident .incident-name.impact-major,
.layout-content.status.status-index .incidents-list .incident-title.impact-major a,
.status-orange .icon-indicator,
.incident-history .impact-major,
.components-container .component-inner-container.status-orange .component-status,
.components-container .component-inner-container.status-orange .icon-indicator {
color:#F96E10;
}
.layout-content.status.status-index .unresolved-incident.impact-major .updates {
border-color:#F96E10;
}
/* CSS YELLOWS */ /* CSS YELLOWS */ /* CSS YELLOWS */ /* CSS YELLOWS */ /* CSS YELLOWS */ /* CSS YELLOWS */
.layout-content.status.status-index .status-day .update-title.impact-minor a,
.layout-content.status.status-index .status-day .update-title.impact-minor a:hover,
.layout-content.status.status-index .page-status.status-minor,
.layout-content.status.status-index .unresolved-incident.impact-minor .incident-title,
.layout-content.status.status-index .scheduled-incidents-container .tab {
background-color:#FFB000;
}
.layout-content.status-index .components-statuses .component-container.status-yellow:after,
.layout-content.status-full-history .month .incident-container .impact-minor,
.layout-content.status-incident .incident-name.impact-minor,
.layout-content.status.status-index .incidents-list .incident-title.impact-minor a,
.status-yellow .icon-indicator,
.incident-history .impact-minor,
.components-container .component-inner-container.status-yellow .component-status,
.components-container .component-inner-container.status-yellow .icon-indicator,
.layout-content.status.manage-subscriptions .confirmation-infobox .fa {
color:#FFB000;
}
.layout-content.status.status-index .unresolved-incident.impact-minor .updates,
.layout-content.status.status-index .scheduled-incidents-container {
border-color:#FFB000;
}
/* CSS BLUES */ /* CSS BLUES */ /* CSS BLUES */ /* CSS BLUES */ /* CSS BLUES */ /* CSS BLUES */
.layout-content.status.status-index .status-day .update-title.impact-maintenance a,
.layout-content.status.status-index .status-day .update-title.impact-maintenance a:hover,
.layout-content.status.status-index .page-status.status-maintenance,
.layout-content.status.status-index .unresolved-incident.impact-maintenance .incident-title,
.layout-content.status.status-index .scheduled-incidents-container .tab {
background-color:#0298EC;
}
.layout-content.status-index .components-statuses .component-container.status-blue:after,
.layout-content.status-full-history .month .incident-container .impact-maintenance,
.layout-content.status-incident .incident-name.impact-maintenance,
.layout-content.status.status-index .incidents-list .incident-title.impact-maintenance a,
.status-blue .icon-indicator,
.incident-history .impact-maintenance,
.components-container .component-inner-container.status-blue .component-status,
.components-container .component-inner-container.status-blue .icon-indicator {
color:#0298EC;
}
.layout-content.status.status-index .unresolved-incident.impact-maintenance .updates,
.layout-content.status.status-index .scheduled-incidents-container {
border-color:#0298EC;
}
/* CSS GREENS */ /* CSS GREENS */ /* CSS GREENS */ /* CSS GREENS */ /* CSS GREENS */ /* CSS GREENS */ /* CSS GREENS */
.layout-content.status.status-index .page-status.status-none {
background-color:#64BA00;
}
.layout-content.status-index .components-statuses .component-container.status-green:after,
.status-green .icon-indicator,
.components-container .component-inner-container.status-green .component-status,
.components-container .component-inner-container.status-green .icon-indicator {
color:#64BA00;
}
/* CSS LINK COLOR */ /* CSS LINK COLOR */ /* CSS LINK COLOR */ /* CSS LINK COLOR */ /* CSS LINK COLOR */ /* CSS LINK COLOR */
a,
a:hover,
.layout-content.status-index .page-footer span a:hover,
.layout-content.status-index .timeframes-container .timeframe:not(.active):hover,
.layout-content.status-incident .subheader a:hover {
color:#0278CD;
}
.flat-button,
.masthead .updates-dropdown-container .show-updates-dropdown,
.layout-content.status-full-history .show-filter.open {
background-color:#0278CD;
}
/* CUSTOM COLOR OVERRIDES FOR UPTIME SHOWCASE */
.components-section .components-uptime-link {
color: #aaaaaa;
}
.layout-content.status .shared-partial.uptime-90-days-wrapper .legend .legend-item {
color: #aaaaaa;
opacity: 1;
}
.layout-content.status .shared-partial.uptime-90-days-wrapper .legend .legend-item.light {
color: #aaaaaa;
opacity: 1;
}
.layout-content.status .shared-partial.uptime-90-days-wrapper .legend .spacer {
background: #aaaaaa;
opacity: 1;
}
</style>
<!-- custom css -->
<!-- polyfills -->
<script crossorigin="anonymous" src="https://cdnjs.cloudflare.com/polyfill/v3/polyfill.js"></script>
<!-- Le HTML5 shim -->
<!--[if lt IE 9]>
<script src="//html5shim.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->
<!-- injection for static -->
<meta name="twitter:card" content="summary">
<meta name="twitter:site" content="honeycombio">
<meta name="twitter:creator" content="honeycombio">
<meta name="robots" content="noindex,nofollow">
<meta property="og:type" content="article">
<meta property="og:url" content="https://status.honeycomb.io/incidents/pjzh0mtqw3vt">
<meta property="og:image" content="https://dka575ofm4ao0.cloudfront.net/assets/logos/favicon-2b86ed00cfa6258307d4a3d0c482fd733c7973f82de213143b24fc062c540367.png">
<meta property="og:title" content="Querying and Ingest issues in EU">
<meta property="og:description">
</head>
<body class="status incident status-none sp-a11y-status-focus-visibility a11y-focus-indicators">
<style>
/* BODY BACKGROUND */ /* BODY BACKGROUND */ /* BODY BACKGROUND */ /* BODY BACKGROUND */ /* BODY BACKGROUND */
body,
.layout-content.status.status-api .section .example-container .example-opener .color-secondary,
.grouped-items-selector,
.layout-content.status.status-full-history .history-nav a.current,
div[id^="subscribe-modal"] .modal-footer,
div[id^="subscribe-modal"],
div[id^="updates-dropdown"] .updates-dropdown-section,
#uptime-tooltip .tooltip-box {
background-color:#FFFFFF;
}
#uptime-tooltip .pointer-container .pointer-smaller {
border-bottom-color:#FFFFFF;
}
/* PRIMARY FONT COLOR */ /* PRIMARY FONT COLOR */ /* PRIMARY FONT COLOR */ /* PRIMARY FONT COLOR */
body.status,
.color-primary,
.color-primary:hover,
.layout-content.status-index .status-day .update-title.impact-none a,
.layout-content.status-index .status-day .update-title.impact-none a:hover,
.layout-content.status-index .timeframes-container .timeframe.active,
.layout-content.status-full-history .month .incident-container .impact-none,
.layout-content.status.status-index .incidents-list .incident-title.impact-none a,
.incident-history .impact-none,
.layout-content.status .grouped-items-selector.inline .grouped-item.active,
.layout-content.status.status-full-history .history-nav a.current,
.layout-content.status.status-full-history .history-nav a:not(.current):hover,
div[id^="subscribe-modal"] .modal-header .close,
.grouped-item-label,
#uptime-tooltip .tooltip-box .tooltip-content .related-events .related-event a.related-event-link {
color:#25303E;
}
.layout-content.status.status-index .components-statuses .component-container .name {
color:#25303E;
color:rgba(37,48,62,.8);
}
/* SECONDARY FONT COLOR */ /* SECONDARY FONT COLOR */ /* SECONDARY FONT COLOR */ /* SECONDARY FONT COLOR */
small,
.layout-content.status .table-row .date,
.color-secondary,
.layout-content.status .grouped-items-selector.inline .grouped-item,
.layout-content.status.status-full-history .history-footer .pagination a.disabled,
.layout-content.status.status-full-history .history-nav a,
#uptime-tooltip .tooltip-box .tooltip-content .related-events #related-event-header {
color:#AAAAAA;
}
/* BORDER COLOR */ /* BORDER COLOR */ /* BORDER COLOR */ /* BORDER COLOR */ /* BORDER COLOR */ /* BORDER COLOR */
body.status .layout-content.status .border-color,
hr,
.tooltip-base,
.markdown-display table,
div[id^="subscribe-modal"],
#uptime-tooltip .tooltip-box {
border-color:#D0D3D4;
}
div[id^="subscribe-modal"] .modal-footer,
.markdown-display table td {
border-top-color:#D0D3D4;
}
.markdown-display table td + td, .markdown-display table th + th {
border-left-color:#D0D3D4;
}
div[id^="subscribe-modal"] .modal-header,
#uptime-tooltip .pointer-container .pointer-larger {
border-bottom-color:#D0D3D4;
}
#uptime-tooltip .tooltip-box .outage-field {
/*
Generate the background-color for the outage-field from the css_body_background_color and css_border_color.
For the default background (#ffffff) and default css_border_color (#e0e0e0), use the luminosity of the default background with a magic number to arrive at
the original outage-field background color (#f4f5f7). I used the formula Target Color = Color * alpha + Background * (1 - alpha) to find the magic number of ~0.08.
For darker css_body_background_color, luminosity values are lower so alpha trends toward becoming transparent (thus outage-field background becomes same as css_body_background_color).
*/
background-color: rgba(208,211,212,0.31);
/*
outage-field border-color alpha is inverse to the luminosity of css_body_background_color.
That is to say, with a default white background this border is transparent, but on a black background, it's opaque css_border_color.
*/
border-color: rgba(208,211,212,0.0);
}
/* CSS REDS */ /* CSS REDS */ /* CSS REDS */ /* CSS REDS */ /* CSS REDS */ /* CSS REDS */ /* CSS REDS */
.layout-content.status.status-index .status-day .update-title.impact-critical a,
.layout-content.status.status-index .status-day .update-title.impact-critical a:hover,
.layout-content.status.status-index .page-status.status-critical,
.layout-content.status.status-index .unresolved-incident.impact-critical .incident-title,
.flat-button.background-red {
background-color:#D93E0E;
}
.layout-content.status-index .components-statuses .component-container.status-red:after,
.layout-content.status-full-history .month .incident-container .impact-critical,
.layout-content.status-incident .incident-name.impact-critical,
.layout-content.status.status-index .incidents-list .incident-title.impact-critical a,
.status-red .icon-indicator,
.incident-history .impact-critical,
.components-container .component-inner-container.status-red .component-status,
.components-container .component-inner-container.status-red .icon-indicator {
color:#D93E0E;
}
.layout-content.status.status-index .unresolved-incident.impact-critical .updates {
border-color:#D93E0E;
}
/* CSS ORANGES */ /* CSS ORANGES */ /* CSS ORANGES */ /* CSS ORANGES */ /* CSS ORANGES */ /* CSS ORANGES */
.layout-content.status.status-index .status-day .update-title.impact-major a,
.layout-content.status.status-index .status-day .update-title.impact-major a:hover,
.layout-content.status.status-index .page-status.status-major,
.layout-content.status.status-index .unresolved-incident.impact-major .incident-title {
background-color:#F96E10;
}
.layout-content.status-index .components-statuses .component-container.status-orange:after,
.layout-content.status-full-history .month .incident-container .impact-major,
.layout-content.status-incident .incident-name.impact-major,
.layout-content.status.status-index .incidents-list .incident-title.impact-major a,
.status-orange .icon-indicator,
.incident-history .impact-major,
.components-container .component-inner-container.status-orange .component-status,
.components-container .component-inner-container.status-orange .icon-indicator {
color:#F96E10;
}
.layout-content.status.status-index .unresolved-incident.impact-major .updates {
border-color:#F96E10;
}
/* CSS YELLOWS */ /* CSS YELLOWS */ /* CSS YELLOWS */ /* CSS YELLOWS */ /* CSS YELLOWS */ /* CSS YELLOWS */
.layout-content.status.status-index .status-day .update-title.impact-minor a,
.layout-content.status.status-index .status-day .update-title.impact-minor a:hover,
.layout-content.status.status-index .page-status.status-minor,
.layout-content.status.status-index .unresolved-incident.impact-minor .incident-title,
.layout-content.status.status-index .scheduled-incidents-container .tab {
background-color:#FFB000;
}
.layout-content.status-index .components-statuses .component-container.status-yellow:after,
.layout-content.status-full-history .month .incident-container .impact-minor,
.layout-content.status-incident .incident-name.impact-minor,
.layout-content.status.status-index .incidents-list .incident-title.impact-minor a,
.status-yellow .icon-indicator,
.incident-history .impact-minor,
.components-container .component-inner-container.status-yellow .component-status,
.components-container .component-inner-container.status-yellow .icon-indicator,
.layout-content.status.manage-subscriptions .confirmation-infobox .fa {
color:#FFB000;
}
.layout-content.status.status-index .unresolved-incident.impact-minor .updates,
.layout-content.status.status-index .scheduled-incidents-container {
border-color:#FFB000;
}
/* CSS BLUES */ /* CSS BLUES */ /* CSS BLUES */ /* CSS BLUES */ /* CSS BLUES */ /* CSS BLUES */
.layout-content.status.status-index .status-day .update-title.impact-maintenance a,
.layout-content.status.status-index .status-day .update-title.impact-maintenance a:hover,
.layout-content.status.status-index .page-status.status-maintenance,
.layout-content.status.status-index .unresolved-incident.impact-maintenance .incident-title,
.layout-content.status.status-index .scheduled-incidents-container .tab {
background-color:#0298EC;
}
.layout-content.status-index .components-statuses .component-container.status-blue:after,
.layout-content.status-full-history .month .incident-container .impact-maintenance,
.layout-content.status-incident .incident-name.impact-maintenance,
.layout-content.status.status-index .incidents-list .incident-title.impact-maintenance a,
.status-blue .icon-indicator,
.incident-history .impact-maintenance,
.components-container .component-inner-container.status-blue .component-status,
.components-container .component-inner-container.status-blue .icon-indicator {
color:#0298EC;
}
.layout-content.status.status-index .unresolved-incident.impact-maintenance .updates,
.layout-content.status.status-index .scheduled-incidents-container {
border-color:#0298EC;
}
/* CSS GREENS */ /* CSS GREENS */ /* CSS GREENS */ /* CSS GREENS */ /* CSS GREENS */ /* CSS GREENS */ /* CSS GREENS */
.layout-content.status.status-index .page-status.status-none {
background-color:#64BA00;
}
.layout-content.status-index .components-statuses .component-container.status-green:after,
.status-green .icon-indicator,
.components-container .component-inner-container.status-green .component-status,
.components-container .component-inner-container.status-green .icon-indicator {
color:#64BA00;
}
/* CSS LINK COLOR */ /* CSS LINK COLOR */ /* CSS LINK COLOR */ /* CSS LINK COLOR */ /* CSS LINK COLOR */ /* CSS LINK COLOR */
a,
a:hover,
.layout-content.status-index .page-footer span a:hover,
.layout-content.status-index .timeframes-container .timeframe:not(.active):hover,
.layout-content.status-incident .subheader a:hover {
color:#0278CD;
}
.flat-button,
.masthead .updates-dropdown-container .show-updates-dropdown,
.layout-content.status-full-history .show-filter.open {
background-color:#0278CD;
}
/* CUSTOM COLOR OVERRIDES FOR UPTIME SHOWCASE */
.components-section .components-uptime-link {
color: #aaaaaa;
}
.layout-content.status .shared-partial.uptime-90-days-wrapper .legend .legend-item {
color: #aaaaaa;
opacity: 1;
}
.layout-content.status .shared-partial.uptime-90-days-wrapper .legend .legend-item.light {
color: #aaaaaa;
opacity: 1;
}
.layout-content.status .shared-partial.uptime-90-days-wrapper .legend .spacer {
background: #aaaaaa;
opacity: 1;
}
</style>
<div class="layout-content status status-incident">
<div class="container">
<div class="page-title">
<h1 class="color-primary incident-name whitespace-pre-wrap impact-major">Querying and Ingest issues in EU</h1>
<div class="font-largest color-secondary subheader ">
Incident
Report for <a class="color-secondary" href="/">Honeycomb</a>
</div>
</div>
<div class="incident-updates-container">
<!-- postmortem if it's published -->
<div class="row update-row">
<h2 class="update-title span3 font-large">
Postmortem
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<div class="markdown-display">
<p>This incident started on December 5th, and is one of the longest in Honeycomb history, having been actively worked on and closed only on December 17th. Due to its impact and duration, we wanted to offer a partial and preliminary report to explain, at a high level, what happened.</p>
<p>On December 5th, at 20:23 UTC, our Kafka cluster suffered a critical loss of redundancy. Our Kafka cluster contains multiple topics, including all telemetry events submitted by Honeycomb users, the rematerialization of state changes into the activity log, and multiple metadata topics used by Kafka to manage its own workloads. This led multiple partitions leaderless and by 20:35 UTC, we were getting alerts that roughly a quarter of our usual event topic partitions were unable to accept writes.</p>
<p>For most Honeycomb customers, this does not result in ingest outages, because traffic gets redirected to other partitions environments can be assigned to. However, we noted some teams for whom all their assigned partitions fit within the impacted set, and for whom all ingest was down (0.23% of all datasets were impacted). By 21:30 UTC, we had identified impacted teams, were working on a traffic reassignment script, and thought other features (such as SLO evaluation) were unimpacted.</p>
<p>However, at 22:08 UTC, we noticed, through the noise of all the alerts that had been going on, that our ingestion fleet had been in overload protection since 20:38 UTC, impacted everyone more broadly by returning errors on <code>api.eu1.honeycomb.io</code> that would have led to dropped events. Stabilization work was done and 6 minutes later (22:14 UTC), ingestion was stabilized, yielding roughly 1h25 of increased ingest error rates on <code>api.eu1.honeycomb.io</code> for everyone.</p>
<p>At 01:30 UTC on Saturday, December 6th, our responders had managed to force new leader elections on all impacted ingestion partitions, which re-established working traffic internally on all but one of them, which remained shuttered in read-only mode. By then, our brokers were severely imbalanced, we attempted to tweak retention settings, and monitored disk usage to come back during the day to repair the remaining broken partitions.</p>
<p>By 11:00 UTC, our disk usage on Kafka brokers reached a threshold where it became necessary to devise and perform emergency operations. We noticed that our metadata partitions, some of which handle the tracking of Kafka offloading its storage to S3 and its autobalancing features, were still leaderless, and we thought this could be the problem.</p>
<p>In fixing them, we also repaired consumer group metadata topics, which revealed, at 14:27 UTC, that our SLO product had been partially stuck since the start of the outage. By having broken consumer groups topic partitions, our event consumers for SLOs had been working fine on some partitions but were fully stalled on others, despite reporting healthy—they were idling as “online” but not seeing that they were late. They caught up and by 19:15 UTC, they were backfilled and brought back to normal.</p>
<p>However, before then, the issue with Kafka disk storage got worse. We feared that with full disks, the entirety of our Kafka cluster would reach an irreparable state (where the only way to free space is to do dangerous untested operations to free storage data), and at 15:09 UTC, we instead chose to turn off ingest entirely with less than 5% of disk space left. Ingest would turn itself off if the disk were to be filled, so we elected to keep recovery simpler by cutting traffic off a few minutes ahead of time.</p>
<p>We quickly shuffled to try and add new Kafka brokers and partitions, which wouldn’t be full, to shift traffic onto them – but before we were done, we thought of disabling our tiered storage. Our Kafka brokers store only a few hours of data locally (“hotset” data), and tier out a longer retention period (2-3 days) to S3. All data written to Kafka is quickly replicated to our storage engine and longer retention is only kept for disaster recovery. We had tried repairing topics, and reducing the size of our hotset for many hours before, but nothing had the desired effect. Our theory was that tiering data was broken anyway, and we were stuck waiting for the offload of data that wouldn’t happen.</p>
<p>At 21:32 UTC, we disabled tiered storage altogether and most of the disk space was recovered. We abandoned the cluster expansion, which was minutes from completion, and at 21:49 UTC, Ingestion of customer data was turned back on. In total, we were unable to accept traffic for 6 hours and 23 minutes.</p>
<p>Our responders were online trying to clean up issues further until roughly 5 am UTC on December 7, and they disbanded to come back during the day. Sunday was mostly spent stabilizing the emergency configuration changes that were made, and the response team, who had worked around the clock since Friday night, was able to take some resting time while our EU cluster was mostly healthy again, aside from the activity log and one of our storage partitions that was unavailable still for querying.</p>
<p>On Monday December 8, employees who weren’t on call over the weekend took over stabilization work. Efforts went to successfully restore the partial querying outage of roughly 1/40th of our storage data. No significant progress was made on improving our activity log feature’s storage, and instead the team increased the storage retention of its changesets to 7 days (the max allowed in our storage engine). Our thinking was that once the Kafka cluster would be stable and these partitions fixed, we could simply replay events and insert everything back.</p>
<p>Tuesday December 9 was spent trying to further stabilize our Kafka cluster and to turn on some features, but we were able to make little progress in salvaging partitions, and started doing corrective work while our Kafka experts tried to see what they could do with the many still damaged metadata and internal topics that were less critical to staying up, but still very important.</p>
<p>On December 10, knowing we only had a few days to salvage Activity Log data, we decided to stop trying to save its Kafka topics, and to instead recreate them. At 19:25 UTC, we found that deletion operations fail, and that our Kafka cluster’s control plane no longer lets us do any manipulation whatsoever aside from listing topics. We can’t delete, create, describe, or mutate any of them. They are fine accepting and transmitting events, but we are essentially unable to administer the cluster anymore.</p>
<p>What we realized at that point in time is that our initial cluster outage severely damaged multiple internal partitions used to manage the cluster itself; as we cut off tiered storage (and as it aged out as well), the damage became more or less irreversible.</p>
<p>We then considered our chances of salvaging the cluster to be rather low, and feared that most small changes (such as assigning a new controller) could collapse the cluster and result in a large outage with data loss. Internal details of some of our applications are relevant here: our storage engine is tightly coupled to Kafka’s own event offsets. To prevent data consistency issues, it will refuse to “roll back” to an older offset, which could indicate a misconfiguration or some other problem that could lead to re-reading, duplicating, or losing data. As such, just “resetting” the cluster was not doable at this point in time, without adding a lot of infrastructure, or writing emergency fixes that significantly change our storage engine’s boot sequences and safety checks.</p>
<p>Later, still on December 10, we decided to split efforts into a) keep trying to save our cluster to the extent it doesn’t risk its stability, and b) start an emergency migration project that requires figuring out how to modify our storage engine, infrastructure, and multiple services to tolerate moving from the damaged Kafka cluster to a new one. We also prepared for multiple contingencies in case the current cluster were to die before the migration was ready to run.</p>
<p>December 11 and December 12 were spent working on this migration at a high priority. Likewise, during that time, our low-risk efforts to salvage the Kafka cluster yielded no great results. As we neared the end of Friday December 12, our retention of activity for the Activity Log also came to an end. In deciding between causing a major outage to rush an evacuation of our Kafka cluster before we were ready or losing days of Activity Log data to wait for a safer migration path, we chose the latter.</p>
<p>On Monday December 15, we managed to boot a new Kafka cluster with new infrastructure, and had all the fixes required for all services to do a migration, along with runbooks from every team involved. We decided to do a “dress rehearsal” in a pre-production environment, where we tried a full evacuation from a functional Kafka cluster to a brand new one, finding what the tricky parts were and making sure that if we were to damage or lose data, it would be internal telemetry, and not customer data.</p>
<p>Fortunately, everything went well, and on Tuesday December 16, we ran the full emergency evacuation. Running it required doing a switchover of all ingest from the old damaged Kafka cluster to the new Kafka cluster. We started by switching components that write data into the cluster first (producers), letting the consumers catch up on all topics and partitions. We then started migrating our consuming services, first with the query engine, and then the other consumers. This order of operation ensured that we would not corrupt, damage, or miss any data, but forced us to have delays on alerting and the freshness of query data. This happened without major issues.</p>
<p>During the migration, we believe our trigger data was stale and runs using incomplete data for roughly 30 minutes, if we average out most partitions. SLOs were delayed for a bit more than one hour, and service maps will have flat out skipped that hour as well. Activity Log data was lost between December 5 at 20:23 UTC and December 9 at 23:45 UTC.</p>
<p>These are added to the 10 days during which Activity Log data was unavailable, the ingestion issues of December 5, the 6h23 minutes of full ingest outage on December 6, and the 18h of delays on SLO processing between December 5 and 6.</p>
<p>At this point in time, we have fully mitigated this incident and developed new processes that promise better readiness to respond if similar outages were to happen again. A more in-depth review will be published in a few weeks, in January. This has been a significant outage, and we will need some time to do a proper analysis of it. </p>
<p>FEBRUARY 2025 EDIT: The public incident report has been published at <a href="https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations">https://www.honeycomb.io/blog/incident-report-exercises-cleanups-and-evacuations</a></p>
</div>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1766085099000"></span>Dec <var data-var='date'>18</var>, <var data-var='year'>2025</var> - <var data-var='time'>11:11</var> PST
</div>
</div>
</div>
<hr>
<!-- incident updates in reverse order -->
<div class="row update-row">
<h2 class="update-title span3 font-large">
Resolved
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">As of 23:23UTC on December 16, the Activity Log has fully caught up and has remained up to date.<br /><br />All systems are now operational.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765986463000"></span>Dec <var data-var='date'>17</var>, <var data-var='year'>2025</var> - <var data-var='time'>07:47</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Monitoring
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We successfully created and migrated over to a new kafka cluster in the EU. We have resumed consuming events for our SLO and Service Maps features. No further data loss is expected for service maps. SLOs are caught up and are fully synchronized. Activity logs are still delayed and expect to be caught up in about 5 hours. We consider this incident to be mitigated, but will continue to monitor until Activity logs are caught up.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765916235000"></span>Dec <var data-var='date'>16</var>, <var data-var='year'>2025</var> - <var data-var='time'>12:17</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">As of today, our old EU cluster is still functional, yet in a bad state. We do not consider this outage over. However, after running a full evacuation in a pre-production cluster, we are now ready to run a production version of it. This will only be in the impacted EU cluster. We are planning to do it on 6pm UTC December 16th. We expect no ingestion downtime, although we will expect delays on processing events and alerts, will suffer minor data loss in our Service Maps feature, and lose 11 days worth of Activity log events starting the night of December 5th.<br /><br />We will provide updates throughout the migration.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765849979000"></span>Dec <var data-var='date'>15</var>, <var data-var='year'>2025</var> - <var data-var='time'>17:52</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We continue to work toward restoring the Activity Log. We’ve made meaningful progress on the underlying issues, but have had to pivot on our remediation plan due to previously unknown hard retention limits. Our engineering team is working through the next steps required to restore Activity Log processing. Next updates to come during Monday US business hours</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765581619000"></span>Dec <var data-var='date'>12</var>, <var data-var='year'>2025</var> - <var data-var='time'>15:20</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We continue to work toward restoring the Activity Log. We’ve made meaningful progress on the underlying issues, but have had to pivot on our remediation plan due to previously unknown hard retention limits. Our engineering team is actively working through the next steps required to restore Activity Log processing.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765498167000"></span>Dec <var data-var='date'>11</var>, <var data-var='year'>2025</var> - <var data-var='time'>16:09</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We continue to work toward fully restoring the Activity Log. We’ve made meaningful progress on the underlying issues and continue to work through our remediation plan. Our engineering team is actively working through the next steps required to restore Activity Log processing.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765476159000"></span>Dec <var data-var='date'>11</var>, <var data-var='year'>2025</var> - <var data-var='time'>10:02</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We continue to work toward fully restoring the Activity Log. While functionality has not yet improved, we’ve made progress on the underlying issue and have begun the next phase of remediation. Our engineering team is pausing for the evening and will resume working through the remaining steps required to resume Activity Log processing during US business hours tomorrow. We will provide another update tomorrow morning.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765415331000"></span>Dec <var data-var='date'>10</var>, <var data-var='year'>2025</var> - <var data-var='time'>17:08</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We continue to work toward fully restoring the Activity Log. While functionality has not yet improved, we’ve made progress on the underlying issue and have begun the next phase of remediation. Our engineering team is actively working through the remaining steps required to resume Activity Log processing.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765388042000"></span>Dec <var data-var='date'>10</var>, <var data-var='year'>2025</var> - <var data-var='time'>09:34</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We continue to work toward fully restoring the Activity Log. While functionality has not yet improved, we have a clear path to recovery, and remediation work is underway. Our next steps involve coordinated infrastructure changes that will allow us to resume Activity Log processing. We’ll provide another update during US business hours Wednesday.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765320011000"></span>Dec <var data-var='date'>09</var>, <var data-var='year'>2025</var> - <var data-var='time'>14:40</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We are continuing to work on a fix for this issue.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765245916000"></span>Dec <var data-var='date'>08</var>, <var data-var='year'>2025</var> - <var data-var='time'>18:05</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We fixed the issue that was causing subsets of temporarily missing data in queries against certain partitions. We continue to work on restoring full functionality to Activity Log. We'll provide another update during US business hours Tuesday.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765245686000"></span>Dec <var data-var='date'>08</var>, <var data-var='year'>2025</var> - <var data-var='time'>18:01</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We continue to work on restoring full functionality to Activity Log and repair the partition causing certain customers to have subsets of unqueryable data. We will next provide an update during US business hours Tuesday.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765242134000"></span>Dec <var data-var='date'>08</var>, <var data-var='year'>2025</var> - <var data-var='time'>17:02</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">The Activity Log is operational but is experiencing ingestion delays, which means recent activity may not be reflected immediately. We are working to restore full functionality.<br />Currently, a small subset of data for certain customers may not be queryable due to a partition needing repair. We are working to return the data to queryability.<br />After investigation of the impact starting on Friday to SLO and Trigger functionality, we're providing the update below with the details.<br /><br />Overview<br />SLOs and Triggers were affected by Kafka and ingest outages beginning Friday, December 5 at 8:40 PM UTC. Trigger evaluations are now behaving normally except for some baseline triggers. SLOs with evaluation periods that include missing ingest or are on the affected partition will reflect that currently unqueryable data in their ongoing calculations.<br />SLOs<br />- No burn alerts were fired between December 5, 2025 at 8:40 PM UTC and December 6, 2025 at 7:15 PM UTC due to a processing delay.<br />- Now that ingest is restored, burn alerts should behave normally again minus a subset that are also impacted by the small amount of currently unqueryable data.<br />Triggers<br />- When ingest was down on December 6, no events were received, so all EU Trigger evaluations were impacted.<br />- Baseline Triggers with lookback windows that query data in the affected partition returning will also be affected by the small amount of currently unqueryable data.<br />- Non-baseline triggers (which can only look back 24 hours) should behave normally going forward.<br /><br />We will provide an update at the end of the workday (5PM PT) or as the situation changes.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765230747000"></span>Dec <var data-var='date'>08</var>, <var data-var='year'>2025</var> - <var data-var='time'>13:52</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">Our Kafka cluster is still operational and we are continuing to work towards restoring full reliability. We will resume work during business hours to restore full Activity Log functionality. We will continue to update as work progresses.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765139033000"></span>Dec <var data-var='date'>07</var>, <var data-var='year'>2025</var> - <var data-var='time'>12:23</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">The Kafka cluster is still operational, but is not back to full capacity and resiliency. Work will continue tomorrow to ensure sufficient capacity, as well as during business hours to resume full Activity Log functionality. We will provide more updates tomorrow and as-needed if there are any changes to availability.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765073884000"></span>Dec <var data-var='date'>06</var>, <var data-var='year'>2025</var> - <var data-var='time'>18:18</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We have gotten the Kafka cluster to a stable-enough state, however it will still require remediation before the Activity Log is fully functional and it is back to a level of service sustainable until business hours. We will provide another update in 2 hours or as the situation changes.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765066220000"></span>Dec <var data-var='date'>06</var>, <var data-var='year'>2025</var> - <var data-var='time'>16:10</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">Ingest has been fully re-enabled, and SLOs and Triggers are now up-to-date.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765058895000"></span>Dec <var data-var='date'>06</var>, <var data-var='year'>2025</var> - <var data-var='time'>14:08</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Identified
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We have resumed ingest to api.eu1.honeycomb.io as we were able to restore partial service to the Kafka cluster.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765057881000"></span>Dec <var data-var='date'>06</var>, <var data-var='year'>2025</var> - <var data-var='time'>13:51</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We are continuing to investigate issues with the Kafka cluster. We will continue providing updates every 2 hours unless there are significant changes.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765056187000"></span>Dec <var data-var='date'>06</var>, <var data-var='year'>2025</var> - <var data-var='time'>13:23</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We are continuing to investigate issues with the Kafka cluster.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765048802000"></span>Dec <var data-var='date'>06</var>, <var data-var='year'>2025</var> - <var data-var='time'>11:20</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">External API access is restored, event ingestion is still impacted</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765037727000"></span>Dec <var data-var='date'>06</var>, <var data-var='year'>2025</var> - <var data-var='time'>08:15</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We have temporarily disabled event ingestion for the EU Region, in service of restoring full functionality to our EU Kafka fleet. External API access will also be disabled. Additionally, during the ongoing outage, our Service Level Objectives feature has been impacted and down since 12:30 PM Pacific time on Friday, December 5th. SLO data will not be correct until our systems catch up and we rebuild the cache</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765034365000"></span>Dec <var data-var='date'>06</var>, <var data-var='year'>2025</var> - <var data-var='time'>07:19</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">Automated systems are still working to catch up and Activity Log will remain offline until that that completes. We will post another update Saturday whether or not the Activity Log outage has been remediated.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1765006248000"></span>Dec <var data-var='date'>05</var>, <var data-var='year'>2025</var> - <var data-var='time'>23:30</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">Ingest, querying, SLOs, and Trigger alerting is back to normal. Activity Log is still impacted, we have identified the cause and are working to resolve. We are still evaluating the scope of the outage for ingest and expect to have a full answer for that posted during US business hours on Monday.<br />We will post at least one more update this evening.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1764979465000"></span>Dec <var data-var='date'>05</var>, <var data-var='year'>2025</var> - <var data-var='time'>16:04</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We are continuing the investigation after business hours to stabilize ingest and determine what work will be needed to fully recover. Known impact will be updated here as the situation changes. We will post at least one more update this evening.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1764978373000"></span>Dec <var data-var='date'>05</var>, <var data-var='year'>2025</var> - <var data-var='time'>15:46</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We have identified that 0.23% of datasets are fully affected, and a larger percentage are seeing intermittent ingestion and query failures (500s at the API level). We are also investigating a replication error in our ingestion pipeline.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1764973990000"></span>Dec <var data-var='date'>05</var>, <var data-var='year'>2025</var> - <var data-var='time'>14:33</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Update
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">A subset of customer environments may see higher than usual error rates when sending events to api.eu1.honeycomb.io, and notifications for SLOs and Triggers may be delayed for that subset. We are continuing to investigate the issue.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1764969575000"></span>Dec <var data-var='date'>05</var>, <var data-var='year'>2025</var> - <var data-var='time'>13:19</var> PST
</div>
</div>
</div>
<div class="row update-row">
<h2 class="update-title span3 font-large">
Investigating
</h2>
<div class="update-container span9">
<div class="update-body font-regular">
<span class="whitespace-pre-wrap">We are currently investigating this issue.</span>
</div>
<div class="update-timestamp font-small color-secondary">
Posted <span class="ago" data-datetime-unix="1764967747000"></span>Dec <var data-var='date'>05</var>, <var data-var='year'>2025</var> - <var data-var='time'>12:49</var> PST
</div>
</div>
</div>
<!-- affected components -->
<div class="components-affected font-small color-secondary border-color">
This incident affected: api.eu1.honeycomb.io - EU1 Event Ingest, ui.eu1.honeycomb.io - EU1 Querying, ui.eu1.honeycomb.io - EU1 Trigger &amp; SLO Alerting, and ui.eu1.honeycomb.io - EU1 Activity Log.
</div>
</div>
<div class="page-footer border-color font-small">
<a href="/"><span style="font-family:arial">&larr;</span> Current Status</a>
<span class="color-secondary powered-by"><a class="color-secondary" target="_blank" rel="noopener noreferrer nofollow" href="https://www.atlassian.com/software/statuspage?utm_campaign=status.honeycomb.io&amp;utm_content=SP-notifications&amp;utm_medium=powered-by&amp;utm_source=inapp">Powered by Atlassian Statuspage</a></span>
</div>
</div>
</div>
<div class="modal hide fade modal-open-incident-subscribe" id="subscribe-modal-pjzh0mtqw3vt" style="display: none"
data-js-hook="incident-subscription-modal" role="dialog" aria-labelledby="incident-subscription-dialog-header"
aria-describedby="incident-subscription-dialog-description" aria-modal="true">
<form class="modal-content" id="subscribe-form-pjzh0mtqw3vt" action="/subscriptions/incident.json" accept-charset="UTF-8" data-remote="true" method="post">
<input type="hidden" name="incident_code" id="incident_code" value="pjzh0mtqw3vt" autocomplete="off" />
<div class="modal-header">
<a href="#" data-dismiss="modal" class="close" data-js-hook="incident-modal-close" aria-label="close" role="button">×</a>
<h1 id="incident-subscription-dialog-header"> Subscribe to Incident </h1>
</div>
<div class="modal-body">
<p style="margin-bottom:25px" id="incident-subscription-dialog-description">
Subscribe to updates for <strong>Querying and Ingest issues in EU</strong> via email and/or text message. You'll receive email notifications when incidents are updated, and text message notifications whenever Honeycomb <strong>creates</strong> or <strong>resolves</strong> an incident.
</p>
<div class="control-group">
<label for="email-pjzh0mtqw3vt">VIA EMAIL:</label>
<div class="controls">
<input type="text" name="email" id="email-pjzh0mtqw3vt" data-js-hook="email" class="full-width" />
</div>
</div>
<div class="control-group">
<label class="phone-country-code" for=phone-country-pjzh0mtqw3vt>VIA SMS:</label>
<div class="controls phone-number">
<div class="row">
<div id='phone-number-code-pjzh0mtqw3vt' class="phone-country-wrapper">
<select name="phone_country" id="phone-country-pjzh0mtqw3vt" data-js-hook="phone-country" class="span6 phone-country-dropdown"><option value="af" data-otp-enabled="false" >Afghanistan (+93)</option>
<option value="al" data-otp-enabled="false" >Albania (+355)</option>
<option value="dz" data-otp-enabled="false" >Algeria (+213)</option>
<option value="as" data-otp-enabled="false" >American Samoa (+1)</option>
<option value="ad" data-otp-enabled="false" >Andorra (+376)</option>
<option value="ao" data-otp-enabled="false" >Angola (+244)</option>
<option value="ai" data-otp-enabled="false" >Anguilla (+1)</option>
<option value="ag" data-otp-enabled="false" >Antigua and Barbuda (+1)</option>
<option value="ar" data-otp-enabled="false" >Argentina (+54)</option>
<option value="am" data-otp-enabled="false" >Armenia (+374)</option>
<option value="aw" data-otp-enabled="false" >Aruba (+297)</option>
<option value="au" data-otp-enabled="false" >Australia/Cocos/Christmas Island (+61)</option>
<option value="at" data-otp-enabled="false" >Austria (+43)</option>
<option value="az" data-otp-enabled="false" >Azerbaijan (+994)</option>
<option value="bs" data-otp-enabled="false" >Bahamas (+1)</option>
<option value="bh" data-otp-enabled="false" >Bahrain (+973)</option>
<option value="bd" data-otp-enabled="false" >Bangladesh (+880)</option>
<option value="bb" data-otp-enabled="false" >Barbados (+1)</option>
<option value="by" data-otp-enabled="false" >Belarus (+375)</option>
<option value="be" data-otp-enabled="false" >Belgium (+32)</option>
<option value="bz" data-otp-enabled="false" >Belize (+501)</option>
<option value="bj" data-otp-enabled="false" >Benin (+229)</option>
<option value="bm" data-otp-enabled="false" >Bermuda (+1)</option>
<option value="bo" data-otp-enabled="false" >Bolivia (+591)</option>
<option value="ba" data-otp-enabled="false" >Bosnia and Herzegovina (+387)</option>
<option value="bw" data-otp-enabled="false" >Botswana (+267)</option>
<option value="br" data-otp-enabled="false" >Brazil (+55)</option>
<option value="bn" data-otp-enabled="false" >Brunei (+673)</option>
<option value="bg" data-otp-enabled="false" >Bulgaria (+359)</option>
<option value="bf" data-otp-enabled="false" >Burkina Faso (+226)</option>
<option value="bi" data-otp-enabled="false" >Burundi (+257)</option>
<option value="kh" data-otp-enabled="false" >Cambodia (+855)</option>
<option value="cm" data-otp-enabled="false" >Cameroon (+237)</option>
<option value="ca" data-otp-enabled="false" >Canada (+1)</option>
<option value="cv" data-otp-enabled="false" >Cape Verde (+238)</option>
<option value="ky" data-otp-enabled="false" >Cayman Islands (+1)</option>
<option value="cf" data-otp-enabled="false" >Central Africa (+236)</option>
<option value="td" data-otp-enabled="false" >Chad (+235)</option>
<option value="cl" data-otp-enabled="false" >Chile (+56)</option>
<option value="cn" data-otp-enabled="false" >China (+86)</option>
<option value="co" data-otp-enabled="false" >Colombia (+57)</option>
<option value="km" data-otp-enabled="false" >Comoros (+269)</option>
<option value="cg" data-otp-enabled="false" >Congo (+242)</option>
<option value="cd" data-otp-enabled="false" >Congo, Dem Rep (+243)</option>
<option value="cr" data-otp-enabled="false" >Costa Rica (+506)</option>
<option value="hr" data-otp-enabled="false" >Croatia (+385)</option>
<option value="cy" data-otp-enabled="false" >Cyprus (+357)</option>
<option value="cz" data-otp-enabled="false" >Czech Republic (+420)</option>
<option value="dk" data-otp-enabled="false" >Denmark (+45)</option>
<option value="dj" data-otp-enabled="false" >Djibouti (+253)</option>
<option value="dm" data-otp-enabled="false" >Dominica (+1)</option>
<option value="do" data-otp-enabled="false" >Dominican Republic (+1)</option>
<option value="eg" data-otp-enabled="false" >Egypt (+20)</option>
<option value="sv" data-otp-enabled="false" >El Salvador (+503)</option>
<option value="gq" data-otp-enabled="false" >Equatorial Guinea (+240)</option>
<option value="ee" data-otp-enabled="false" >Estonia (+372)</option>
<option value="et" data-otp-enabled="false" >Ethiopia (+251)</option>
<option value="fo" data-otp-enabled="false" >Faroe Islands (+298)</option>
<option value="fj" data-otp-enabled="false" >Fiji (+679)</option>
<option value="fi" data-otp-enabled="false" >Finland/Aland Islands (+358)</option>
<option value="fr" data-otp-enabled="false" >France (+33)</option>
<option value="gf" data-otp-enabled="false" >French Guiana (+594)</option>
<option value="pf" data-otp-enabled="false" >French Polynesia (+689)</option>
<option value="ga" data-otp-enabled="false" >Gabon (+241)</option>
<option value="gm" data-otp-enabled="false" >Gambia (+220)</option>
<option value="ge" data-otp-enabled="false" >Georgia (+995)</option>
<option value="de" data-otp-enabled="false" >Germany (+49)</option>
<option value="gh" data-otp-enabled="false" >Ghana (+233)</option>
<option value="gi" data-otp-enabled="false" >Gibraltar (+350)</option>
<option value="gr" data-otp-enabled="false" >Greece (+30)</option>
<option value="gl" data-otp-enabled="false" >Greenland (+299)</option>
<option value="gd" data-otp-enabled="false" >Grenada (+1)</option>
<option value="gp" data-otp-enabled="false" >Guadeloupe (+590)</option>
<option value="gu" data-otp-enabled="false" >Guam (+1)</option>
<option value="gt" data-otp-enabled="false" >Guatemala (+502)</option>
<option value="gn" data-otp-enabled="false" >Guinea (+224)</option>
<option value="gy" data-otp-enabled="false" >Guyana (+592)</option>
<option value="ht" data-otp-enabled="false" >Haiti (+509)</option>
<option value="hn" data-otp-enabled="false" >Honduras (+504)</option>
<option value="hk" data-otp-enabled="false" >Hong Kong (+852)</option>
<option value="hu" data-otp-enabled="false" >Hungary (+36)</option>
<option value="is" data-otp-enabled="false" >Iceland (+354)</option>
<option value="in" data-otp-enabled="false" >India (+91)</option>
<option value="id" data-otp-enabled="false" >Indonesia (+62)</option>
<option value="iq" data-otp-enabled="false" >Iraq (+964)</option>
<option value="ie" data-otp-enabled="false" >Ireland (+353)</option>
<option value="il" data-otp-enabled="false" >Israel (+972)</option>
<option value="it" data-otp-enabled="false" >Italy (+39)</option>
<option value="jm" data-otp-enabled="false" >Jamaica (+1)</option>
<option value="jp" data-otp-enabled="false" >Japan (+81)</option>
<option value="jo" data-otp-enabled="false" >Jordan (+962)</option>
<option value="ke" data-otp-enabled="false" >Kenya (+254)</option>
<option value="kr" data-otp-enabled="false" >Korea, Republic of (+82)</option>
<option value="xk" data-otp-enabled="false" >Kosovo (+383)</option>
<option value="kw" data-otp-enabled="false" >Kuwait (+965)</option>
<option value="kg" data-otp-enabled="false" >Kyrgyzstan (+996)</option>
<option value="la" data-otp-enabled="false" >Laos (+856)</option>
<option value="lv" data-otp-enabled="false" >Latvia (+371)</option>
<option value="lb" data-otp-enabled="false" >Lebanon (+961)</option>
<option value="ls" data-otp-enabled="false" >Lesotho (+266)</option>
<option value="lr" data-otp-enabled="false" >Liberia (+231)</option>
<option value="ly" data-otp-enabled="false" >Libya (+218)</option>
<option value="li" data-otp-enabled="false" >Liechtenstein (+423)</option>
<option value="lt" data-otp-enabled="false" >Lithuania (+370)</option>
<option value="lu" data-otp-enabled="false" >Luxembourg (+352)</option>
<option value="mo" data-otp-enabled="false" >Macao (+853)</option>
<option value="mk" data-otp-enabled="false" >Macedonia (+389)</option>
<option value="mg" data-otp-enabled="false" >Madagascar (+261)</option>
<option value="mw" data-otp-enabled="false" >Malawi (+265)</option>
<option value="my" data-otp-enabled="false" >Malaysia (+60)</option>
<option value="mv" data-otp-enabled="false" >Maldives (+960)</option>
<option value="ml" data-otp-enabled="false" >Mali (+223)</option>
<option value="mt" data-otp-enabled="false" >Malta (+356)</option>
<option value="mq" data-otp-enabled="false" >Martinique (+596)</option>
<option value="mr" data-otp-enabled="false" >Mauritania (+222)</option>
<option value="mu" data-otp-enabled="false" >Mauritius (+230)</option>
<option value="mx" data-otp-enabled="false" >Mexico (+52)</option>
<option value="mc" data-otp-enabled="false" >Monaco (+377)</option>
<option value="mn" data-otp-enabled="false" >Mongolia (+976)</option>
<option value="me" data-otp-enabled="false" >Montenegro (+382)</option>
<option value="ms" data-otp-enabled="false" >Montserrat (+1)</option>
<option value="ma" data-otp-enabled="false" >Morocco/Western Sahara (+212)</option>
<option value="mz" data-otp-enabled="false" >Mozambique (+258)</option>
<option value="na" data-otp-enabled="false" >Namibia (+264)</option>
<option value="np" data-otp-enabled="false" >Nepal (+977)</option>
<option value="nl" data-otp-enabled="false" >Netherlands (+31)</option>
<option value="nz" data-otp-enabled="false" >New Zealand (+64)</option>
<option value="ni" data-otp-enabled="false" >Nicaragua (+505)</option>
<option value="ne" data-otp-enabled="false" >Niger (+227)</option>
<option value="ng" data-otp-enabled="false" >Nigeria (+234)</option>
<option value="no" data-otp-enabled="false" >Norway (+47)</option>
<option value="om" data-otp-enabled="false" >Oman (+968)</option>
<option value="pk" data-otp-enabled="false" >Pakistan (+92)</option>
<option value="ps" data-otp-enabled="false" >Palestinian Territory (+970)</option>
<option value="pa" data-otp-enabled="false" >Panama (+507)</option>
<option value="py" data-otp-enabled="false" >Paraguay (+595)</option>
<option value="pe" data-otp-enabled="false" >Peru (+51)</option>
<option value="ph" data-otp-enabled="false" >Philippines (+63)</option>
<option value="pl" data-otp-enabled="false" >Poland (+48)</option>
<option value="pt" data-otp-enabled="false" >Portugal (+351)</option>
<option value="pr" data-otp-enabled="false" >Puerto Rico (+1)</option>
<option value="qa" data-otp-enabled="false" >Qatar (+974)</option>
<option value="re" data-otp-enabled="false" >Reunion/Mayotte (+262)</option>
<option value="ro" data-otp-enabled="false" >Romania (+40)</option>
<option value="ru" data-otp-enabled="false" >Russia/Kazakhstan (+7)</option>
<option value="rw" data-otp-enabled="false" >Rwanda (+250)</option>
<option value="ws" data-otp-enabled="false" >Samoa (+685)</option>
<option value="sm" data-otp-enabled="false" >San Marino (+378)</option>
<option value="sa" data-otp-enabled="false" >Saudi Arabia (+966)</option>
<option value="sn" data-otp-enabled="false" >Senegal (+221)</option>
<option value="rs" data-otp-enabled="false" >Serbia (+381)</option>
<option value="sc" data-otp-enabled="false" >Seychelles (+248)</option>
<option value="sl" data-otp-enabled="false" >Sierra Leone (+232)</option>
<option value="sg" data-otp-enabled="false" >Singapore (+65)</option>
<option value="sk" data-otp-enabled="false" >Slovakia (+421)</option>
<option value="si" data-otp-enabled="false" >Slovenia (+386)</option>
<option value="za" data-otp-enabled="false" >South Africa (+27)</option>
<option value="es" data-otp-enabled="false" >Spain (+34)</option>
<option value="lk" data-otp-enabled="false" >Sri Lanka (+94)</option>
<option value="kn" data-otp-enabled="false" >St Kitts and Nevis (+1)</option>
<option value="lc" data-otp-enabled="false" >St Lucia (+1)</option>
<option value="vc" data-otp-enabled="false" >St Vincent Grenadines (+1)</option>
<option value="sd" data-otp-enabled="false" >Sudan (+249)</option>
<option value="sr" data-otp-enabled="false" >Suriname (+597)</option>
<option value="sz" data-otp-enabled="false" >Swaziland (+268)</option>
<option value="se" data-otp-enabled="false" >Sweden (+46)</option>
<option value="ch" data-otp-enabled="false" >Switzerland (+41)</option>
<option value="tw" data-otp-enabled="false" >Taiwan (+886)</option>
<option value="tj" data-otp-enabled="false" >Tajikistan (+992)</option>
<option value="tz" data-otp-enabled="false" >Tanzania (+255)</option>
<option value="th" data-otp-enabled="false" >Thailand (+66)</option>
<option value="tg" data-otp-enabled="false" >Togo (+228)</option>
<option value="to" data-otp-enabled="false" >Tonga (+676)</option>
<option value="tt" data-otp-enabled="false" >Trinidad and Tobago (+1)</option>
<option value="tn" data-otp-enabled="false" >Tunisia (+216)</option>
<option value="tr" data-otp-enabled="false" >Turkey (+90)</option>
<option value="tc" data-otp-enabled="false" >Turks and Caicos Islands (+1)</option>
<option value="ug" data-otp-enabled="false" >Uganda (+256)</option>
<option value="ua" data-otp-enabled="false" >Ukraine (+380)</option>
<option value="ae" data-otp-enabled="false" >United Arab Emirates (+971)</option>
<option value="gb" data-otp-enabled="false" >United Kingdom (+44)</option>
<option value="us" data-otp-enabled="false" selected>United States (+1)</option>
<option value="uy" data-otp-enabled="false" >Uruguay (+598)</option>
<option value="uz" data-otp-enabled="false" >Uzbekistan (+998)</option>
<option value="ve" data-otp-enabled="false" >Venezuela (+58)</option>
<option value="vn" data-otp-enabled="false" >Vietnam (+84)</option>
<option value="vg" data-otp-enabled="false" >Virgin Islands, British (+1)</option>
<option value="vi" data-otp-enabled="false" >Virgin Islands, U.S. (+1)</option>
<option value="ye" data-otp-enabled="false" >Yemen (+967)</option>
<option value="zm" data-otp-enabled="false" >Zambia (+260)</option>
<option value="zw" data-otp-enabled="false" >Zimbabwe (+263)</option></select>
</div>
</div>
</div>
<label class="sub-label" for="phone-number-pjzh0mtqw3vt">Enter mobile number</label>
<div class="controls phone-number" data-js-hook="modal-open-incident-subscribe-phone-number">
<div class="flex-container">
<input type="text" name="phone_number" id="phone-number-pjzh0mtqw3vt" data-js-hook="phone-number" />
<input type="hidden" name="incident-code-pjzh0mtqw3vt" id="incident-code-pjzh0mtqw3vt" value="pjzh0mtqw3vt" autocomplete="off" />
<a class="btn-change-number-incident" id="btn-change-number-incident-pjzh0mtqw3vt" style="display: none;" data-incident-code="pjzh0mtqw3vt" href="#">
Edit number
</a> <a class="btn-subcriber-send-otp" id="btn-subcriber-send-otp-pjzh0mtqw3vt" style="display: none;" data-incident-code="pjzh0mtqw3vt" href="#">
Send OTP
</a> </div>
</div>
<div class="sms-atl-error" id='sms-atl-error-pjzh0mtqw3vt'></div>
<div class="opt-container-section" id='otp-container-incident-pjzh0mtqw3vt' style="display:none">
<label class="sub-label" for="otp">Enter the OTP sent</label>
<div class="flex-container">
<input name="otp" id='otp-field-pjzh0mtqw3vt' type="text" class="prepend" disabled>
<a class="resend-otp" id="resend-otp-pjzh0mtqw3vt" style="display: none;" data-incident-code="pjzh0mtqw3vt" href="#">
Resend OTP
</a> <span class="timer-incident" id='timer-incident-pjzh0mtqw3vt' style="display:none">&nbsp;in <span id='countdown-incident-pjzh0mtqw3vt'>30</span> seconds</span>
</div>
<div class="info-row flex-container">
<div class="info-icon"><svg width="2vh" height="2vh" viewBox="0 0 21 21" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" id="status-embed-svg-info-icon">
<defs>
<path d="M2,12 C2,6.47666667 6.47666667,2 12,2 C17.5233333,2 22,6.47666667 22,12 C22,17.5233333 17.5233333,22 12,22 C6.47666667,22 2,17.5233333 2,12 Z M4,12 C4,16.4187638 7.58123617,20 12,20 C16.4187638,20 20,16.4187638 20,12 C20,7.58123617 16.4187638,4 12,4 C7.58123617,4 4,7.58123617 4,12 Z M11,11.0029293 C11,10.4490268 11.4438648,10 12,10 C12.5522847,10 13,10.4378814 13,11.0029293 L13,15.9970707 C13,16.5509732 12.5561352,17 12,17 C11.4477153,17 11,16.5621186 11,15.9970707 L11,11.0029293 Z M12,9 C11.4477153,9 11,8.55228475 11,8 C11,7.44771525 11.4477153,7 12,7 C12.5522847,7 13,7.44771525 13,8 C13,8.55228475 12.5522847,9 12,9 Z" id="path-info"></path>
</defs>
<g stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
<g transform="translate(-702.000000, -945.000000)">
<g id="global/info" transform="translate(700.500000, 943.500000)">
<mask id="mask-info" fill="white">
<use xlink:href="#path-info"></use>
</mask>
<use id="Combined-Shape" fill="#42526E" fill-rule="nonzero" xlink:href="#path-info"></use>
<g id="Neutral-/-N000" mask="url(#mask-info)" fill-rule="evenodd">
<polygon points="0 24 24 24 24 0 0 0"></polygon>
</g>
</g>
</g>
</g>
</svg>
</div>
<div>To receive SMS updates, please verify your number. To proceed with just email click ‘Subscribe’ </div>
</div>
</div>
</div>
</div>
<div class="modal-footer incident-subscribe">
<!-- tests fail if static width isn't present ¯\_(ツ)_/¯ -->
<button name="button" type="submit" class="flat-button cpt-button incident-subscribe-btn-captcha" id="subscribe-btn-pjzh0mtqw3vt" onclick="submitCaptchaIncidentSubscribe(event)">Subscribe to Incident</button>
<input type="hidden" name="captcha_error" id="captcha_error" value="false" autocomplete="off" />
<input type="hidden" name="g-recaptcha-response" id="g-recaptcha-response" value="false" autocomplete="off" />
<div class="terms_and_privacy_information left small">Message and data rates may apply. By subscribing you agree to the Atlassian <a target="_blank" rel="noopener" class="accessible-link" href="https://www.atlassian.com/legal/product-specific-terms#statuspage-specific-terms">Terms of Service</a>, and the Atlassian <a target="_blank" rel="noopener" class="accessible-link" href="https://www.atlassian.com/legal/privacy-policy">Privacy Policy</a>. This site is protected by reCAPTCHA and the Google <a target="_blank" rel="noopener" class="accessible-link" href="https://policies.google.com/privacy">Privacy Policy</a> and <a target="_blank" rel="noopener" class="accessible-link" data-js-hook="captcha-terms-of-service-link" href="https://policies.google.com/terms">Terms of Service</a> apply.</div>
</div>
</form> </div>
<script>
$(function () {
const phoneNumberInputIncident = $('#phone-number-pjzh0mtqw3vt');
const phoneCountrySelect = $('#phone-country-pjzh0mtqw3vt');
const errorDiv = $('#sms-atl-error-pjzh0mtqw3vt');
const phoneCountryDiv = $('#phone-number-code-pjzh0mtqw3vt');
if(errorDiv.length){
function checkSelectedCountry() {
const selectedCountry = phoneCountrySelect.val();
const isOtpEnabled = phoneCountryDiv.attr('data-otp-enabled') === 'true';
if(false && selectedCountry === 'sg') { // Replace 'SG' with the actual value representing Singapore in your select tag
phoneNumberInputIncident.prop('disabled', true);
errorDiv.html(`Due to new Singapore government regulations, we're currently not supporting text subscriptions in Singapore.<a href="https://community.atlassian.com/t5/Statuspage-articles/Attention-SMS-notifications-will-be-disabled-on-August-1st-2023/ba-p/2424398" target="_blank"> Learn more.</a> <br> Select another method to subscribe.`);
} else {
phoneNumberInputIncident.prop('readOnly', false);
errorDiv.html('');
if(false){
$('#btn-change-number-incident-pjzh0mtqw3vt').css('display', 'none');
$('#resend-otp-pjzh0mtqw3vt').css('display', 'none');
$('#timer-incident-pjzh0mtqw3vt').css('display', 'none');
if(isOtpEnabled){
$('#otp-container-incident-pjzh0mtqw3vt').css('display', 'block');
$('#btn-subcriber-send-otp-pjzh0mtqw3vt').css('display', 'block');
}
else {
$('#otp-container-incident-pjzh0mtqw3vt').css('display', 'none');
$('#btn-subcriber-send-otp-pjzh0mtqw3vt').css('display', 'none');
}
}
}
}
phoneCountrySelect.on('change', checkSelectedCountry);
checkSelectedCountry();
}
});
document.addEventListener('DOMContentLoaded', function() {
const dropdowns = document.querySelectorAll('.phone-country-dropdown');
if (dropdowns.length > 0) {
dropdowns.forEach(function(dropdown) {
const dropdownId = dropdown.id;
const incidentCode = dropdownId.split('-').pop();
const wrapperDiv = $('#phone-number-code-' + incidentCode);
function updateOtpEnabledAttribute() {
const selectedOption = dropdown.options[dropdown.selectedIndex];
const otpEnabled = selectedOption.getAttribute('data-otp-enabled');
wrapperDiv.attr('data-otp-enabled', otpEnabled);
}
dropdown.addEventListener('change', updateOtpEnabledAttribute);
updateOtpEnabledAttribute();
});
}
});
var countdownTimers = {};
var phoneNumberInputIncident = $('#phone-number-pjzh0mtqw3vt');
var RESEND_TIMER = 30;
$(function() {
$('#subscribe-form-pjzh0mtqw3vt').on('ajax:success', function(e, data, status, xhr){
var $form = $(this);
var formId = $form.attr('id');
var incidentCode = formId.split('-').pop();
$('#btn-change-number-incident-' + incidentCode).css('display', 'block');
if ( $('#btn-change-number-incident-'+ incidentCode).css('display') !== 'none') {
$('#btn-change-number-incident-'+ incidentCode).css('display', 'none');
$('#btn-subcriber-send-otp-'+ incidentCode).css('display', 'block');
$('#otp-container-incident-'+ incidentCode).css('display', 'block');
$('#resend-otp-'+ incidentCode).css('display', 'none');
$('#timer-incident-'+ incidentCode).css('display', 'none');
$('#phone-number-' + incidentCode).prop('readOnly', false);
$('#otp-field-' + incidentCode).val('').prop('disabled', true);
}
if(countdownTimers){
clearInterval(countdownTimers[incidentCode]);
}
});
$('#btn-change-number-incident-pjzh0mtqw3vt').on('click', function (e) {
var incidentCode = $(this).data('incident-code');
showSendOTP(incidentCode);
return false;
});
$('#btn-subcriber-send-otp-pjzh0mtqw3vt').on('click', function (e) {
let incidentCode = $(this).data('incident-code');
let phoneNumber = $('#phone-number-' + incidentCode).val();
let countryCode = $('#phone-country-' + incidentCode).val();
let incidentCodeField = $('#incident-code-' + incidentCode).val();
sendOtpRequest(phoneNumber, countryCode, incidentCodeField, function (data, status) {
var messageOptions = (data.type !== undefined && data.type !== null) ? { cssClass: data.type } : {};
HRB.utils.notify(data.text, messageOptions);
e.preventDefault();
if (data.type === 'success') {
showChangeNumber(incidentCode);
}
});
});
$('#resend-otp-pjzh0mtqw3vt').on('click', function(e) {
let incidentCode = $(this).data('incident-code');
let phoneNumber = $('#phone-number-' + incidentCode).val();
let countryCode = $('#phone-country-' + incidentCode).val();
let incidentCodeField = $('#incident-code-' + incidentCode).val();
sendOtpRequest(phoneNumber, countryCode, incidentCodeField, function (data, status) {
var messageOptions = (data.type !== undefined && data.type !== null) ? { cssClass: data.type } : {};
HRB.utils.notify(data.text, messageOptions);
e.preventDefault();
disableResendIncident(incidentCode);
if (data.type === 'success') {
var display = $('#countdown-incident-' + incidentCode);
startTimerIncident(RESEND_TIMER, display, incidentCode );
}
});
});
})
function sendOtpRequest(phoneNumber, countryCode, incidentCode, onSuccess) {
$.ajax({
type: 'POST',
url: "/subscriptions/new-sms",
data: {
phone_number: phoneNumber,
phone_country: countryCode,
incident_code: incidentCode,
type: 'resend'
},
}).done(onSuccess);
}
function showChangeNumber(incidentCode) {
$('#btn-change-number-incident-' + incidentCode).css('display', 'block');
$('#btn-subcriber-send-otp-' + incidentCode).css('display', 'none');
$('#otp-field-' + incidentCode).val('').prop('disabled', false);
$('#phone-number-' + incidentCode).prop('readOnly', true);
$('#resend-otp-'+ incidentCode).css('display', 'block');
var display = $('#countdown-incident-' + incidentCode);
startTimerIncident(RESEND_TIMER, display, incidentCode)
}
function showSendOTP(incidentCode) {
$('#btn-change-number-incident-' + incidentCode).css('display', 'none');
$('#btn-subcriber-send-otp-' + incidentCode).css('display', 'block');
$('#otp-field-' + incidentCode).val('').prop('disabled', true);
$('#phone-number-' + incidentCode).prop('readOnly', false);
$('#resend-otp-'+ incidentCode).css('display', 'none');
$('#timer-incident-'+ incidentCode).css('display', 'none');
clearInterval(countdownTimers[incidentCode]);
}
function startTimerIncident(duration, display, incidentCode){
var timerIncident = duration, seconds;
clearInterval(countdownTimers[incidentCode]);
countdownTimers[incidentCode] = setInterval(function () {
seconds = parseInt(timerIncident % 60, 10);
display.text(seconds);
$('#timer-incident-'+ incidentCode).css('display', 'block');
if(--timerIncident < 0){
enableResendIncident(incidentCode);
clearInterval(countdownTimers[incidentCode]);
}
}, 1000);
disableResendIncident(incidentCode);
}
function enableResendIncident(incidentCode){
$('#resend-otp-' + incidentCode).css('color', '');
$('#resend-otp-' + incidentCode).css('pointer-events', '');
$('#timer-incident-'+ incidentCode).css('display', 'none');
}
function disableResendIncident(incidentCode){
$('#resend-otp-' + incidentCode).css('color', 'grey');
$('#resend-otp-' + incidentCode).css('pointer-events', 'none');
}
</script>
<script src="https://dka575ofm4ao0.cloudfront.net/assets/status_manifest-9446cbd3af4b0e3706e85200ffeb5f09d6410458c7aea582354bfd4bc5546b5e.js"></script>
<div id="cpt-notification-container" role="status" aria-live="polite" aria-atomic="true"></div>
<!-- all of the content_for stuff -->
<script src="https://dka575ofm4ao0.cloudfront.net/assets/register_subscription_form-589b657fec607087fc5c740c568270907310bc4f6aaa20256e70f01b103025ca.js"></script>
<script type="text/javascript">
$(function() {
SP.currentPage.registerSubscriptionForm('email');
SP.currentPage.registerSubscriptionForm('sms');
SP.currentPage.registerSubscriptionForm('webhook');
SP.currentPage.registerSubscriptionForm('teams');
});
$(function() {
SP.currentPage.registerSubscriptionForm('pjzh0mtqw3vt');
});
</script>
<script src="https://dka575ofm4ao0.cloudfront.net/assets/status_common-a86dffb81955091741c3de8bded6ce66ae0c6d8f736b57b580f853d9f892727e.js"></script>
<script>
$(function() { // docks custom footer
var content = $('.layout-content > .container')
, header = $('.custom-header-container')
, footer = $('.custom-footer-container')
if (!footer.length) {
content.addClass('default-spacing');
}
else {
var screenHeight = $(window).outerHeight()
, containerHeight = $('.layout-content > .container').outerHeight(true) || null
, headerHeight = $('.custom-header-container').outerHeight(true) || null
, footerHeight = $('.custom-footer-container').outerHeight(true) || null;
if (screenHeight > containerHeight + footerHeight + headerHeight) {
$('.layout-content > .container').css('padding-bottom', screenHeight - containerHeight - footerHeight - headerHeight);
}
}
});
$(function() {
$(document).on('ajax:complete', '.modal.in', function(e) {
// Close the active modal.
$('.modal.in').modal('hide');
});
});
$(function() {
var timeDifference = function(previous) {
var msPerMinute = 60 * 1000;
var msPerHour = msPerMinute * 60;
var msPerDay = msPerHour * 24;
var msPerMonth = msPerDay * 30;
var msPerYear = msPerDay * 365;
var now = Date.now();
var elapsedMs = now - previous;
var elapsed = Math.round(elapsedMs/msPerMinute);
if (elapsed === 0) {
return elapsedString(1) + ' minute ago. ';
}
else if (elapsed < 60) {
return elapsedString(elapsed) + ' minute' + plural(elapsed) + ' ago. ';
}
elapsed = Math.round(elapsedMs/msPerHour);
if (elapsed < 24) {
return elapsedString(elapsed) + ' hour' + plural(elapsed) + ' ago. ';
}
elapsed = Math.round(elapsedMs/msPerDay);
if (elapsed < 31) {
return elapsedString(elapsed) + ' day' + plural(elapsed) + ' ago. ';
}
elapsed = Math.round(elapsedMs/msPerMonth);
if (elapsed < 12) {
return elapsedString(elapsed) + ' month' + plural(elapsed) + ' ago. ';
}
elapsed = Math.round(elapsedMs/msPerYear);
return elapsedString(elapsed) + ' year' + plural(elapsed) + ' ago. ';
};
var elapsedString = function(elapsed) {
return '<var data-var="num">' + elapsed + '</var>'
};
var plural = function(x) {
return (x <= 1) ? '' : 's';
};
var relative_dates = $('span.ago');
relative_dates.each(function() {
var $el = $(this);
var time = $el.data("datetime-unix");
$el.html(timeDifference(time));
})
});
</script>
<script>
/** INITIALIZATION **/
var recaptchaIds = {}
// Unfortunately there's no unique selectors on the parent divs that recaptcha adds. The first unique selector
// is the iframe rendered 2 levels deep. So this waits until the iframes are added to the page, then finds
// the parent div and sets the z index so that it'll render above our modals & dropdowns from the start.
function setZIndex(captchaCount, startTime) {
// bail after 10s just in case so we don't do this forever if something whaky happens
if (new Date() - startTime > 10000) {
return;
}
var iframes = document.querySelectorAll('iframe[title="recaptcha challenge"]');
if (iframes.length != captchaCount) {
setTimeout(function() {
setZIndex(captchaCount, startTime);
}, 500);
}
for (var i = 0; i < iframes.length; i++) {
// incident subscribe modal is 1050, so this has to be above that
iframes[i].parentElement.parentElement.style.zIndex = "1100";
}
}
function updateCaptchaIframeTitle(captchaCount, startTime, updates=0) {
if (new Date() - startTime > 10000 || captchaCount === updates) {
return;
}
var iframesWithTitle = document.querySelectorAll('iframe[title="recaptcha challenge expires in two minutes"]');
if (iframesWithTitle.length != captchaCount) {
setTimeout(function() {
updateCaptchaIframeTitle(captchaCount, startTime, iframesWithTitle.length + updates);
}, 500);
}
for (var i = 0; i < iframesWithTitle.length; i++) {
iframesWithTitle[i].title = "recaptcha";
}
}
function addIncidentCaptcha() {
var incidentCaptcha = document.createElement('div');
incidentCaptcha.setAttribute('id', 'subscribe-incident-recaptcha');
incidentCaptcha.setAttribute('class', 'g-recaptcha');
incidentCaptcha.setAttribute('data-sitekey', '6LcZ-b0UAAAAAENi956aWzynTT2ZJ80dGU3F80Op');
incidentCaptcha.setAttribute('data-callback', 'submitIncidentSubscriberSuccess');
incidentCaptcha.setAttribute('data-error-callback', 'submitIncidentSubscriberError');
incidentCaptcha.setAttribute('data-size', 'invisible');
document.body.appendChild(incidentCaptcha);
var incidentCode = document.createElement('input');
incidentCode.setAttribute('type', 'hidden');
incidentCode.setAttribute('id', 'submit_incident_code');
document.body.appendChild(incidentCode);
}
var onloadCallback = function() {
// if there is an incident, then add incident captcha element
if (document.getElementsByClassName('modal-open-incident-subscribe').length > 0) {
addIncidentCaptcha();
}
var captchas = document.getElementsByClassName("g-recaptcha");
for(var i = 0; i < captchas.length; i++) {
var elId = captchas[i].id;
recaptchaIds[elId] = grecaptcha.enterprise.render(elId);
}
setZIndex(captchas.length, new Date());
updateCaptchaIframeTitle(captchas.length, new Date());
}
/** SUBSCRIBE DROPDOWN */
// callbacks for captcha success
function submitNewSubscriber(type, error) {
if (error) document.querySelector('#subscribe-form-' + type + ' #captcha_error').value = 'true';
document.getElementById('subscribe-form-' + type).dispatchEvent(new Event('submit', {bubbles: true, cancelable: true}));
grecaptcha.enterprise.reset(recaptchaIds['subscribe-btn-' + type]);
}
function submitNewEmailSubscriber(token) {
submitNewSubscriber('email');
}
function submitNewSmsSubscriber(token) {
submitNewSubscriber('sms');
}
function submitNewWebhookSubscriber(token) {
submitNewSubscriber('webhook');
}
function submitIncidentSubscriber(token, error) {
var incidentCode = document.getElementById('submit_incident_code').value;
var incidentForm = document.getElementById('subscribe-form-' + incidentCode);
incidentForm.querySelector('input[name="captcha_error"]').value = error;
incidentForm.querySelector('input[name="g-recaptcha-response"]').value = token;
incidentForm.dispatchEvent(new Event('submit', {bubbles: true, cancelable: true}));
grecaptcha.enterprise.reset(recaptchaIds['subscribe-incident-recaptcha']);
}
function submitIncidentSubscriberSuccess(token) {
submitIncidentSubscriber(token, 'false');
}
// callbacks if we get captcha network errors
function emailSubscriberCaptchaError(token) {
submitNewSubscriber('email', true);
}
function smsSubscriberCaptchaError(token) {
submitNewSubscriber('sms', true);
}
function webhookSubscriberCaptchaError(token) {
submitNewSubscriber('webhook', true);
}
function submitIncidentSubscriberError(token) {
submitIncidentSubscriber(token, 'true');
}
// tracking clicks
['email', 'sms', 'webhook'].forEach(function(type) {
var el = document.getElementById('subscribe-btn-' + type);
el && el.addEventListener("click", function() {
$.ajax({
type: "POST",
url: "/subscriptions/track_attempt",
data: {
type: type
}
})
})
})
// form submission success callbacks
$('#subscribe-form-email').on('ajax:success', function(e, data, status, xhr){
if (data.type === 'success') {
SP.currentPage.updatesDropdown.hide();
var emailField = document.getElementById('email');
if (emailField) {
emailField.value = '';
}
}
});
$('#subscribe-form-sms').on('ajax:success', function(e, data, status, xhr){
if (data.type === 'success' && data.otp_flow !== true) {
SP.currentPage.updatesDropdown.hide();
var phoneField = document.getElementById('phone-number');
if (phoneField) {
phoneField.value = '';
}
}
});
$('#subscribe-form-webhook').on('ajax:success', function(e, data, status, xhr){
if (data.type === 'success') {
SP.currentPage.updatesDropdown.hide();
document.getElementById('endpoint-webhooks').value = '';
document.getElementById('email-webhooks').value = '';
}
});
$('a.subscribe').on('click', function() {
document.body.style.overflow = "hidden";
document.body.style.height = "100vh";
});
$('div.modal-open-incident-subscribe').on('hidden', function(){
document.body.style.overflow = "";
document.body.style.height = "";
});
function submitCaptchaIncidentSubscribe(event) {
var incidentCode = event.target.id.split('-')[2];
event.preventDefault();
$.ajax({
type: "POST",
url: "/subscriptions/track_attempt",
data: {
type: 'incident'
}
})
document.getElementById('submit_incident_code').value = incidentCode;
grecaptcha.enterprise.execute(recaptchaIds['subscribe-incident-recaptcha']);
}
</script>
<script src='https://www.recaptcha.net/recaptcha/enterprise.js?onload=onloadCallback&render=explicit' async defer></script>
<script src="https://dka575ofm4ao0.cloudfront.net/packs/common-3e6e8c878eea26f45b29.chunk.js"></script>
<script src="https://dka575ofm4ao0.cloudfront.net/packs/globals-f12471d5e878ec0f0705.chunk.js"></script>
<script src="https://dka575ofm4ao0.cloudfront.net/packs/runtime-33399279767e29d9b814.js"></script>
<script>
window.addEventListener('load', function () {
const urlParams = new URLSearchParams(window.location.search);
const messageToken = urlParams.get('slack_message_token');
const channelName = escape(urlParams.get('channel_name'));
if(!!messageToken) {
switch(messageToken) {
case 'slack_auth_error':
HRB.utils.notify('The Slack authorization attempt was unsuccessful. Try again.', {cssClass:'error'});
break;
case 'subscribers_disabled_error':
HRB.utils.notify('Slack subscriptions are not enabled on this page.', {cssClass:'error'});
break;
case 'direct_message_channel_error':
HRB.utils.notify('Subscriptions aren’t supported in direct messages. Try subscribing again and choose a channel instead.', {cssClass:'error'});
break
case 'duplicate_error':
HRB.utils.notify("You're already subscribed to get Slack notifications in that channel.", {cssClass:'error'});
break;
case 'duplicate_private_channel_error':
HRB.utils.notify(`You're already subscribed to get Slack notifications in #${channelName}. Invite the @Statuspage app to that channel to start getting status updates.`, {cssClass: 'error'});
break;
case 'default_success':
HRB.utils.notify("You're now subscribed to get Statuspage updates in Slack!", {cssClass:'success'});
break;
case 'private_channel_success':
HRB.utils.notify(`IMPORTANT: Invite the @Statuspage app to your Slack channel #${channelName} to start getting status updates.`, {cssClass:'success'});
break;
}
}
});
</script>
<!-- FOR FLASH NOTICES -->
<!-- FOR ERROR -->
<script>
$(function() {
var $link = $('<span class="color-secondary powered-by"><a class="color-secondary" target="_blank" rel="noopener noreferrer nofollow" href="https://www.atlassian.com/software/statuspage?utm_campaign=status.honeycomb.io&amp;utm_content=SP-notifications&amp;utm_medium=powered-by&amp;utm_source=inapp">Powered by Atlassian Statuspage</a></span>');
var setPoweredByStyles = function() {
if (!$('.powered-by').length) {
$link.appendTo($('.page-footer'))
}
$('.powered-by').attr('style', 'display: inline !important; visibility:visible !important; opacity: 1 !important; position:static !important; text-indent:0px !important; transform:scale(1) !important');
}
setInterval(setPoweredByStyles, 1000);
});
</script>
</body>
</html>