3748 lines
253 KiB
HTML
3748 lines
253 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<title>Incident Retro: Failing Comment Creation + Erroneous Push Notifications - DEV Community</title>
|
||
|
||
<link rel="preload" href="/reactions?article_id=759150" as="fetch" crossorigin="same-origin">
|
||
<link rel="canonical" href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj" />
|
||
<meta name="description" content="This incident retro was tougher than most to share because, despite the seriousness of the issue, it... Tagged with incident, retro, postmortem.">
|
||
<meta name="keywords" content="incident, retro, postmortem, software, coding, development, engineering, inclusive, community">
|
||
|
||
<meta property="og:type" content="article" />
|
||
<meta property="og:url" content="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj" />
|
||
<meta property="og:title" content="Incident Retro: Failing Comment Creation + Erroneous Push Notifications" />
|
||
<meta property="og:description" content="This incident retro was tougher than most to share because, despite the seriousness of the issue, it..." />
|
||
<meta property="og:site_name" content="DEV Community" />
|
||
<meta name="twitter:site" content="@thepracticaldev">
|
||
<meta name="twitter:creator" content="@molly_struve">
|
||
<meta name="author-trust" content="4">
|
||
<meta name="twitter:title" content="Incident Retro: Failing Comment Creation + Erroneous Push Notifications">
|
||
<meta name="twitter:description" content="This incident retro was tougher than most to share because, despite the seriousness of the issue, it...">
|
||
<meta name="twitter:card" content="summary_large_image">
|
||
<meta name="twitter:widgets:new-embed-design" content="on">
|
||
<meta name="robots" content="max-snippet:-1, max-image-preview:large, max-video-preview:-1">
|
||
<meta property="og:image" content="https://media2.dev.to/dynamic/image/width=1000,height=500,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2Fivyfkw7k643ivg4ltdd5.png" />
|
||
<meta name="twitter:image:src" content="https://media2.dev.to/dynamic/image/width=1000,height=500,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2Fivyfkw7k643ivg4ltdd5.png">
|
||
|
||
<meta name="last-updated" content="2026-09-12 01:47:43 UTC">
|
||
<meta name="user-signed-in" content="false">
|
||
<meta name="head-cached-at" content="1789177663">
|
||
<meta name="environment" content="production">
|
||
<link rel="stylesheet" href="https://assets.dev.to/assets/minimal-3a438d8d138f95f478f01cbe53363cacf36bc6c47172aeb2e6c9172eebe29aa6.css" media="all" id="main-minimal-stylesheet" />
|
||
<link rel="stylesheet" href="https://assets.dev.to/assets/views-ca4b6f6e9ec63c7deefbdd4067e585d752cd4b06c12da1a49c7b9644defa2758.css" media="all" id="main-views-stylesheet" />
|
||
<link rel="stylesheet" href="https://assets.dev.to/assets/crayons-537acd1cbd95b8d8d524dab489c8923b7316dd045ead39e7a2dbaa42cdc208b2.css" media="all" id="main-crayons-stylesheet" />
|
||
|
||
<script src="https://assets.dev.to/assets/base-2e294c52c5fe6b771fda1a5f38d2f10756d0b4d8359c6f9825fe47df0f54e6ae.js" defer="defer"></script>
|
||
<script src="https://assets.dev.to/assets/application-9ee88e198a0d2b64238c8557b5225d20e2c18ff5b877f1159ee5a31f6e3174f9.js" defer="defer"></script>
|
||
<script src="https://assets.dev.to/assets/baseInitializers-624554190e72cf52de9498fd225c2e71ab334dfc4919535dcc5a26a3a0ae2359.js" defer="defer"></script>
|
||
<script src="https://assets.dev.to/assets/baseTracking-cb478131fb1b41bc25e80ebcee996318777e081c9aa87367fabc1c118461a2d3.js" defer="defer"></script>
|
||
<script src="https://assets.dev.to/assets/followButtons-e3595664847e9f43945a9b8a0445b32cda9a149f1f169f78b8fcc7714ccab45a.js" defer="defer"></script>
|
||
<script src="https://assets.dev.to/assets/eventSignupButtons-a2b5da712a1b8eeb0eda4d63fafc54973138a70b88503a2163806c27fc1e4c39.js" defer="defer"></script>
|
||
|
||
<meta name="search-script" content="https://assets.dev.to/assets/Search-a570c3428c9b6cb070d3f18817c957f80d0dbdf36a0f4a1d6e23a990305fbc12.js">
|
||
<meta name="mermaid-script" content="https://assets.dev.to/assets/mermaidRenderer-b9ba305a9767f9203ac04b8043493fb0542090e9a7981428cecf8c7d2ccaf177.js">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover">
|
||
<link rel="icon" type="image/x-icon" href="https://media2.dev.to/dynamic/image/width=32,height=,fit=scale-down,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2F8j7kvp660rqzt99zui8e.png" />
|
||
<link rel="apple-touch-icon" href="https://media2.dev.to/dynamic/image/width=180,height=,fit=scale-down,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2F8j7kvp660rqzt99zui8e.png">
|
||
<link rel="apple-touch-icon" sizes="152x152" href="https://media2.dev.to/dynamic/image/width=152,height=,fit=scale-down,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2F8j7kvp660rqzt99zui8e.png">
|
||
<link rel="apple-touch-icon" sizes="180x180" href="https://media2.dev.to/dynamic/image/width=180,height=,fit=scale-down,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2F8j7kvp660rqzt99zui8e.png">
|
||
<link rel="apple-touch-icon" sizes="167x167" href="https://media2.dev.to/dynamic/image/width=167,height=,fit=scale-down,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2F8j7kvp660rqzt99zui8e.png">
|
||
<link href="https://media2.dev.to/dynamic/image/width=192,height=,fit=scale-down,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2F8j7kvp660rqzt99zui8e.png" rel="icon" sizes="192x192" />
|
||
<link href="https://media2.dev.to/dynamic/image/width=128,height=,fit=scale-down,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2F8j7kvp660rqzt99zui8e.png" rel="icon" sizes="128x128" />
|
||
<meta name="apple-mobile-web-app-title" content="dev.to">
|
||
<meta name="application-name" content="dev.to">
|
||
<meta name="theme-color" content="#ffffff" media="(prefers-color-scheme: light)">
|
||
<meta name="theme-color" content="#000000" media="(prefers-color-scheme: dark)">
|
||
<link rel="search" href="https://dev.to/open-search.xml" type="application/opensearchdescription+xml" title="DEV Community" />
|
||
|
||
<link rel="describedby" href="https://dev.to/llms.txt" type="text/plain" title="Guidance for AI agents" />
|
||
|
||
<meta property="forem:name" content="DEV Community" />
|
||
<meta property="forem:logo" content="https://media2.dev.to/dynamic/image/width=512,height=,fit=scale-down,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2F8j7kvp660rqzt99zui8e.png" />
|
||
<meta property="forem:domain" content="dev.to" />
|
||
<style>
|
||
body:has(.pageslug-aie) #topbar {
|
||
display: none !important;
|
||
}
|
||
|
||
@media screen and (max-width: 768px) and (pointer: coarse) {
|
||
/* Target WebKit/Blink specific behaviors */
|
||
@supports (-webkit-app-region: none) {
|
||
body:has(.pageslug-aie) .dc-page {
|
||
padding-top: 56px !important;
|
||
}
|
||
}
|
||
}
|
||
|
||
body:not(.hidden-shell) .pageslug-aie {
|
||
margin-top: -56px !important;
|
||
}
|
||
|
||
/* ==========================================================================
|
||
THE 418 CHALLENGE: THEME-AWARE RETRO EDITION
|
||
========================================================================== */
|
||
.articletag-418challenge {
|
||
font-family: "Comic Sans MS", "Chalkboard SE", "Marker Felt", sans-serif !important;
|
||
cursor: crosshair !important;
|
||
}
|
||
|
||
/* Destroy modern rounded corners and add brutal borders */
|
||
.articletag-418challenge .crayons-card,
|
||
.articletag-418challenge .crayons-article__main,
|
||
.articletag-418challenge .crayons-article__header {
|
||
border-radius: 0 !important;
|
||
margin-bottom: 2rem !important;
|
||
}
|
||
|
||
/* Typography specifics */
|
||
.articletag-418challenge h1,
|
||
.articletag-418challenge h2,
|
||
.articletag-418challenge h3 {
|
||
text-transform: uppercase !important;
|
||
}
|
||
|
||
/* Fake <marquee> effect for the main article title */
|
||
.articletag-418challenge h1.fs-3xl {
|
||
animation: retro-slide 5s linear infinite alternate;
|
||
overflow: visible;
|
||
}
|
||
|
||
/* Spin those avatars */
|
||
.articletag-418challenge .crayons-article__header__meta img {
|
||
border-radius: 0 !important;
|
||
animation: retro-spin 5s linear infinite;
|
||
}
|
||
|
||
/* Obnoxious tags */
|
||
.articletag-418challenge .crayons-tag {
|
||
border-radius: 0 !important;
|
||
font-weight: 900 !important;
|
||
transform: rotate(-3deg);
|
||
display: inline-block;
|
||
}
|
||
.articletag-418challenge .crayons-tag:nth-child(even) {
|
||
transform: rotate(3deg);
|
||
}
|
||
|
||
/* --------------------------------------------------------------------------
|
||
2. LIGHT THEME (Windows 95 / Office 97 Vibes)
|
||
-------------------------------------------------------------------------- */
|
||
body:not(.dark-theme) .articletag-418challenge {
|
||
background-color: #008080 !important; /* Win95 Teal Desktop */
|
||
}
|
||
|
||
body:not(.dark-theme) .articletag-418challenge .crayons-card,
|
||
body:not(.dark-theme) .articletag-418challenge .crayons-article__main,
|
||
body:not(.dark-theme) .articletag-418challenge .crayons-article__header {
|
||
background-color: #c0c0c0 !important; /* Classic dialog gray */
|
||
color: #000000 !important;
|
||
border: 4px outset #ffffff !important;
|
||
box-shadow: 8px 8px 0px #000000 !important;
|
||
}
|
||
|
||
body:not(.dark-theme) .articletag-418challenge h1,
|
||
body:not(.dark-theme) .articletag-418challenge h2,
|
||
body:not(.dark-theme) .articletag-418challenge h3 {
|
||
color: #ff0000 !important;
|
||
text-shadow: 2px 2px 0px #ffff00 !important;
|
||
border-bottom: 3px dashed #0000ff !important;
|
||
}
|
||
|
||
body:not(.dark-theme) .articletag-418challenge a,
|
||
body:not(.dark-theme) .articletag-418challenge .crayons-link {
|
||
color: #0000ff !important; /* Standard unvisited blue */
|
||
text-decoration: underline !important;
|
||
font-weight: bold !important;
|
||
}
|
||
|
||
body:not(.dark-theme) .articletag-418challenge a:hover,
|
||
body:not(.dark-theme) .articletag-418challenge .crayons-link:hover {
|
||
background-color: #ffff00 !important;
|
||
color: #ff0000 !important;
|
||
}
|
||
|
||
body:not(.dark-theme) .articletag-418challenge .crayons-tag {
|
||
background: #ffff00 !important;
|
||
color: #ff0000 !important;
|
||
border: 2px dotted #0000ff !important;
|
||
}
|
||
|
||
/* --------------------------------------------------------------------------
|
||
3. DARK THEME (1999 Hacker / Deep GeoCities Vibes)
|
||
-------------------------------------------------------------------------- */
|
||
body.dark-theme .articletag-418challenge {
|
||
background-color: #000000 !important;
|
||
/* Dumb CSS: A scrolling, eye-bleeding neon green hacker grid with CRT scanlines */
|
||
background-image:
|
||
linear-gradient(transparent 50%, rgba(0, 255, 0, 0.25) 50%),
|
||
linear-gradient(90deg, rgba(0, 255, 0, 0.5) 1px, transparent 1px),
|
||
linear-gradient(rgba(0, 255, 0, 0.5) 1px, transparent 1px) !important;
|
||
background-size: 100% 4px, 20px 20px, 20px 20px !important;
|
||
animation: retro-pan 60s linear infinite !important;
|
||
}
|
||
|
||
body.dark-theme .articletag-418challenge .crayons-card,
|
||
body.dark-theme .articletag-418challenge .crayons-article__main,
|
||
body.dark-theme .articletag-418challenge .crayons-article__header {
|
||
background-color: #111111 !important;
|
||
color: #00ff00 !important; /* Terminal Green */
|
||
border: 4px outset #555555 !important;
|
||
box-shadow: 8px 8px 0px #ff00ff !important; /* Hot pink shadow */
|
||
}
|
||
|
||
body.dark-theme .articletag-418challenge h1,
|
||
body.dark-theme .articletag-418challenge h2,
|
||
body.dark-theme .articletag-418challenge h3 {
|
||
color: #ff00ff !important; /* Magenta headers */
|
||
text-shadow: 2px 2px 0px #00ffff !important; /* Cyan shadow */
|
||
border-bottom: 3px dashed #00ff00 !important;
|
||
}
|
||
|
||
body.dark-theme .articletag-418challenge a,
|
||
body.dark-theme .articletag-418challenge .crayons-link {
|
||
color: #00ffff !important; /* Cyan links */
|
||
text-decoration: underline !important;
|
||
font-weight: bold !important;
|
||
}
|
||
|
||
body.dark-theme .articletag-418challenge a:hover,
|
||
body.dark-theme .articletag-418challenge .crayons-link:hover {
|
||
background-color: #ff00ff !important;
|
||
color: #ffffff !important;
|
||
}
|
||
|
||
body.dark-theme .articletag-418challenge .crayons-tag {
|
||
background: #000000 !important;
|
||
color: #00ff00 !important;
|
||
border: 2px dotted #ff00ff !important;
|
||
}
|
||
|
||
/* --------------------------------------------------------------------------
|
||
4. ANIMATIONS
|
||
-------------------------------------------------------------------------- */
|
||
@keyframes retro-spin {
|
||
100% { transform: rotate(360deg); }
|
||
}
|
||
|
||
@keyframes retro-slide {
|
||
0% { transform: translateX(-2%); }
|
||
100% { transform: translateX(2%); }
|
||
}
|
||
|
||
@keyframes retro-pan {
|
||
0% { background-position: 0 0, 0 0, 0 0; }
|
||
/* Numbers must be multiples of the background-size (4px and 20px) to loop perfectly */
|
||
100% { background-position: 0 100px, 40px 40px, 40px 40px; }
|
||
}
|
||
|
||
/* --------------------------------------------------------------------------
|
||
5. BONUS: CAUTION TAPE REACTION BAR
|
||
-------------------------------------------------------------------------- */
|
||
.articletag-418challenge .crayons-article-actions {
|
||
background: repeating-linear-gradient(
|
||
45deg,
|
||
#ffff00,
|
||
#ffff00 10px,
|
||
#000000 10px,
|
||
#000000 20px
|
||
) !important;
|
||
border: 4px solid #ff0000 !important;
|
||
border-radius: 0 !important;
|
||
}
|
||
|
||
.articletag-418challenge .crayons-article-actions button {
|
||
background:black;
|
||
color: #ff00ff !important;
|
||
}
|
||
|
||
.articletag-418challenge .crayons-article-actions .crayons-reaction__count {
|
||
color: #ff00ff !important;
|
||
font-weight: bold !important;
|
||
}
|
||
|
||
.articletag-418challenge .crayons-icon:not(.crayons-icon--default) * {
|
||
fill: #ffff00 !important;
|
||
}
|
||
|
||
.articletag-418challenge .c-embed .crayons-btn--primary {
|
||
color: #00ff00 !important;
|
||
}
|
||
|
||
.articletag-418challenge .popover-billboard {
|
||
margin-bottom: 0 !important;
|
||
}
|
||
|
||
.crayons-story__contentpreview .ltag__link--embedded {
|
||
margin-top: 0 !important;
|
||
}
|
||
|
||
.subscription-icon {
|
||
max-height: 16px !important;
|
||
display: inline-block !important;
|
||
}
|
||
|
||
@media screen and (min-width: 950px) {
|
||
.event-show-layout {
|
||
margin-top: 24px !important;
|
||
}
|
||
}
|
||
|
||
@media screen and (max-width: 949px) {
|
||
.event-show-layout {
|
||
padding: 11px !important;
|
||
}
|
||
}
|
||
|
||
|
||
</style>
|
||
</head>
|
||
<body
|
||
class="sans-serif-article-body default-header"
|
||
data-user-status="logged-out"
|
||
data-is-root-subforem="false"
|
||
data-subforem-id="1"
|
||
data-side-nav-visible="true"
|
||
data-community-name="DEV Community"
|
||
data-subscription-icon="https://assets.dev.to/assets/subscription-icon-805dfa7ac7dd660f07ed8d654877270825b07a92a03841aa99a1093bd00431b2.png"
|
||
data-community-leader-icon="https://assets.dev.to/assets/community-leader-icon-b72c9e74eff54916e5c46c962f47ba40c9f611a71f8b157511f9613f69c0001b.svg"
|
||
data-locale="en"
|
||
data-honeybadger-key="hbp_nqu4Y66HuEKlD6YRGssZuRQnPOjDm50J8Zkr"
|
||
data-deployed-at="2026-09-11T18:06:01Z"
|
||
data-latest-commit-id="ac54b3b29af145ea947eb7f5d6acba8a86c9d109"
|
||
data-ga-tracking="UA-71991109-1"
|
||
data-cookie-banner-user-context="logged_out_only"
|
||
data-cookie-banner-platform-context="off"
|
||
data-algolia-id="PRSOBFP46H"
|
||
data-algolia-search-key="9aa7d31610cba78851c9b1f63776a9dd"
|
||
data-algolia-display="true"
|
||
data-dynamic-url-component="bmar11"
|
||
data-ga4-tracking-id="G-TYEM8Y3JN3"
|
||
data-global-feature-flags-enabled="data_update_scripts onboarding_newsletter_content schedule_articles moderator_role display_ad_tags replace_unicorn_with_jump_to_comments article_id_adjusted_weight cloudflare_preferred_for_hosted_images multiple_reactions subscriber_icon ab_experiment_feed_strategy use_stories_endpoint digest_subject_testing digest_results_count_testing consistent_rendering discover_and_following_tabs hero_billboard onboarding_drip_emails algolia_frontend gemini_onboarding optimize_article_tag_query dev_core_user_sync personalized_email_digests community_favorites"
|
||
data-style-fingerprint="c75bfc6a82">
|
||
|
||
<script>
|
||
if (navigator.userAgent === 'ForemWebView/1' || window.frameElement) {
|
||
document.body.classList.add("hidden-shell");
|
||
}
|
||
if (new Date() > new Date("2026-02-04T09:00:00-05:00")) {
|
||
document.body.dataset.sideNavVisible = "false";
|
||
}
|
||
</script>
|
||
|
||
<link rel="stylesheet" href="https://assets.dev.to/assets/minimal-3a438d8d138f95f478f01cbe53363cacf36bc6c47172aeb2e6c9172eebe29aa6.css" media="all" id="secondary-minimal-stylesheet" />
|
||
<link rel="stylesheet" href="https://assets.dev.to/assets/views-ca4b6f6e9ec63c7deefbdd4067e585d752cd4b06c12da1a49c7b9644defa2758.css" media="all" id="secondary-views-stylesheet" />
|
||
<link rel="stylesheet" href="https://assets.dev.to/assets/crayons-537acd1cbd95b8d8d524dab489c8923b7316dd045ead39e7a2dbaa42cdc208b2.css" media="all" id="secondary-crayons-stylesheet" />
|
||
|
||
<div id="body-styles">
|
||
<style>
|
||
:root {
|
||
--accent-brand-lighter-rgb: 86, 105, 285;
|
||
--accent-brand-rgb: 64, 78, 211;
|
||
--accent-brand-darker-rgb: 51, 62, 169;
|
||
}
|
||
</style>
|
||
</div>
|
||
<div id="audiocontent" data-podcast="">
|
||
|
||
</div>
|
||
<div class="navigation-progress" id="navigation-progress"></div>
|
||
|
||
<header id="topbar" class="crayons-header topbar print-hidden">
|
||
<span id="route-change-target" tabindex="-1"></span>
|
||
<a href="#main-content" class="skip-content-link">Skip to content</a>
|
||
<div class="crayons-header__container">
|
||
<span class="inline-block m:hidden ">
|
||
<button class="c-btn c-btn--icon-alone js-hamburger-trigger mx-2">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="aruvo2rpo57c4n0k16na0k9p7jzflq9t" class="crayons-icon"><title id="aruvo2rpo57c4n0k16na0k9p7jzflq9t">Navigation menu</title>
|
||
<path d="M3 4h18v2H3V4zm0 7h18v2H3v-2zm0 7h18v2H3v-2z"></path>
|
||
</svg>
|
||
|
||
</button>
|
||
</span>
|
||
<a href="/" class="site-logo" aria-label="DEV Community Home" >
|
||
<img class="site-logo__img"
|
||
src="https://media2.dev.to/dynamic/image/quality=100/https://dev-to-uploads.s3.amazonaws.com/uploads/logos/resized_logo_UQww2soKuUsjaOGNB38o.png"
|
||
style="aspect-ratio: 10 / 8"
|
||
alt="DEV Community">
|
||
</a>
|
||
|
||
|
||
<div class="crayons-header--search js-search-form" id="header-search">
|
||
<form accept-charset="UTF-8" method="get" action="/search" role="search">
|
||
<div class="crayons-fields crayons-fields--horizontal">
|
||
<div class="crayons-field flex-1 relative">
|
||
<input id="search-input" class="crayons-header--search-input crayons-textfield js-search-input" type="text" id="nav-search" name="q" placeholder="Find related posts..." autocomplete="off" />
|
||
<button type="submit" aria-label="Search" class="c-btn c-btn--icon-alone absolute inset-px right-auto mt-0 py-0">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="aiq9db7ozbcmwh6eoldh5cd3o87b6at5" aria-hidden="true" class="crayons-icon"><title id="aiq9db7ozbcmwh6eoldh5cd3o87b6at5">Search</title>
|
||
<path d="M18.031 16.617l4.283 4.282-1.415 1.415-4.282-4.283A8.96 8.96 0 0111 20c-4.968 0-9-4.032-9-9s4.032-9 9-9 9 4.032 9 9a8.96 8.96 0 01-1.969 5.617zm-2.006-.742A6.977 6.977 0 0018 11c0-3.868-3.133-7-7-7-3.868 0-7 3.132-7 7 0 3.867 3.132 7 7 7a6.977 6.977 0 004.875-1.975l.15-.15z"></path>
|
||
</svg>
|
||
|
||
</button>
|
||
|
||
<a class="crayons-header--search-brand-indicator" href="https://www.algolia.com/developers/?utm_source=devto&utm_medium=referral" target="_blank" rel="noopener noreferrer">
|
||
Powered by Algolia
|
||
<svg xmlns="http://www.w3.org/2000/svg" id="Layer_1" width="24" height="24" viewBox="0 0 500 500.34" role="img" aria-labelledby="a6bdkpr5zi8vs8qvb57rqf806kx8crir" aria-hidden="true" class="crayons-icon"><title id="a6bdkpr5zi8vs8qvb57rqf806kx8crir">Search</title>
|
||
<defs></defs><path class="cls-1" d="M250,0C113.38,0,2,110.16,.03,246.32c-2,138.29,110.19,252.87,248.49,253.67,42.71,.25,83.85-10.2,120.38-30.05,3.56-1.93,4.11-6.83,1.08-9.52l-23.39-20.74c-4.75-4.22-11.52-5.41-17.37-2.92-25.5,10.85-53.21,16.39-81.76,16.04-111.75-1.37-202.04-94.35-200.26-206.1,1.76-110.33,92.06-199.55,202.8-199.55h202.83V407.68l-115.08-102.25c-3.72-3.31-9.43-2.66-12.43,1.31-18.47,24.46-48.56,39.67-81.98,37.36-46.36-3.2-83.92-40.52-87.4-86.86-4.15-55.28,39.65-101.58,94.07-101.58,49.21,0,89.74,37.88,93.97,86.01,.38,4.28,2.31,8.28,5.53,11.13l29.97,26.57c3.4,3.01,8.8,1.17,9.63-3.3,2.16-11.55,2.92-23.6,2.07-35.95-4.83-70.39-61.84-127.01-132.26-131.35-80.73-4.98-148.23,58.18-150.37,137.35-2.09,77.15,61.12,143.66,138.28,145.36,32.21,.71,62.07-9.42,86.2-26.97l150.36,133.29c6.45,5.71,16.62,1.14,16.62-7.48V9.49C500,4.25,495.75,0,490.51,0H250Z"></path>
|
||
</svg>
|
||
|
||
</a>
|
||
</div>
|
||
</div>
|
||
</form>
|
||
</div>
|
||
|
||
<div class="flex items-center h-100 ml-auto">
|
||
<div class="flex" id="authentication-top-nav-actions">
|
||
<span class="hidden m:block">
|
||
<a href="https://dev.to/enter?signup_subforem=1" class="c-link c-link--block mr-2 whitespace-nowrap ml-auto" data-no-instant>
|
||
Log in
|
||
</a>
|
||
</span>
|
||
|
||
<a href="https://dev.to/enter?signup_subforem=1&state=new-user" data-tracking-id="ca_top_nav" data-tracking-source="top_navbar" class="c-cta c-cta--branded whitespace-nowrap mr-2" data-no-instant>
|
||
Create account
|
||
</a>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</header>
|
||
|
||
<div class="hamburger">
|
||
<div class="hamburger__content">
|
||
<header class="hamburger__content__header">
|
||
<h2 class="fs-l fw-bold flex-1 break-word lh-tight">DEV Community</h2>
|
||
|
||
<button class="c-btn c-btn--icon-alone js-hamburger-trigger shrink-0" aria-label="Close">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="arcg0njp3t7axq6j0zhpp66ph1alglc3" aria-hidden="true" class="crayons-icon c-btn__icon"><title id="arcg0njp3t7axq6j0zhpp66ph1alglc3">Close</title><path d="M12 10.586l4.95-4.95 1.414 1.414-4.95 4.95 4.95 4.95-1.414 1.414-4.95-4.95-4.95 4.95-1.414-1.414 4.95-4.95-4.95-4.95L7.05 5.636l4.95 4.95z"></path></svg>
|
||
|
||
</button>
|
||
</header>
|
||
|
||
<div class="p-2 js-navigation-links-container" id="authentication-hamburger-actions">
|
||
</div>
|
||
</div>
|
||
<div class="hamburger__overlay js-hamburger-trigger"></div>
|
||
</div>
|
||
|
||
|
||
<div id="active-broadcast" class="broadcast-wrapper"></div>
|
||
<div id="page-content" class="wrapper stories stories-show articletag-incident articletag-retro articletag-postmortem articleuser-119473 articleorg-1" data-current-page="stories-show">
|
||
<div id="page-content-inner" data-internal-nav="false" data-viewable-id="759150" data-viewable-type="Article">
|
||
<div id="page-route-change" class="screen-reader-only" aria-live="polite" aria-atomic="true"></div>
|
||
|
||
|
||
<style>
|
||
.html-variant-wrapper { display: none}
|
||
</style>
|
||
|
||
|
||
|
||
<script src="https://unpkg.com/@webcomponents/webcomponentsjs@2.2.10/webcomponents-loader.js"
|
||
integrity="sha384-3HK5hxQbkFqOIxMbpROlRmRtYl2LBZ52t+tqcjzsmr9NJuOWQxl8RgQSyFvq2lhy"
|
||
crossorigin="anonymous" defer></script>
|
||
|
||
<script src="https://assets.dev.to/assets/webShare-0686f0b9ac40589694ef6ae6a6202c44119bc781c254f6cf6d52d8a008461156.js" defer="defer"></script>
|
||
<script src="https://assets.dev.to/assets/articlePage-7386c745667022e7147503d288454cfce459f2019ee90ce454e641a116844837.js" defer="defer"></script>
|
||
<script src="https://assets.dev.to/assets/commentDropdowns-7a28d130e5b78d38b30a9495a964003a66bd64fa455fc70b766d69cf06b9ba24.js" defer="defer"></script>
|
||
|
||
|
||
<script type="application/ld+json">
|
||
{"@context":"http://schema.org","@type":"Article","mainEntityOfPage":{"@type":"WebPage","@id":"https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj"},"url":"https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj","image":["https://media2.dev.to/dynamic/image/width=1080,height=1080,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2Fivyfkw7k643ivg4ltdd5.png","https://media2.dev.to/dynamic/image/width=1280,height=720,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2Fivyfkw7k643ivg4ltdd5.png","https://media2.dev.to/dynamic/image/width=1600,height=900,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2Fivyfkw7k643ivg4ltdd5.png"],"publisher":{"@context":"http://schema.org","@type":"Organization","name":"DEV Community","logo":{"@context":"http://schema.org","@type":"ImageObject","url":"https://media2.dev.to/dynamic/image/width=192,height=,fit=scale-down,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2F8j7kvp660rqzt99zui8e.png","width":"192","height":"192"}},"headline":"Incident Retro: Failing Comment Creation + Erroneous Push Notifications","author":{"@context":"http://schema.org","@type":"Person","url":"https://dev.to/molly","name":"Molly Struve (she/her)"},"datePublished":"2021-07-14T20:53:41Z","dateModified":"2021-07-14T20:53:41Z","mainEntity":{"@type":"DiscussionForumPosting","@id":"#article-discussion-759150","headline":"Incident Retro: Failing Comment Creation + Erroneous Push Notifications","text":"\u003cp\u003e\u003cem\u003eThis incident retro was tougher than most to share because, despite the seriousness of the issue, it affected only a very small percentage of our user base. However, we learned some incredibly valuable lessons and I think it's only right that I give others the chance to learn from our mistake as well.\u003c/em\u003e\u003c/p\u003e\n\n\u003ch2\u003e\n \u003ca name=\"context\" href=\"#context\"\u003e\n \u003c/a\u003e\n Context\n\u003c/h2\u003e\n\n\u003cp\u003eOn Thursday July 8th, we merged a \u003ca href=\"https://github.com/forem/forem/pull/14121\" target=\"_blank\" rel=\"noopener noreferrer\"\u003every large PR\u003c/a\u003e that updated our code to start using our new User Settings instead of the deprecated fields on users. The goal of moving these fields to user settings is to lighten up the user model and make these types of settings more configurable on a per-Forem basis. \u003c/p\u003e\n\n\u003ch2\u003e\n \u003ca name=\"problem\" href=\"#problem\"\u003e\n \u003c/a\u003e\n Problem\n\u003c/h2\u003e\n\n\u003cp\u003eOn July 9th around midday, it was brought to our attention \u003ca href=\"https://github.com/forem/forem/issues/14183\" target=\"_blank\" rel=\"noopener noreferrer\"\u003ethrough a bug report\u003c/a\u003e that the comment creation experience on the frontend seemed to be broken. At this point folks on the team started looking into the problem. Using \u003ca href=\"https://www.honeycomb.io/\" target=\"_blank\" rel=\"noopener noreferrer\"\u003eHoneycomb\u003c/a\u003e, we were able to confirm that the problem started when the User Settings PR was deployed which made it the prime suspect. \u003c/p\u003e\n\n\u003cp\u003e\u003ca href=\"https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fforem.team%2Fremoteimages%2Fuploads%2Farticles%2Facuvsc0l8y4135nois0f.png\" class=\"article-body-image-wrapper\"\u003e\u003cimg src=\"https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fforem.team%2Fremoteimages%2Fuploads%2Farticles%2Facuvsc0l8y4135nois0f.png\" alt=\"Picture of a graph where the line is steady under 100ms and jumps to thousands\" loading=\"lazy\" width=\"800\" height=\"400\"\u003e\u003c/a\u003e\u003c/p\u003e\n\n\u003cp\u003eI immediately dove into the PR and quickly found \u003ca href=\"https://github.com/forem/forem/pull/14121/files#diff-56d67da7087534bc7d4cffd953e00dc2e3eb41047f1e82f3124cd6d69c509873R38\" target=\"_blank\" rel=\"noopener noreferrer\"\u003ethe line of code that had been broken\u003c/a\u003e.\u003cbr\u003e\n\u003ca href=\"https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fforem.team%2Fremoteimages%2Fuploads%2Farticles%2Fjh78q3nsi71vrdyekkk0.png\" class=\"article-body-image-wrapper\"\u003e\u003cimg src=\"https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fforem.team%2Fremoteimages%2Fuploads%2Farticles%2Fjh78q3nsi71vrdyekkk0.png\" alt=\"Github PR dif\" loading=\"lazy\" width=\"800\" height=\"400\"\u003e\u003c/a\u003e\u003c/p\u003e\n\n\u003cp\u003eWe had incorrectly removed the \u003ccode\u003euser_ids\u003c/code\u003e scope from the filtering used to collect user IDs for sending mobile push notifications. This was causing us to attempt to grab notification settings and IDs for all users with mobile notifications turned on which was often timing out. These timeouts would occur after a comment was created so the comment in most cases still existed but we never returned a success response to the frontend. This caused the frontend to freeze creating a bad experience for the user. \u003c/p\u003e\n\u003ch2\u003e\n \u003ca name=\"fix\" href=\"#fix\"\u003e\n \u003c/a\u003e\n Fix\n\u003c/h2\u003e\n\n\u003cp\u003eIn order to remedy this situation, we quickly added back the scope and deployed. Immediately the errors and timeouts went away and things returned to normal for comment creation. \u003c/p\u003e\n\n\u003cp\u003e\u003ca href=\"https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fforem.team%2Fremoteimages%2Fuploads%2Farticles%2Fvpicn144aagfpsrbnnhs.png\" class=\"article-body-image-wrapper\"\u003e\u003cimg src=\"https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fforem.team%2Fremoteimages%2Fuploads%2Farticles%2Fvpicn144aagfpsrbnnhs.png\" alt=\"Graph of increased duration returning to normal\" loading=\"lazy\" width=\"800\" height=\"400\"\u003e\u003c/a\u003e\u003c/p\u003e\n\n\u003cp\u003eAt this point, we concluded that possibly some extra notifications had been sent but that the majority of these requests had timed out so we assumed we were in a good state now. I looked for some sort of push notification model in our app where we might have stored unsent notifications but upon not finding one figured we were clear. \u003c/p\u003e\n\u003ch2\u003e\n \u003ca name=\"more-problems\" href=\"#more-problems\"\u003e\n \u003c/a\u003e\n More Problems\n\u003c/h2\u003e\n\n\u003cp\u003e\u003cstrong\u003eJuly 9th ~12pm EDT\u003c/strong\u003e: Unfortunately, the above assumption was incorrect. Just before midnight eastern, a team member reported she was getting random notifications on her phone. At this point, our mobile engineer jumped in to investigate and found that we had a ton of push notifications enqueued in Redis.\u003c/p\u003e\n\n\u003cp\u003e\u003cstrong\u003eJuly 9th ~2am EDT\u003c/strong\u003e: Mobile engineer posts about the issue in our #emergency Slack channel.\u003c/p\u003e\n\n\u003cp\u003e\u003cstrong\u003eJuly 9th ~7am EDT\u003c/strong\u003e: On-call engineer wakes up and sees the #emergency Slack message and jumps in to help investigate and try to resolve the issue. Shortly after, I also jumped in. It was at this point that I learned all about how our push notification system worked. We use \u003ca href=\"https://github.com/rpush/rpush\" target=\"_blank\" rel=\"noopener noreferrer\"\u003eRPush\u003c/a\u003e for communicating with various push-notification services. RPush stores the data about the push notifications in Redis for quick insertion and removal times. We quickly assessed that we had a large number of push notifications enqueued in Redis waiting to be sent, despite the solution deployed the day before. \u003c/p\u003e\n\u003ch2\u003e\n \u003ca name=\"fix-2\" href=\"#fix-2\"\u003e\n \u003c/a\u003e\n Fix 2\n\u003c/h2\u003e\n\n\u003cp\u003eUpon realizing we had all of these records in Redis, we took the following steps to resolve the issue:\u003c/p\u003e\n\n\u003cul\u003e\n\u003cli\u003eRemoval of the iOS cert from Heroku to prevent sending any notifications. Without this cert we could ensure we would not send anymore bad notifications and that would give us time to fix the data in Redis.\u003c/li\u003e\n\u003cli\u003eTo be safe, we removed all of the \u003ccode\u003ePushNotifications::DeliveryWorker\u003c/code\u003e jobs from Sidekiq so as not to be pinging Redis for the keys causing our key collection to slow down. \n\n\u003cul\u003e\n\u003cli\u003eAt the same time, we looped through all of the keys in Redis and collected all of the rpush ones. These included all delivered and undelivered keys. There were 1.2 million rpush keys so this process took about 30 mins. We used the below script:\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"highlight js-code-highlight\"\u003e\n\u003cpre class=\"highlight ruby\"\u003e\u003ccode\u003e\u003cspan class=\"n\"\u003eredis\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"no\"\u003eRails\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003ecache\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003eredis\u003c/span\u003e\n\u003cspan class=\"n\"\u003estart\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"mi\"\u003e0\u003c/span\u003e\n\u003cspan class=\"n\"\u003etotal\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"mi\"\u003e0\u003c/span\u003e\n\u003cspan class=\"n\"\u003ekey_collection\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"p\"\u003e[]\u003c/span\u003e\n\u003cspan class=\"n\"\u003eindex\u003c/span\u003e\u003cspan class=\"p\"\u003e,\u003c/span\u003e \u003cspan class=\"n\"\u003ekeys\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"n\"\u003eredis\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003escan\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"n\"\u003estart\u003c/span\u003e\u003cspan class=\"p\"\u003e);\u003c/span\u003e\n\u003cspan class=\"k\"\u003ewhile\u003c/span\u003e \u003cspan class=\"n\"\u003eindex\u003c/span\u003e \u003cspan class=\"o\"\u003e!=\u003c/span\u003e \u003cspan class=\"s2\"\u003e\"0\"\u003c/span\u003e\n \u003cspan class=\"n\"\u003estart\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"n\"\u003eindex\u003c/span\u003e\n \u003cspan class=\"n\"\u003etotal\u003c/span\u003e \u003cspan class=\"o\"\u003e+=\u003c/span\u003e \u003cspan class=\"n\"\u003ekeys\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003ecount\u003c/span\u003e\n \u003cspan class=\"n\"\u003ekeys\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003eeach\u003c/span\u003e \u003cspan class=\"k\"\u003edo\u003c/span\u003e \u003cspan class=\"o\"\u003e|\u003c/span\u003e\u003cspan class=\"n\"\u003ekey\u003c/span\u003e\u003cspan class=\"o\"\u003e|\u003c/span\u003e\n \u003cspan class=\"n\"\u003ekey_collection\u003c/span\u003e \u003cspan class=\"o\"\u003e\u0026lt;\u0026lt;\u003c/span\u003e \u003cspan class=\"n\"\u003ekey\u003c/span\u003e \u003cspan class=\"k\"\u003eif\u003c/span\u003e \u003cspan class=\"n\"\u003ekey\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003einclude?\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"s1\"\u003e'rpush:notifications'\u003c/span\u003e\u003cspan class=\"p\"\u003e)\u003c/span\u003e\n \u003cspan class=\"k\"\u003eend\u003c/span\u003e\n \u003cspan class=\"n\"\u003eindex\u003c/span\u003e\u003cspan class=\"p\"\u003e,\u003c/span\u003e \u003cspan class=\"n\"\u003ekeys\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"n\"\u003eredis\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003escan\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"n\"\u003estart\u003c/span\u003e\u003cspan class=\"p\"\u003e)\u003c/span\u003e\n\u003cspan class=\"k\"\u003eend\u003c/span\u003e\n\u003c/code\u003e\u003c/pre\u003e\n\u003cdiv class=\"highlight__panel js-actions-panel\"\u003e\n\u003cdiv class=\"highlight__panel-action js-fullscreen-code-action\"\u003e\n \u003csvg xmlns=\"http://www.w3.org/2000/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"\u003e\u003ctitle\u003eEnter fullscreen mode\u003c/title\u003e\n \u003cpath d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\u003e\u003c/path\u003e\n\u003c/svg\u003e\n\n \u003csvg xmlns=\"http://www.w3.org/2000/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"\u003e\u003ctitle\u003eExit fullscreen mode\u003c/title\u003e\n \u003cpath d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\u003e\u003c/path\u003e\n\u003c/svg\u003e\n\n\u003c/div\u003e\n\u003c/div\u003e\n\u003c/div\u003e\n\n\n\u003cul\u003e\n\u003cli\u003eOnce we had the keys collected we double checked that they were the keys we wanted and then we deleted them ALL in batches of 10 using this code:\n\u003c/li\u003e\n\u003c/ul\u003e\n\u003cdiv class=\"highlight js-code-highlight\"\u003e\n\u003cpre class=\"highlight ruby\"\u003e\u003ccode\u003e\u003cspan class=\"n\"\u003ekey_collection\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003eeach_slice\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"mi\"\u003e10\u003c/span\u003e\u003cspan class=\"p\"\u003e){\u003c/span\u003e\u003cspan class=\"o\"\u003e|\u003c/span\u003e\u003cspan class=\"n\"\u003es\u003c/span\u003e\u003cspan class=\"o\"\u003e|\u003c/span\u003e \u003cspan class=\"nb\"\u003eputs\u003c/span\u003e \u003cspan class=\"n\"\u003eredis\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003edel\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"n\"\u003es\u003c/span\u003e\u003cspan class=\"p\"\u003e)}\u003c/span\u003e\n\u003c/code\u003e\u003c/pre\u003e\n\u003cdiv class=\"highlight__panel js-actions-panel\"\u003e\n\u003cdiv class=\"highlight__panel-action js-fullscreen-code-action\"\u003e\n \u003csvg xmlns=\"http://www.w3.org/2000/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"\u003e\u003ctitle\u003eEnter fullscreen mode\u003c/title\u003e\n \u003cpath d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\u003e\u003c/path\u003e\n\u003c/svg\u003e\n\n \u003csvg xmlns=\"http://www.w3.org/2000/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"\u003e\u003ctitle\u003eExit fullscreen mode\u003c/title\u003e\n \u003cpath d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\u003e\u003c/path\u003e\n\u003c/svg\u003e\n\n\u003c/div\u003e\n\u003c/div\u003e\n\u003c/div\u003e\n\n\n\u003cul\u003e\n\u003cli\u003eOnce the keys were gone, we double checked the count and then added back the iOS cert to Heroku. \u003c/li\u003e\n\u003cli\u003eAfter Heroku restarted, we ran a test to confirm that the notifications were back to sending and that we were recording delivered notifications correctly again in Redis. \u003c/li\u003e\n\u003c/ul\u003e\n\u003ch2\u003e\n \u003ca name=\"impact\" href=\"#impact\"\u003e\n \u003c/a\u003e\n Impact\n\u003c/h2\u003e\n\u003ch3\u003e\n \u003ca name=\"comment-creation\" href=\"#comment-creation\"\u003e\n \u003c/a\u003e\n Comment Creation\n\u003c/h3\u003e\n\n\u003cp\u003eThe comment creation flow on the frontend was broken for 30 hours after the UserSettings PR was deployed on July 8th. However, comment creation levels remained steady.\u003cbr\u003e\n\u003c/p\u003e\n\n\u003cdiv class=\"highlight js-code-highlight\"\u003e\n\u003cpre class=\"highlight plaintext\"\u003e\u003ccode\u003eirb(main):001:0\u0026gt; Comment.where(created_at: 36.hours.ago..Time.now).count\n=\u0026gt; 762\nirb(main):002:0\u0026gt; Comment.where(created_at: 72.hours.ago..36.hours.ago).count\n=\u0026gt; 857\nirb(main):003:0\u0026gt; Comment.where(created_at: 108.hours.ago..72.hours.ago).count\n=\u0026gt; 852\nirb(main):004:0\u0026gt; Comment.where(created_at: 144.hours.ago..108.hours.ago).count\n=\u0026gt; 756\nirb(main):005:0\u0026gt; Comment.where(created_at: 180.hours.ago..144.hours.ago).count\n=\u0026gt; 587\n\u003c/code\u003e\u003c/pre\u003e\n\u003cdiv class=\"highlight__panel js-actions-panel\"\u003e\n\u003cdiv class=\"highlight__panel-action js-fullscreen-code-action\"\u003e\n \u003csvg xmlns=\"http://www.w3.org/2000/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"\u003e\u003ctitle\u003eEnter fullscreen mode\u003c/title\u003e\n \u003cpath d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\u003e\u003c/path\u003e\n\u003c/svg\u003e\n\n \u003csvg xmlns=\"http://www.w3.org/2000/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"\u003e\u003ctitle\u003eExit fullscreen mode\u003c/title\u003e\n \u003cpath d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\u003e\u003c/path\u003e\n\u003c/svg\u003e\n\n\u003c/div\u003e\n\u003c/div\u003e\n\u003c/div\u003e\n\n\n\n\u003cp\u003eIn total, there were 934 comments that were affected by the broken frontend interface. \u003c/p\u003e\n\n\u003cp\u003e\u003ca href=\"https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2F3csl8linhu5fzipww4gq.png\" class=\"article-body-image-wrapper\"\u003e\u003cimg src=\"https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2F3csl8linhu5fzipww4gq.png\" alt=\"Honeycomb.io graph showing a total count of 934 comments over a 30 hour period\" loading=\"lazy\" width=\"800\" height=\"563\"\u003e\u003c/a\u003e\u003c/p\u003e\n\n\u003ch3\u003e\n \u003ca name=\"push-notifications\" href=\"#push-notifications\"\u003e\n \u003c/a\u003e\n Push Notifications\n\u003c/h3\u003e\n\n\u003cp\u003eGiven DEV is early on in its mobile journey, only 0.2% of our users have registered devices that are able to receive push notifications. This means that only 0.2% of our users were affected by this incident. We consider ourselves lucky that we were able to expose some of these issues and have this incident remain very contained. \u003c/p\u003e\n\n\u003cp\u003eSince the numbers of users affected was small, we decide to proactively reach out to all of those folks via email to apologize and explain why they \u003cem\u003emight\u003c/em\u003e have received erroneous notifications. \u003c/p\u003e\n\n\u003ch2\u003e\n \u003ca name=\"learnings\" href=\"#learnings\"\u003e\n \u003c/a\u003e\n Learnings\n\u003c/h2\u003e\n\n\u003ch3\u003e\n \u003ca name=\"large-prs\" href=\"#large-prs\"\u003e\n \u003c/a\u003e\n Large PRs\n\u003c/h3\u003e\n\n\u003cp\u003eLarge PRs naturally come with more risk. The bug that kicked off this stream of events was a small change in a very large PR that got missed by multiple folks. For starters, the diff for the line was not particularly helpful in recognizing the change that caused the issue. In addition, there were so many lines that it was easy to overlook. Breaking down PRs is one way to prevent this. \u003c/p\u003e\n\n\u003cp\u003eHowever, there are times when large, wide spread changes need to be made and in those cases you have to rely on your test suite. \u003c/p\u003e\n\n\u003ch3\u003e\n \u003ca name=\"missing-tests\" href=\"#missing-tests\"\u003e\n \u003c/a\u003e\n Missing tests\n\u003c/h3\u003e\n\n\u003cp\u003eThe feature that broke was completely untested. Had we been testing that feature properly I think there is a good chance the bug could have been caught. Immediately after pushing the hotfix \u003ca href=\"https://github.com/forem/forem/pull/14187\" target=\"_blank\" rel=\"noopener noreferrer\"\u003eI added a test\u003c/a\u003e to ensure this never happens again. \u003c/p\u003e\n\n\u003ch3\u003e\n \u003ca name=\"technical-feature-education\" href=\"#technical-feature-education\"\u003e\n \u003c/a\u003e\n Technical feature education\n\u003c/h3\u003e\n\n\u003cp\u003eAfter fixing the bug, none of us working on the problem were well educated about how our Push Notification systems worked. Being late on a Friday, we skimmed the code and concluded we would be good to go. This ended up being the wrong assumption as we later learned Saturday that push notifications, unlike app ones, are stored and enqueued via Redis. I think it's more important than ever that we are somehow sharing and educating each other about some of these larger features as we roll them out. \u003c/p\u003e\n\n\u003cp\u003eThankfully, we do \u003ca href=\"https://docs.forem.com/backend/push-notifications/\" target=\"_blank\" rel=\"noopener noreferrer\"\u003ehave some great Push Notification documentation\u003c/a\u003e but it was never sought out during the incident. Is there a way we can make docs like this more visible? During the incident we were all heads down in the code, should we maybe have some sort of URL link in the code to the docs?\u003c/p\u003e\n\n\u003ch3\u003e\n \u003ca name=\"being-more-intentional\" href=\"#being-more-intentional\"\u003e\n \u003c/a\u003e\n Being more intentional\n\u003c/h3\u003e\n\n\u003cp\u003eI could have very easily reached out to our mobile team on Friday evening to double check that we were in a good state for Push Notifications. Once again, the whole Friday evening and wanting to be done caused me to accept my assumptions rather than challenge and check them. \u003c/p\u003e\n\n\u003cp\u003eOne easy way to prevent something like this is to use a checklist. A checklist (heavily used in aviation because it has been shown to be key in preventing incidents,) is an easy way to make sure you never miss something and are deliberate and intentional with your decisions. We have great checklists for handling Heroku incidents in our internal Gitbook. However, we don't have a general incident on-call checklist which we plan to add. \u003c/p\u003e\n\n\u003ch2\u003e\n \u003ca name=\"incident-response\" href=\"#incident-response\"\u003e\n \u003c/a\u003e\n Incident Response\n\u003c/h2\u003e\n\n\u003cp\u003eBefore I dive into specifics, I want to point out that this is the first larger incident we have had in a while. Incidents being rare is GREAT! But, it also means that our incident response was a bit rusty and our documented incident processes were a bit out of date to handle this situation. Going forward we will be taking a closer look at all of these things to ensure we are keeping them relevant and useful based on the current state of our application and team. \u003c/p\u003e\n\n\u003ch3\u003e\n \u003ca name=\"broken-comment-creation-reporting-flow\" href=\"#broken-comment-creation-reporting-flow\"\u003e\n \u003c/a\u003e\n Broken Comment Creation Reporting Flow\n\u003c/h3\u003e\n\n\u003cp\u003eThe initial comment creation problem was reported by a DEV user via a GitHub issue and seen about 11 hours later by our internal engineering team. GitHub Issues are naturally not a very urgent form of reporting for us since they are handled during working hours. One way we could improve this flow would be to further communicate that urgent support issues need to be emailed to \u003ccode\u003eyo@forem.com\u003c/code\u003e. Our customer success team is always monitoring these channels diligently and could more quickly triage and escalate an issue to the team. \u003c/p\u003e\n\n\u003cp\u003eManual reporting aside, ideally, we should have caught this programmatically with our monitoring by alerting on the increased HTTP request errors that were detected.\u003cbr\u003e\n\u003ca href=\"https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fforem.team%2Fremoteimages%2Fuploads%2Farticles%2Fianj96377wcnqd10q0l7.png\" class=\"article-body-image-wrapper\"\u003e\u003cimg src=\"https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fforem.team%2Fremoteimages%2Fuploads%2Farticles%2Fianj96377wcnqd10q0l7.png\" alt=\"Datadog graphs showing elevated error rates\" loading=\"lazy\" width=\"800\" height=\"400\"\u003e\u003c/a\u003e\u003c/p\u003e\n\n\u003cp\u003eGoing forward, we plan to set up a monitor to alert us for these changes so that we are the first to find out, rather than relying on users reporting the issue to us. With the proper monitor in place, we could have caught this within a couple of hours instead of over a day. \u003c/p\u003e\n\n\u003ch3\u003e\n \u003ca name=\"broken-push-notifications-reporting-flow\" href=\"#broken-push-notifications-reporting-flow\"\u003e\n \u003c/a\u003e\n Broken Push Notifications Reporting Flow\n\u003c/h3\u003e\n\n\u003cp\u003eWe responded to the broken comment creation issue immediately when it was surfaced, however, the followup push notification issues were not. Our engineer correctly pinged the #emergency channel when he realized that we were having problems with notifications but did not escalate the issue and wake up the on-call dev via PagerDuty. \u003c/p\u003e\n\n\u003cp\u003eOne way we could streamline this process would be to see if we could set up our #emergency Slack channel to automatically ping PageDuty when a message is posted. Another option is address some of our incident response rustiness by better educating the team on our desired incident response flows. This probably needs to be a reoccurring training that happens periodically so as to prevent processes and knowledge from getting stale as our systems evolve. \u003c/p\u003e\n\n\u003ch3\u003e\n \u003ca name=\"technical-remediation\" href=\"#technical-remediation\"\u003e\n \u003c/a\u003e\n Technical Remediation\n\u003c/h3\u003e\n\n\u003cp\u003eOne delay we encountered when trying to fix this issue was that the Rpush keys were stored in the same Redis instance as our Rails cache. Given our Rails cache is massive and contains 4+ million keys, looping through all of them to find only the Rpush keys was tedious and took a while. That cache is also very active which caused us to hit some timeouts when we were trying to perform heavier operations on it. \u003c/p\u003e\n\n\u003cp\u003eWe could have mitigated this friction by having either a separate Redis instance for Rpush OR by having those keys in a separate database away from our Rails cache keys. Going forward, as we grow our mobile platforms, I think we should make one of the above changes to ensure that we have easy access to these keys and records. It will also ensure that as we grow and scale push notifications we don't have to worry about impacting other systems like our core Rails app. \u003c/p\u003e\n\n\u003ch2\u003e\n \u003ca name=\"thanks-everyone\" href=\"#thanks-everyone\"\u003e\n \u003c/a\u003e\n Thanks Everyone 🤗\n\u003c/h2\u003e\n\n\u003cp\u003eA lot of people gave up their personal time to come together on Friday and Saturday to mitigate and fix these issues. I truly appreciate the amazing team we have at Forem and their dedication to this software. We learned a lot about our systems in the process which ensures that this incident will not go to waste. I hope others can learn from this as well.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Molly Struve (she/her)","url":"https://dev.to/molly"},"datePublished":"2021-07-14T20:53:41Z","dateModified":"2021-07-14T20:53:41Z","url":"https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/CommentAction","userInteractionCount":9},{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":52}],"comment":[{"@type":"Comment","@id":"#comment-745008","text":"\u003cp\u003eI made this joke on Slack and I'll make it again here...\u003c/p\u003e\n\n\u003cp\u003e\u003ca href=\"https://res.cloudinary.com/practicaldev/image/fetch/s--J6GJ-e0y--/c_limit%2Cf_auto%2Cfl_progressive%2Cq_auto%2Cw_880/https://dev-to-uploads.s3.amazonaws.com/uploads/articles/22ljq94e39pqjzjym34f.png\" class=\"article-body-image-wrapper\"\u003e\u003cimg src=\"https://res.cloudinary.com/practicaldev/image/fetch/s--J6GJ-e0y--/c_limit%2Cf_auto%2Cfl_progressive%2Cq_auto%2Cw_880/https://dev-to-uploads.s3.amazonaws.com/uploads/articles/22ljq94e39pqjzjym34f.png\" alt=\"https://dev-to-uploads.s3.amazonaws.com/uploads/articles/22ljq94e39pqjzjym34f.png\" loading=\"lazy\"\u003e\u003c/a\u003e\u003c/p\u003e\n\n\u003cp\u003e...is this a \u003ca href=\"https://www.denofgeek.com/tv/loki-what-is-a-nexus-event/\" rel=\"nofollow\"\u003eNexus event\u003c/a\u003e?\u003c/p\u003e\n\n\u003cp\u003eLoving Loki! \u003c/p\u003e\n\n\u003cp\u003e\u003ca href=\"https://i.giphy.com/media/jxsXCHBt4ZaEGYajmD/source.gif\" class=\"article-body-image-wrapper\"\u003e\u003cimg src=\"https://i.giphy.com/media/jxsXCHBt4ZaEGYajmD/source.gif\" alt=\"Mobius says \u0026quot;You're my favorite\u0026quot;\" loading=\"lazy\"\u003e\u003c/a\u003e\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Michael Tharrington","url":"https://dev.to/michaeltharrington"},"datePublished":"2021-07-14T21:30:32Z","dateModified":"2021-07-14T21:30:32Z","url":"https://dev.to/michaeltharrington/comment/1ga24","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":8}],"comment":[{"@type":"Comment","@id":"#comment-745012","text":"\u003cp\u003eI just started watching this last night!!!! LOL 😂\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Molly Struve (she/her)","url":"https://dev.to/molly"},"datePublished":"2021-07-14T21:41:22Z","dateModified":"2021-07-14T21:41:22Z","url":"https://dev.to/molly/comment/1ga28","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":3}],"parentItem":{"@type":"Comment","@id":"#comment-745008"}}]},{"@type":"Comment","@id":"#comment-746616","text":"\u003cp\u003eLooks like you can do \u003ccode\u003eredis.keys('rpush:notifications:*')\u003c/code\u003e in place of the first script. It should be much quicker, because it doesn't send anything over the wire which doesn't match the pattern. If you want to do it in a paginated way, you could do: \u003ccode\u003er.scan_each(match: 'rpush:notifications:*') { |key| }\u003c/code\u003e which basically does what your script did, except that the \u003ccode\u003ematch\u003c/code\u003e is executed on the server, so it doesn't send nonmatching keys over the wire. You can also pass it a \u003ccode\u003ecount\u003c/code\u003e option, but in my experiments, that was always slower (this will tell redis how many keys you want it to scan).\u003c/p\u003e\n\n\u003cp\u003eLocally I added 1.2 million matching keys and 1.2 million nonmatching keys, and then played around with it for a bit:\u003cbr\u003e\n\u003c/p\u003e\n\n\u003cdiv class=\"highlight js-code-highlight\"\u003e\n\u003cpre class=\"highlight ruby\"\u003e\u003ccode\u003e\u003cspan class=\"k\"\u003edef\u003c/span\u003e \u003cspan class=\"nf\"\u003etime\u003c/span\u003e\n \u003cspan class=\"n\"\u003estart\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"no\"\u003eTime\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003enew\u003c/span\u003e\n \u003cspan class=\"k\"\u003eyield\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003etap\u003c/span\u003e \u003cspan class=\"p\"\u003e{\u003c/span\u003e \u003cspan class=\"vg\"\u003e$stderr\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003eprintf\u003c/span\u003e \u003cspan class=\"s2\"\u003e\"%.2f seconds\u003c/span\u003e\u003cspan class=\"se\"\u003e\\n\u003c/span\u003e\u003cspan class=\"s2\"\u003e\"\u003c/span\u003e\u003cspan class=\"p\"\u003e,\u003c/span\u003e \u003cspan class=\"no\"\u003eTime\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003enow\u003c/span\u003e\u003cspan class=\"o\"\u003e-\u003c/span\u003e\u003cspan class=\"n\"\u003estart\u003c/span\u003e \u003cspan class=\"p\"\u003e}\u003c/span\u003e\n\u003cspan class=\"k\"\u003eend\u003c/span\u003e\n\n\u003cspan class=\"n\"\u003er\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"no\"\u003eRedis\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003enew\u003c/span\u003e\n\n\u003cspan class=\"c1\"\u003e# Seed it with records\u003c/span\u003e\n\u003cspan class=\"mi\"\u003e1_200_000\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003etimes\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003eeach\u003c/span\u003e \u003cspan class=\"k\"\u003edo\u003c/span\u003e \u003cspan class=\"o\"\u003e|\u003c/span\u003e\u003cspan class=\"n\"\u003ei\u003c/span\u003e\u003cspan class=\"o\"\u003e|\u003c/span\u003e\n \u003cspan class=\"n\"\u003er\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003eset\u003c/span\u003e \u003cspan class=\"s2\"\u003e\"rpush:notifications:\u003c/span\u003e\u003cspan class=\"si\"\u003e#{\u003c/span\u003e\u003cspan class=\"n\"\u003ei\u003c/span\u003e\u003cspan class=\"si\"\u003e}\u003c/span\u003e\u003cspan class=\"s2\"\u003e\"\u003c/span\u003e\u003cspan class=\"p\"\u003e,\u003c/span\u003e \u003cspan class=\"n\"\u003ei\u003c/span\u003e\n \u003cspan class=\"n\"\u003er\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003eset\u003c/span\u003e \u003cspan class=\"s2\"\u003e\"other:thing:\u003c/span\u003e\u003cspan class=\"si\"\u003e#{\u003c/span\u003e\u003cspan class=\"n\"\u003ei\u003c/span\u003e\u003cspan class=\"si\"\u003e}\u003c/span\u003e\u003cspan class=\"s2\"\u003e\"\u003c/span\u003e\u003cspan class=\"p\"\u003e,\u003c/span\u003e \u003cspan class=\"n\"\u003ei\u003c/span\u003e\n\u003cspan class=\"k\"\u003eend\u003c/span\u003e\n\n\u003cspan class=\"c1\"\u003e# Check total number of keys\u003c/span\u003e\n\u003cspan class=\"n\"\u003er\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003edbsize\u003c/span\u003e \u003cspan class=\"c1\"\u003e# =\u0026gt; 1200014\u003c/span\u003e\n\n\u003cspan class=\"c1\"\u003e# Check how many match our pattern (this is a lua script that calls the `keys\u003c/span\u003e\n\u003cspan class=\"c1\"\u003e# command on the redis server, and then returns its size) It's much faster\u003c/span\u003e\n\u003cspan class=\"c1\"\u003e# just b/c it doesn't need to ship all the keys over the wire\u003c/span\u003e\n\u003cspan class=\"n\"\u003er\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003eeval\u003c/span\u003e \u003cspan class=\"s2\"\u003e\"return #redis.call('keys', 'rpush:notifications:*')\"\u003c/span\u003e\u003cspan class=\"p\"\u003e,\u003c/span\u003e \u003cspan class=\"p\"\u003e[\u003c/span\u003e\u003cspan class=\"mi\"\u003e0\u003c/span\u003e\u003cspan class=\"p\"\u003e]\u003c/span\u003e \u003cspan class=\"c1\"\u003e# =\u0026gt; 1200000\u003c/span\u003e\n\n\u003cspan class=\"c1\"\u003e# Get the list of keys using https://redis.io/commands/keys\u003c/span\u003e\n\u003cspan class=\"n\"\u003ekeys\u003c/span\u003e \u003cspan class=\"o\"\u003e=\u003c/span\u003e \u003cspan class=\"n\"\u003etime\u003c/span\u003e \u003cspan class=\"p\"\u003e{\u003c/span\u003e \u003cspan class=\"n\"\u003er\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003ekeys\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"s1\"\u003e'rpush:notifications:*'\u003c/span\u003e\u003cspan class=\"p\"\u003e)\u003c/span\u003e \u003cspan class=\"p\"\u003e}\u003c/span\u003e \u003cspan class=\"c1\"\u003e# 7.77 seconds\u003c/span\u003e\n\u003cspan class=\"n\"\u003ekeys\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003esize\u003c/span\u003e \u003cspan class=\"c1\"\u003e# =\u0026gt; 1200000\u003c/span\u003e\n\n\u003cspan class=\"c1\"\u003e# Iterate over each key with successive scans (here we just increment a counter)\u003c/span\u003e\n\u003cspan class=\"n\"\u003etime\u003c/span\u003e \u003cspan class=\"p\"\u003e{\u003c/span\u003e \u003cspan class=\"n\"\u003er\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003escan_each\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"ss\"\u003ematch: \u003c/span\u003e\u003cspan class=\"s1\"\u003e'rpush:notifications:*'\u003c/span\u003e\u003cspan class=\"p\"\u003e).\u003c/span\u003e\u003cspan class=\"nf\"\u003ecount\u003c/span\u003e \u003cspan class=\"p\"\u003e}\u003c/span\u003e \u003cspan class=\"c1\"\u003e# 19.16 seconds\u003c/span\u003e\n\u003cspan class=\"c1\"\u003e# =\u0026gt; 1200000\u003c/span\u003e\n\n\u003cspan class=\"c1\"\u003e# Delete the keys in batches of 10k\u003c/span\u003e\n\u003cspan class=\"n\"\u003etime\u003c/span\u003e \u003cspan class=\"p\"\u003e{\u003c/span\u003e \u003cspan class=\"n\"\u003er\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003ekeys\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"s1\"\u003e'rpush:notifications:*'\u003c/span\u003e\u003cspan class=\"p\"\u003e).\u003c/span\u003e\u003cspan class=\"nf\"\u003eeach_slice\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"mi\"\u003e10_000\u003c/span\u003e\u003cspan class=\"p\"\u003e)\u003c/span\u003e \u003cspan class=\"p\"\u003e{\u003c/span\u003e \u003cspan class=\"o\"\u003e|\u003c/span\u003e\u003cspan class=\"n\"\u003es\u003c/span\u003e\u003cspan class=\"o\"\u003e|\u003c/span\u003e \u003cspan class=\"nb\"\u003ep\u003c/span\u003e \u003cspan class=\"n\"\u003er\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003edel\u003c/span\u003e \u003cspan class=\"n\"\u003es\u003c/span\u003e \u003cspan class=\"p\"\u003e}\u003c/span\u003e \u003cspan class=\"p\"\u003e}\u003c/span\u003e \u003cspan class=\"c1\"\u003e# 32.90s\u003c/span\u003e\n\n\u003cspan class=\"c1\"\u003e# They're deleted\u003c/span\u003e\n\u003cspan class=\"n\"\u003etime\u003c/span\u003e \u003cspan class=\"p\"\u003e{\u003c/span\u003e \u003cspan class=\"n\"\u003er\u003c/span\u003e\u003cspan class=\"p\"\u003e.\u003c/span\u003e\u003cspan class=\"nf\"\u003ekeys\u003c/span\u003e\u003cspan class=\"p\"\u003e(\u003c/span\u003e\u003cspan class=\"s1\"\u003e'rpush:notifications:*'\u003c/span\u003e\u003cspan class=\"p\"\u003e).\u003c/span\u003e\u003cspan class=\"nf\"\u003esize\u003c/span\u003e \u003cspan class=\"p\"\u003e}\u003c/span\u003e \u003cspan class=\"c1\"\u003e# 0.57s\u003c/span\u003e\n\u003cspan class=\"c1\"\u003e# =\u0026gt; 0\u003c/span\u003e\n\u003c/code\u003e\u003c/pre\u003e\n\u003cdiv class=\"highlight__panel js-actions-panel\"\u003e\n\u003cdiv class=\"highlight__panel-action js-fullscreen-code-action\"\u003e\n \u003csvg xmlns=\"http://www.w3.org/2000/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-on\"\u003e\u003ctitle\u003eEnter fullscreen mode\u003c/title\u003e\n \u003cpath d=\"M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z\"\u003e\u003c/path\u003e\n\u003c/svg\u003e\n\n \u003csvg xmlns=\"http://www.w3.org/2000/svg\" width=\"20px\" height=\"20px\" viewbox=\"0 0 24 24\" class=\"highlight-action crayons-icon highlight-action--fullscreen-off\"\u003e\u003ctitle\u003eExit fullscreen mode\u003c/title\u003e\n \u003cpath d=\"M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z\"\u003e\u003c/path\u003e\n\u003c/svg\u003e\n\n\u003c/div\u003e\n\u003c/div\u003e\n\u003c/div\u003e\n\n\n\n","author":{"@type":"Person","name":"Josh Cheek","url":"https://dev.to/joshcheek"},"datePublished":"2021-07-18T02:16:50Z","dateModified":"2021-07-18T02:16:50Z","url":"https://dev.to/joshcheek/comment/1gcc0","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":3}],"comment":[{"@type":"Comment","@id":"#comment-748108","text":"\u003cp\u003eI like that you enjoy optimizing things and golfing code down to fit in tweets. 🙂 However, like many optimizations, this one is more complex than simple latency. \u003ccode\u003eKEYS\u003c/code\u003e would indeed have had lower latency, but it locks the Redis event loop for the duration, so we avoid that command wherever possible. To put it into perspective, your \u003ccode\u003eKEYS\u003c/code\u003e query took 7 seconds in your benchmark (we had almost 5M keys in Redis, so it would've actually taken longer, but I'll work with your numbers for the moment). In that amount of time we normally execute over 10,000 Redis queries, so running \u003ccode\u003eKEYS\u003c/code\u003e would introduce a pretty sizable head-of-line-blocking issue. On top of that, every HTTP request that makes it past Fastly to our backend hits this Redis instance at least once, so blocking Redis means blocking all HTTP requests. Since we don't want to do that, Redis needs to be \u003cem\u003eavailable af\u003c/em\u003e. We also don't allow other \u003ccode\u003eO(n)\u003c/code\u003e calls where \u003ccode\u003en\u003c/code\u003e can ever be nontrivial for that reason, so nothing like \u003ccode\u003eLRANGE my-list 0 -1\u003c/code\u003e, either.\u003c/p\u003e\n\n\u003cp\u003eIn fairness, it almost certainly doesn't lock the main Redis thread for the \u003cem\u003efull\u003c/em\u003e amount of time it takes the Ruby app to receive the entire list of keys (we're using Redis 6, which \u003ca href=\"http://antirez.com/news/132\" rel=\"nofollow\"\u003edoes I/O in a separate thread\u003c/a\u003e), but without any insight into how much of that time is spent in the main Redis event loop, we don't take chances. Since this issue impacted only a tiny fraction of our total traffic, we needed to continue optimizing for availability of Redis over shaving a few seconds off of the latency of the solution.\u003c/p\u003e\n\n\u003cp\u003eThe code Molly posted was adapted from \u003ca href=\"https://github.com/forem/forem/blob/94171f0cfbe16d9f320753ac6e67b0169ef586cc/app/workers/push_notifications/cleanup_worker.rb#L10-L18\" rel=\"nofollow\"\u003ethis worker\u003c/a\u003e. I didn't know about \u003ccode\u003eRedis#scan_each\u003c/code\u003e, though. That completely obviates our manual tracking of the cursor, which I \u003cem\u003ereally\u003c/em\u003e like.\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Jamie Gaskins","url":"https://dev.to/jgaskins"},"datePublished":"2021-07-21T02:04:25Z","dateModified":"2021-07-21T02:04:25Z","url":"https://dev.to/jgaskins/comment/1geha","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":2}],"parentItem":{"@type":"Comment","@id":"#comment-746616"}}]},{"@type":"Comment","@id":"#comment-745011","text":"\u003cp\u003eGreat run down thanks for being open\u003c/p\u003e\n\n","author":{"@type":"Person","name":"Benny Powers 🇮🇱🦁️","url":"https://dev.to/bennypowers"},"datePublished":"2021-07-14T21:40:21Z","dateModified":"2021-07-14T21:40:21Z","url":"https://dev.to/bennypowers/comment/1ga27","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":3}]},{"@type":"Comment","@id":"#comment-745009","text":"\u003cp\u003eWow Thanks for the update of the experience! \u003c/p\u003e\n\n","author":{"@type":"Person","name":"Jepsen","url":"https://dev.to/0xjepsen"},"datePublished":"2021-07-14T21:32:32Z","dateModified":"2021-07-14T21:32:32Z","url":"https://dev.to/0xjepsen/comment/1ga25","interactionStatistic":[{"@type":"InteractionCounter","interactionType":"https://schema.org/LikeAction","userInteractionCount":3}]}]}}
|
||
</script>
|
||
|
||
|
||
<div class="crayons-layout crayons-layout--3-cols crayons-layout--article">
|
||
<aside class="crayons-layout__sidebar-left" aria-label="Article actions">
|
||
<script>
|
||
if (/Twitter for (iPhone|iPad|Android)/i.test(navigator.userAgent)) {
|
||
document.documentElement.classList.add('is-twitter-in-app');
|
||
}
|
||
</script>
|
||
|
||
<style>
|
||
.community-leader-visible-block {
|
||
display: none !important;
|
||
}
|
||
.trusted-visible-block {
|
||
display: none !important;
|
||
}
|
||
.only-sidebar-menu-item {
|
||
display: none !important;
|
||
}
|
||
@media (min-width: 640px) {
|
||
body.community-leader-status-true .only-sidebar-menu-item.community-leader-visible-block {
|
||
display: block !important;
|
||
}
|
||
body.trusted-status-true .only-sidebar-menu-item.trusted-visible-block {
|
||
display: block !important;
|
||
}
|
||
}
|
||
.only-mobile-menu-item {
|
||
display: none !important;
|
||
}
|
||
body.community-leader-status-true .only-mobile-menu-item.community-leader-visible-block {
|
||
display: block !important;
|
||
}
|
||
body.trusted-status-true .only-mobile-menu-item.trusted-visible-block {
|
||
display: block !important;
|
||
}
|
||
@media (min-width: 640px) {
|
||
body.community-leader-status-true .only-mobile-menu-item.community-leader-visible-block {
|
||
display: none !important;
|
||
}
|
||
body.trusted-status-true .only-mobile-menu-item.trusted-visible-block {
|
||
display: none !important;
|
||
}
|
||
}
|
||
.crayons-reaction__count {
|
||
line-height: 18px !important;
|
||
height: 18px !important;
|
||
min-height: 18px !important;
|
||
display: inline-flex !important;
|
||
align-items: center !important;
|
||
justify-content: center !important;
|
||
}
|
||
@media (min-width: 768px) {
|
||
.crayons-reaction__count {
|
||
display: flex !important;
|
||
}
|
||
}
|
||
.crayons-reaction__count > span {
|
||
height: 16px !important;
|
||
width: 16px !important;
|
||
padding: 0 !important;
|
||
display: inline-block !important;
|
||
}
|
||
</style>
|
||
|
||
<div class="crayons-article-actions print-hidden">
|
||
<div class="crayons-article-actions__inner">
|
||
|
||
|
||
<div class="reaction-drawer__outer hoverdown" style="">
|
||
<button
|
||
id="reaction-drawer-trigger"
|
||
aria-label="reaction-drawer-trigger"
|
||
aria-pressed="false"
|
||
class="hoverdown-trigger crayons-reaction pseudo-reaction crayons-tooltip__activator relative">
|
||
<span class="crayons-reaction__icon crayons-reaction__icon--borderless crayons-reaction--like crayons-reaction__icon--inactive" style="width: 40px; height: 40px">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" role="img" aria-hidden="true" class="crayons-icon">
|
||
<g clip-path="url(#clip0_988_3276)">
|
||
<path d="M19 14V17H22V19H18.999L19 22H17L16.999 19H14V17H17V14H19ZM20.243 4.75698C22.505 7.02498 22.583 10.637 20.479 12.992L19.059 11.574C20.39 10.05 20.32 7.65998 18.827 6.16998C17.324 4.67098 14.907 4.60698 13.337 6.01698L12.002 7.21498L10.666 6.01798C9.09103 4.60598 6.67503 4.66798 5.17203 6.17198C3.68203 7.66198 3.60703 10.047 4.98003 11.623L13.412 20.069L12 21.485L3.52003 12.993C1.41603 10.637 1.49503 7.01898 3.75603 4.75698C6.02103 2.49298 9.64403 2.41698 12 4.52898C14.349 2.41998 17.979 2.48998 20.242 4.75698H20.243Z" fill="#525252"></path>
|
||
</g>
|
||
<defs>
|
||
<clipPath id="clip0_988_3276">
|
||
<rect width="24" height="24" fill="white"></rect>
|
||
</clipPath>
|
||
</defs>
|
||
</svg>
|
||
|
||
</span>
|
||
<span class="crayons-reaction__icon crayons-reaction__icon--borderless crayons-reaction__icon--active" style="width: 40px; height: 40px">
|
||
<img aria_hidden="true" height="24" width="24" src="https://assets.dev.to/assets/heart-plus-active-9ea3b22f2bc311281db911d416166c5f430636e76b15cd5df6b3b841d830eefa.svg" />
|
||
</span>
|
||
<span class="crayons-reaction__count" id="reaction_total_count">
|
||
<span class="bg-base-40 opacity-25 inline-block radius-default" style="height: 16px; width: 16px;"></span>
|
||
</span>
|
||
<span class="crayons-tooltip__content">
|
||
Add reaction
|
||
</span>
|
||
</button>
|
||
|
||
<div class="reaction-drawer" aria-expanded="false">
|
||
<div class="reaction-drawer__container">
|
||
<button
|
||
id="reaction-butt-like"
|
||
name="Like"
|
||
aria-label="Like"
|
||
aria-pressed="false"
|
||
class="crayons-reaction crayons-tooltip__activator relative pt-2 pr-2 pb-1 pl-2"
|
||
data-category="like">
|
||
<span class="crayons-reaction__icon crayons-reaction__icon--inactive p-0">
|
||
<img aria_hidden="true" height="32" width="32" src="https://assets.dev.to/assets/sparkle-heart-5f9bee3767e18deb1bb725290cb151c25234768a0e9a2bd39370c382d02920cf.svg" />
|
||
</span>
|
||
<span class="crayons-reaction__count" id="reaction-number-like"><span class="bg-base-40 opacity-25 p-2 inline-block radius-default"></span></span>
|
||
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Like
|
||
</span>
|
||
</button>
|
||
|
||
<button
|
||
id="reaction-butt-unicorn"
|
||
name="Unicorn"
|
||
aria-label="Unicorn"
|
||
aria-pressed="false"
|
||
class="crayons-reaction crayons-tooltip__activator relative pt-2 pr-2 pb-1 pl-2"
|
||
data-category="unicorn">
|
||
<span class="crayons-reaction__icon crayons-reaction__icon--inactive p-0">
|
||
<img aria_hidden="true" height="32" width="32" src="https://assets.dev.to/assets/multi-unicorn-b44d6f8c23cdd00964192bedc38af3e82463978aa611b4365bd33a0f1f4f3e97.svg" />
|
||
</span>
|
||
<span class="crayons-reaction__count" id="reaction-number-unicorn"><span class="bg-base-40 opacity-25 p-2 inline-block radius-default"></span></span>
|
||
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Unicorn
|
||
</span>
|
||
</button>
|
||
|
||
<button
|
||
id="reaction-butt-exploding_head"
|
||
name="Exploding Head"
|
||
aria-label="Exploding Head"
|
||
aria-pressed="false"
|
||
class="crayons-reaction crayons-tooltip__activator relative pt-2 pr-2 pb-1 pl-2"
|
||
data-category="exploding_head">
|
||
<span class="crayons-reaction__icon crayons-reaction__icon--inactive p-0">
|
||
<img aria_hidden="true" height="32" width="32" src="https://assets.dev.to/assets/exploding-head-daceb38d627e6ae9b730f36a1e390fca556a4289d5a41abb2c35068ad3e2c4b5.svg" />
|
||
</span>
|
||
<span class="crayons-reaction__count" id="reaction-number-exploding_head"><span class="bg-base-40 opacity-25 p-2 inline-block radius-default"></span></span>
|
||
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Exploding Head
|
||
</span>
|
||
</button>
|
||
|
||
<button
|
||
id="reaction-butt-raised_hands"
|
||
name="Raised Hands"
|
||
aria-label="Raised Hands"
|
||
aria-pressed="false"
|
||
class="crayons-reaction crayons-tooltip__activator relative pt-2 pr-2 pb-1 pl-2"
|
||
data-category="raised_hands">
|
||
<span class="crayons-reaction__icon crayons-reaction__icon--inactive p-0">
|
||
<img aria_hidden="true" height="32" width="32" src="https://assets.dev.to/assets/raised-hands-74b2099fd66a39f2d7eed9305ee0f4553df0eb7b4f11b01b6b1b499973048fe5.svg" />
|
||
</span>
|
||
<span class="crayons-reaction__count" id="reaction-number-raised_hands"><span class="bg-base-40 opacity-25 p-2 inline-block radius-default"></span></span>
|
||
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Raised Hands
|
||
</span>
|
||
</button>
|
||
|
||
<button
|
||
id="reaction-butt-fire"
|
||
name="Fire"
|
||
aria-label="Fire"
|
||
aria-pressed="false"
|
||
class="crayons-reaction crayons-tooltip__activator relative pt-2 pr-2 pb-1 pl-2"
|
||
data-category="fire">
|
||
<span class="crayons-reaction__icon crayons-reaction__icon--inactive p-0">
|
||
<img aria_hidden="true" height="32" width="32" src="https://assets.dev.to/assets/fire-f60e7a582391810302117f987b22a8ef04a2fe0df7e3258a5f49332df1cec71e.svg" />
|
||
</span>
|
||
<span class="crayons-reaction__count" id="reaction-number-fire"><span class="bg-base-40 opacity-25 p-2 inline-block radius-default"></span></span>
|
||
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Fire
|
||
</span>
|
||
</button>
|
||
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<button
|
||
id="reaction-butt-comment"
|
||
aria-label="Jump to Comments"
|
||
aria-pressed="false"
|
||
class="crayons-reaction crayons-reaction--comment crayons-tooltip__activator relative"
|
||
data-category="comment">
|
||
<span class="crayons-reaction__icon crayons-reaction__icon--borderless crayons-reaction__icon--inactive">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-hidden="true" class="crayons-icon">
|
||
<path d="M10 3h4a8 8 0 010 16v3.5c-5-2-12-5-12-11.5a8 8 0 018-8zm2 14h2a6 6 0 000-12h-4a6 6 0 00-6 6c0 3.61 2.462 5.966 8 8.48V17z"></path>
|
||
</svg>
|
||
|
||
</span>
|
||
<span class="crayons-reaction__count" id="reaction-number-comment" data-count="9">
|
||
<span class="bg-base-40 opacity-25 inline-block radius-default" style="height: 16px; width: 16px;"></span>
|
||
</span>
|
||
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Jump to Comments
|
||
</span>
|
||
</button>
|
||
|
||
<button
|
||
id="reaction-butt-readinglist"
|
||
aria-label="Add to reading list"
|
||
aria-pressed="false"
|
||
class="crayons-reaction crayons-reaction--readinglist crayons-tooltip__activator relative"
|
||
data-category="readinglist">
|
||
<span class="crayons-reaction__icon crayons-reaction__icon--borderless crayons-reaction__icon--inactive">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-hidden="true" class="crayons-icon">
|
||
<path d="M5 2h14a1 1 0 011 1v19.143a.5.5 0 01-.766.424L12 18.03l-7.234 4.536A.5.5 0 014 22.143V3a1 1 0 011-1zm13 2H6v15.432l6-3.761 6 3.761V4z"></path>
|
||
</svg>
|
||
|
||
</span>
|
||
<span class="crayons-reaction__count" id="reaction-number-readinglist"><span class="bg-base-40 opacity-25 inline-block radius-default" style="height: 16px; width: 16px;"></span></span>
|
||
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Save
|
||
</span>
|
||
</button>
|
||
|
||
|
||
<button
|
||
id="reaction-butt-boost"
|
||
aria-label="Boost"
|
||
aria-pressed="false"
|
||
class="crayons-reaction crayons-reaction--boost crayons-tooltip__activator relative">
|
||
<span class="crayons-reaction__icon crayons-reaction__icon--borderless crayons-reaction__icon--inactive">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor" role="img" aria-hidden="true" class="crayons-icon" width="24" height="24">
|
||
<path transform="translate(24,0) scale(-1,1)" d="M6 4H21C21.5523 4 22 4.44772 22 5V12H20V6H6V9L1 5L6 1V4ZM18 20H3C2.44772 20 2 19.5523 2 19V12H4V18H18V15L23 19L18 23V20Z"></path>
|
||
</svg>
|
||
|
||
</span>
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Boost
|
||
</span>
|
||
</button>
|
||
|
||
|
||
<div class="only-sidebar-menu-item community-leader-visible-block align-center">
|
||
<span class="favorite-control-root" data-favorite-control="true" data-variant="article" data-favoritable-type="Article" data-favoritable-id="759150" data-favoritable-user-id="119473" data-favorited="false" data-label-favorite="Pick as gem" data-label-favorited="Picked as gem" data-label-favorited-by-you="Picked as gem by you" data-modal-title="Gem Picked!" data-modal-body="You've picked this as a community gem!" data-modal-remaining-zero="You have no more gems left to give out today." data-modal-remaining-one="You have 1 more gem left to give out today." data-modal-remaining-other="You have %{count} more gems left to give out today." data-modal-remaining-unlimited="You have unlimited gems to give out." data-modal-close="Got it" data-modal-exhausted-title="Out of Gems" data-modal-exhausted-body="You have no more gems left to give out today. Your allowance will refresh soon!" data-modal-exhausted-close="Got it"><button type="button" class="favorite-reaction favorite-control crayons-tooltip__activator relative" aria-label="Pick as gem"><span class="crayons-reaction__icon crayons-reaction__icon--borderless"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" class="crayons-icon crayons-icon--default"><path d="M5.95 3.5h12.1L23.5 9 12 20.5.5 9 5.95 3.5z" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg>
|
||
</span><span data-testid="tooltip" class="crayons-tooltip__content">Pick as gem</span></button></span>
|
||
</div>
|
||
|
||
<div id="mod-actions-menu-btn-area" class="only-sidebar-menu-item trusted-visible-block align-center">
|
||
</div>
|
||
<div class="align-center m:relative">
|
||
<button id="article-show-more-button" aria-controls="article-show-more-dropdown" aria-expanded="false" aria-haspopup="true" class="dropbtn crayons-btn crayons-btn--ghost-dimmed crayons-btn--icon-rounded" aria-label="Share post options">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="ahhgx3psi08j4nvwiqpn1joxu0kwek4f" aria-hidden="true" class="crayons-icon dropdown-icon"><title id="ahhgx3psi08j4nvwiqpn1joxu0kwek4f">More...</title><path fill-rule="evenodd" clip-rule="evenodd" d="M7 12a2 2 0 11-4 0 2 2 0 014 0zm7 0a2 2 0 11-4 0 2 2 0 014 0zm5 2a2 2 0 100-4 2 2 0 000 4z"></path></svg>
|
||
|
||
</button>
|
||
|
||
<div id="article-show-more-dropdown" class="crayons-dropdown side-bar left-2 right-2 m:right-auto m:left-100 s:left-auto mb-1 m:mb-0 top-unset bottom-100 m:top-0 m:bottom-unset">
|
||
<div>
|
||
<button
|
||
id="copy-post-url-button"
|
||
class="flex justify-between crayons-link crayons-link--block w-100 bg-transparent border-0"
|
||
data-postUrl="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj">
|
||
<span class="fw-bold">Copy link</span>
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" id="article-copy-icon" role="img" aria-labelledby="an5f8pk47wv5id6nlrk8564559p6u7u1" aria-hidden="true" class="crayons-icon mx-2 shrink-0"><title id="an5f8pk47wv5id6nlrk8564559p6u7u1">Copy link</title>
|
||
<path d="M7 6V3a1 1 0 011-1h12a1 1 0 011 1v14a1 1 0 01-1 1h-3v3c0 .552-.45 1-1.007 1H4.007A1 1 0 013 21l.003-14c0-.552.45-1 1.007-1H7zm2 0h8v10h2V4H9v2zm-2 5v2h6v-2H7zm0 4v2h6v-2H7z"></path>
|
||
</svg>
|
||
|
||
</button>
|
||
<div id="article-copy-link-announcer" aria-live="polite" class="crayons-notice crayons-notice--success my-2 p-1" aria-live="polite" hidden>Copied to Clipboard</div>
|
||
</div>
|
||
|
||
<div class="Desktop-only">
|
||
<a
|
||
target="_blank"
|
||
class="crayons-link crayons-link--block"
|
||
rel="noopener"
|
||
href='https://twitter.com/intent/tweet?text=%22Incident%20Retro%3A%20Failing%20Comment%20Creation%20%2B%20Erroneous%20Push%20Notifications%22%20by%20%40molly_struve%20%23DEVCommunity%20https%3A%2F%2Fdev.to%2Fdevteam%2Fincident-retro-failing-comment-creation-erroneous-push-notifications-55dj'>
|
||
Share to X
|
||
</a>
|
||
<a
|
||
target="_blank"
|
||
class="crayons-link crayons-link--block"
|
||
rel="noopener"
|
||
href="https://www.linkedin.com/shareArticle?mini=true&url=https%3A%2F%2Fdev.to%2Fdevteam%2Fincident-retro-failing-comment-creation-erroneous-push-notifications-55dj&title=Incident%20Retro%3A%20Failing%20Comment%20Creation%20%2B%20Erroneous%20Push%20Notifications&summary=This%20incident%20retro%20was%20tougher%20than%20most%20to%20share%20because%2C%20despite%20the%20seriousness%20of%20the%20issue%2C%20it...&source=DEV%20Community">
|
||
Share to LinkedIn
|
||
</a>
|
||
<a
|
||
target="_blank"
|
||
class="crayons-link crayons-link--block"
|
||
rel="noopener"
|
||
href="https://www.facebook.com/sharer.php?u=https%3A%2F%2Fdev.to%2Fdevteam%2Fincident-retro-failing-comment-creation-erroneous-push-notifications-55dj">
|
||
Share to Facebook
|
||
</a>
|
||
<a
|
||
target="_blank"
|
||
class="crayons-link crayons-link--block"
|
||
rel="noopener"
|
||
href="https://s2f.kytta.dev/?text=https%3A%2F%2Fdev.to%2Fdevteam%2Fincident-retro-failing-comment-creation-erroneous-push-notifications-55dj">
|
||
Share to Mastodon
|
||
</a>
|
||
</div>
|
||
|
||
<web-share-wrapper shareurl="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj" sharetitle="Incident Retro: Failing Comment Creation + Erroneous Push Notifications" sharetext="This incident retro was tougher than most to share because, despite the seriousness of the issue, it..." template="web-share-button">
|
||
</web-share-wrapper>
|
||
<template id="web-share-button">
|
||
<a href="#" class="dropdown-link-row crayons-link crayons-link--block">Share Post via...</a>
|
||
</template>
|
||
|
||
<a href="/report-abuse" class="crayons-link crayons-link--block" id="ReportButtonAuthor">Report Abuse</a>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
</aside>
|
||
|
||
<main id="main-content" class="crayons-layout__content grid gap-4">
|
||
<div class="article-wrapper">
|
||
|
||
|
||
<article class="crayons-card crayons-article mb-4"
|
||
id="article-show-container"
|
||
data-article-id="759150"
|
||
data-article-slug="incident-retro-failing-comment-creation-erroneous-push-notifications-55dj"
|
||
data-author-id="119473"
|
||
data-author-name="Molly Struve (she/her)"
|
||
data-author-username="devteam"
|
||
data-co-author-ids=""
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj"
|
||
data-pin-path="/stories/feed/pinned_article"
|
||
data-pinned-article-id=""
|
||
data-published="true"
|
||
data-scheduled="false"
|
||
lang=en
|
||
>
|
||
<script>
|
||
try {
|
||
if(localStorage) {
|
||
let currentUser = localStorage.getItem('current_user');
|
||
|
||
if (currentUser) {
|
||
currentUser = JSON.parse(currentUser);
|
||
if (currentUser.id === 119473) {
|
||
document.getElementById('article-show-container').classList.add('current-user-is-article-author');
|
||
}
|
||
}
|
||
}
|
||
} catch (e) {
|
||
console.error(e);
|
||
}
|
||
</script>
|
||
<header class="crayons-article__header" id="main-title">
|
||
|
||
<div class="crayons-article__header__meta">
|
||
<div class="flex s:items-start flex-col s:flex-row">
|
||
<div id="action-space" class="crayons-article__actions mb-4 s:mb-0 s:order-last"></div>
|
||
<div class="flex flex-1 mb-5 items-start">
|
||
<div class="relative">
|
||
<a href="/devteam"><img src="https://media2.dev.to/dynamic/image/width=50,height=50,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Forganization%2Fprofile_image%2F1%2Fd908a186-5651-4a5a-9f76-15200bc6801f.jpg" class="radius-default align-middle" width="40" height="40" alt="The DEV Team profile image"></a>
|
||
<a href="/molly" class="absolute -right-2 -bottom-2 radius-full border border-solid border-2 border-base-inverted inline-flex">
|
||
<img class="radius-full align-middle" src="https://media2.dev.to/dynamic/image/width=50,height=50,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F119473%2F4fe2a414-c5d4-4cfe-b9da-8b9da90fb5e6.jpg" width="24" height="24" alt="Molly Struve (she/her)" />
|
||
</a>
|
||
</div>
|
||
<div class="pl-3 flex-1">
|
||
<a href="/molly" class="crayons-link fw-bold">Molly Struve (she/her)</a>
|
||
|
||
|
||
<span class="color-base-60"> for </span><a href="/devteam" class="crayons-link">The DEV Team</a>
|
||
<p class="fs-xs color-base-60 flex flex-wrap items-center gap-1">
|
||
<span>Posted on <time datetime="2021-07-14T20:53:41Z" class="date">Jul 14, 2021</time></span>
|
||
|
||
|
||
|
||
</p>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="multiple_reactions_engagement">
|
||
<button
|
||
class="reaction_engagement_like hidden crayons-reaction crayons-reaction--like"
|
||
data-category="like"
|
||
aria-label="Like"
|
||
type="button"
|
||
><span class="crayons-reaction__icon crayons-reaction__icon--inactive p-1"><img src="https://assets.dev.to/assets/sparkle-heart-5f9bee3767e18deb1bb725290cb151c25234768a0e9a2bd39370c382d02920cf.svg" width="24" height="24" /></span>
|
||
<span class="crayons-reaction__count" id="reaction_engagement_like_count"> </span>
|
||
</button> <button
|
||
class="reaction_engagement_unicorn hidden crayons-reaction crayons-reaction--unicorn"
|
||
data-category="unicorn"
|
||
aria-label="Unicorn"
|
||
type="button"
|
||
><span class="crayons-reaction__icon crayons-reaction__icon--inactive p-1"><img src="https://assets.dev.to/assets/multi-unicorn-b44d6f8c23cdd00964192bedc38af3e82463978aa611b4365bd33a0f1f4f3e97.svg" width="24" height="24" /></span>
|
||
<span class="crayons-reaction__count" id="reaction_engagement_unicorn_count"> </span>
|
||
</button> <button
|
||
class="reaction_engagement_exploding_head hidden crayons-reaction crayons-reaction--exploding_head"
|
||
data-category="exploding_head"
|
||
aria-label="Exploding Head"
|
||
type="button"
|
||
><span class="crayons-reaction__icon crayons-reaction__icon--inactive p-1"><img src="https://assets.dev.to/assets/exploding-head-daceb38d627e6ae9b730f36a1e390fca556a4289d5a41abb2c35068ad3e2c4b5.svg" width="24" height="24" /></span>
|
||
<span class="crayons-reaction__count" id="reaction_engagement_exploding_head_count"> </span>
|
||
</button> <button
|
||
class="reaction_engagement_raised_hands hidden crayons-reaction crayons-reaction--raised_hands"
|
||
data-category="raised_hands"
|
||
aria-label="Raised Hands"
|
||
type="button"
|
||
><span class="crayons-reaction__icon crayons-reaction__icon--inactive p-1"><img src="https://assets.dev.to/assets/raised-hands-74b2099fd66a39f2d7eed9305ee0f4553df0eb7b4f11b01b6b1b499973048fe5.svg" width="24" height="24" /></span>
|
||
<span class="crayons-reaction__count" id="reaction_engagement_raised_hands_count"> </span>
|
||
</button> <button
|
||
class="reaction_engagement_fire hidden crayons-reaction crayons-reaction--fire"
|
||
data-category="fire"
|
||
aria-label="Fire"
|
||
type="button"
|
||
><span class="crayons-reaction__icon crayons-reaction__icon--inactive p-1"><img src="https://assets.dev.to/assets/fire-f60e7a582391810302117f987b22a8ef04a2fe0df7e3258a5f49332df1cec71e.svg" width="24" height="24" /></span>
|
||
<span class="crayons-reaction__count" id="reaction_engagement_fire_count"> </span>
|
||
</button>
|
||
</div>
|
||
|
||
|
||
<h1 class=" fs-3xl m:fs-4xl l:fs-5xl fw-bold s:fw-heavy lh-tight mb-2 long">
|
||
Incident Retro: Failing Comment Creation + Erroneous Push Notifications
|
||
</h1>
|
||
|
||
<div class="spec__tags flex flex-wrap">
|
||
<a class="crayons-tag " style="
|
||
--tag-bg: rgba(64, 78, 211, 0.10);
|
||
--tag-prefix: #404ED3;
|
||
--tag-bg-hover: rgba(64, 78, 211, 0.10);
|
||
--tag-prefix-hover: #404ED3;
|
||
" href="/t/incident"><span class="crayons-tag__prefix">#</span>incident</a>
|
||
<a class="crayons-tag " style="
|
||
--tag-bg: rgba(64, 78, 211, 0.10);
|
||
--tag-prefix: #404ED3;
|
||
--tag-bg-hover: rgba(64, 78, 211, 0.10);
|
||
--tag-prefix-hover: #404ED3;
|
||
" href="/t/retro"><span class="crayons-tag__prefix">#</span>retro</a>
|
||
<a class="crayons-tag " style="
|
||
--tag-bg: rgba(64, 78, 211, 0.10);
|
||
--tag-prefix: #404ED3;
|
||
--tag-bg-hover: rgba(64, 78, 211, 0.10);
|
||
--tag-prefix-hover: #404ED3;
|
||
" href="/t/postmortem"><span class="crayons-tag__prefix">#</span>postmortem</a>
|
||
</div>
|
||
</div>
|
||
</header>
|
||
|
||
<div class="crayons-article__main ">
|
||
<div class="crayons-article__body text-styles spec__body" data-article-id="759150"
|
||
id="article-body">
|
||
<p><em>This incident retro was tougher than most to share because, despite the seriousness of the issue, it affected only a very small percentage of our user base. However, we learned some incredibly valuable lessons and I think it's only right that I give others the chance to learn from our mistake as well.</em></p>
|
||
|
||
<h2>
|
||
<a name="context" href="#context">
|
||
</a>
|
||
Context
|
||
</h2>
|
||
|
||
<p>On Thursday July 8th, we merged a <a href="https://github.com/forem/forem/pull/14121" target="_blank" rel="noopener noreferrer">very large PR</a> that updated our code to start using our new User Settings instead of the deprecated fields on users. The goal of moving these fields to user settings is to lighten up the user model and make these types of settings more configurable on a per-Forem basis. </p>
|
||
|
||
<h2>
|
||
<a name="problem" href="#problem">
|
||
</a>
|
||
Problem
|
||
</h2>
|
||
|
||
<p>On July 9th around midday, it was brought to our attention <a href="https://github.com/forem/forem/issues/14183" target="_blank" rel="noopener noreferrer">through a bug report</a> that the comment creation experience on the frontend seemed to be broken. At this point folks on the team started looking into the problem. Using <a href="https://www.honeycomb.io/" target="_blank" rel="noopener noreferrer">Honeycomb</a>, we were able to confirm that the problem started when the User Settings PR was deployed which made it the prime suspect. </p>
|
||
|
||
<p><a href="https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fforem.team%2Fremoteimages%2Fuploads%2Farticles%2Facuvsc0l8y4135nois0f.png" class="article-body-image-wrapper"><img src="https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fforem.team%2Fremoteimages%2Fuploads%2Farticles%2Facuvsc0l8y4135nois0f.png" alt="Picture of a graph where the line is steady under 100ms and jumps to thousands" loading="lazy" width="800" height="400"></a></p>
|
||
|
||
<p>I immediately dove into the PR and quickly found <a href="https://github.com/forem/forem/pull/14121/files#diff-56d67da7087534bc7d4cffd953e00dc2e3eb41047f1e82f3124cd6d69c509873R38" target="_blank" rel="noopener noreferrer">the line of code that had been broken</a>.<br>
|
||
<a href="https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fforem.team%2Fremoteimages%2Fuploads%2Farticles%2Fjh78q3nsi71vrdyekkk0.png" class="article-body-image-wrapper"><img src="https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fforem.team%2Fremoteimages%2Fuploads%2Farticles%2Fjh78q3nsi71vrdyekkk0.png" alt="Github PR dif" loading="lazy" width="800" height="400"></a></p>
|
||
|
||
<p>We had incorrectly removed the <code>user_ids</code> scope from the filtering used to collect user IDs for sending mobile push notifications. This was causing us to attempt to grab notification settings and IDs for all users with mobile notifications turned on which was often timing out. These timeouts would occur after a comment was created so the comment in most cases still existed but we never returned a success response to the frontend. This caused the frontend to freeze creating a bad experience for the user. </p>
|
||
<h2>
|
||
<a name="fix" href="#fix">
|
||
</a>
|
||
Fix
|
||
</h2>
|
||
|
||
<p>In order to remedy this situation, we quickly added back the scope and deployed. Immediately the errors and timeouts went away and things returned to normal for comment creation. </p>
|
||
|
||
<p><a href="https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fforem.team%2Fremoteimages%2Fuploads%2Farticles%2Fvpicn144aagfpsrbnnhs.png" class="article-body-image-wrapper"><img src="https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fforem.team%2Fremoteimages%2Fuploads%2Farticles%2Fvpicn144aagfpsrbnnhs.png" alt="Graph of increased duration returning to normal" loading="lazy" width="800" height="400"></a></p>
|
||
|
||
<p>At this point, we concluded that possibly some extra notifications had been sent but that the majority of these requests had timed out so we assumed we were in a good state now. I looked for some sort of push notification model in our app where we might have stored unsent notifications but upon not finding one figured we were clear. </p>
|
||
<h2>
|
||
<a name="more-problems" href="#more-problems">
|
||
</a>
|
||
More Problems
|
||
</h2>
|
||
|
||
<p><strong>July 9th ~12pm EDT</strong>: Unfortunately, the above assumption was incorrect. Just before midnight eastern, a team member reported she was getting random notifications on her phone. At this point, our mobile engineer jumped in to investigate and found that we had a ton of push notifications enqueued in Redis.</p>
|
||
|
||
<p><strong>July 9th ~2am EDT</strong>: Mobile engineer posts about the issue in our #emergency Slack channel.</p>
|
||
|
||
<p><strong>July 9th ~7am EDT</strong>: On-call engineer wakes up and sees the #emergency Slack message and jumps in to help investigate and try to resolve the issue. Shortly after, I also jumped in. It was at this point that I learned all about how our push notification system worked. We use <a href="https://github.com/rpush/rpush" target="_blank" rel="noopener noreferrer">RPush</a> for communicating with various push-notification services. RPush stores the data about the push notifications in Redis for quick insertion and removal times. We quickly assessed that we had a large number of push notifications enqueued in Redis waiting to be sent, despite the solution deployed the day before. </p>
|
||
<h2>
|
||
<a name="fix-2" href="#fix-2">
|
||
</a>
|
||
Fix 2
|
||
</h2>
|
||
|
||
<p>Upon realizing we had all of these records in Redis, we took the following steps to resolve the issue:</p>
|
||
|
||
<ul>
|
||
<li>Removal of the iOS cert from Heroku to prevent sending any notifications. Without this cert we could ensure we would not send anymore bad notifications and that would give us time to fix the data in Redis.</li>
|
||
<li>To be safe, we removed all of the <code>PushNotifications::DeliveryWorker</code> jobs from Sidekiq so as not to be pinging Redis for the keys causing our key collection to slow down.
|
||
|
||
<ul>
|
||
<li>At the same time, we looped through all of the keys in Redis and collected all of the rpush ones. These included all delivered and undelivered keys. There were 1.2 million rpush keys so this process took about 30 mins. We used the below script:
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
<div class="highlight js-code-highlight">
|
||
<pre class="highlight ruby"><code><span class="n">redis</span> <span class="o">=</span> <span class="no">Rails</span><span class="p">.</span><span class="nf">cache</span><span class="p">.</span><span class="nf">redis</span>
|
||
<span class="n">start</span> <span class="o">=</span> <span class="mi">0</span>
|
||
<span class="n">total</span> <span class="o">=</span> <span class="mi">0</span>
|
||
<span class="n">key_collection</span> <span class="o">=</span> <span class="p">[]</span>
|
||
<span class="n">index</span><span class="p">,</span> <span class="n">keys</span> <span class="o">=</span> <span class="n">redis</span><span class="p">.</span><span class="nf">scan</span><span class="p">(</span><span class="n">start</span><span class="p">);</span>
|
||
<span class="k">while</span> <span class="n">index</span> <span class="o">!=</span> <span class="s2">"0"</span>
|
||
<span class="n">start</span> <span class="o">=</span> <span class="n">index</span>
|
||
<span class="n">total</span> <span class="o">+=</span> <span class="n">keys</span><span class="p">.</span><span class="nf">count</span>
|
||
<span class="n">keys</span><span class="p">.</span><span class="nf">each</span> <span class="k">do</span> <span class="o">|</span><span class="n">key</span><span class="o">|</span>
|
||
<span class="n">key_collection</span> <span class="o"><<</span> <span class="n">key</span> <span class="k">if</span> <span class="n">key</span><span class="p">.</span><span class="nf">include?</span><span class="p">(</span><span class="s1">'rpush:notifications'</span><span class="p">)</span>
|
||
<span class="k">end</span>
|
||
<span class="n">index</span><span class="p">,</span> <span class="n">keys</span> <span class="o">=</span> <span class="n">redis</span><span class="p">.</span><span class="nf">scan</span><span class="p">(</span><span class="n">start</span><span class="p">)</span>
|
||
<span class="k">end</span>
|
||
</code></pre>
|
||
<div class="highlight__panel js-actions-panel">
|
||
<div class="highlight__panel-action js-fullscreen-code-action">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="20px" height="20px" viewbox="0 0 24 24" class="highlight-action crayons-icon highlight-action--fullscreen-on"><title>Enter fullscreen mode</title>
|
||
<path d="M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z"></path>
|
||
</svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="20px" height="20px" viewbox="0 0 24 24" class="highlight-action crayons-icon highlight-action--fullscreen-off"><title>Exit fullscreen mode</title>
|
||
<path d="M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z"></path>
|
||
</svg>
|
||
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<ul>
|
||
<li>Once we had the keys collected we double checked that they were the keys we wanted and then we deleted them ALL in batches of 10 using this code:
|
||
</li>
|
||
</ul>
|
||
<div class="highlight js-code-highlight">
|
||
<pre class="highlight ruby"><code><span class="n">key_collection</span><span class="p">.</span><span class="nf">each_slice</span><span class="p">(</span><span class="mi">10</span><span class="p">){</span><span class="o">|</span><span class="n">s</span><span class="o">|</span> <span class="nb">puts</span> <span class="n">redis</span><span class="p">.</span><span class="nf">del</span><span class="p">(</span><span class="n">s</span><span class="p">)}</span>
|
||
</code></pre>
|
||
<div class="highlight__panel js-actions-panel">
|
||
<div class="highlight__panel-action js-fullscreen-code-action">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="20px" height="20px" viewbox="0 0 24 24" class="highlight-action crayons-icon highlight-action--fullscreen-on"><title>Enter fullscreen mode</title>
|
||
<path d="M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z"></path>
|
||
</svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="20px" height="20px" viewbox="0 0 24 24" class="highlight-action crayons-icon highlight-action--fullscreen-off"><title>Exit fullscreen mode</title>
|
||
<path d="M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z"></path>
|
||
</svg>
|
||
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<ul>
|
||
<li>Once the keys were gone, we double checked the count and then added back the iOS cert to Heroku. </li>
|
||
<li>After Heroku restarted, we ran a test to confirm that the notifications were back to sending and that we were recording delivered notifications correctly again in Redis. </li>
|
||
</ul>
|
||
<h2>
|
||
<a name="impact" href="#impact">
|
||
</a>
|
||
Impact
|
||
</h2>
|
||
<h3>
|
||
<a name="comment-creation" href="#comment-creation">
|
||
</a>
|
||
Comment Creation
|
||
</h3>
|
||
|
||
<p>The comment creation flow on the frontend was broken for 30 hours after the UserSettings PR was deployed on July 8th. However, comment creation levels remained steady.<br>
|
||
</p>
|
||
|
||
<div class="highlight js-code-highlight">
|
||
<pre class="highlight plaintext"><code>irb(main):001:0> Comment.where(created_at: 36.hours.ago..Time.now).count
|
||
=> 762
|
||
irb(main):002:0> Comment.where(created_at: 72.hours.ago..36.hours.ago).count
|
||
=> 857
|
||
irb(main):003:0> Comment.where(created_at: 108.hours.ago..72.hours.ago).count
|
||
=> 852
|
||
irb(main):004:0> Comment.where(created_at: 144.hours.ago..108.hours.ago).count
|
||
=> 756
|
||
irb(main):005:0> Comment.where(created_at: 180.hours.ago..144.hours.ago).count
|
||
=> 587
|
||
</code></pre>
|
||
<div class="highlight__panel js-actions-panel">
|
||
<div class="highlight__panel-action js-fullscreen-code-action">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="20px" height="20px" viewbox="0 0 24 24" class="highlight-action crayons-icon highlight-action--fullscreen-on"><title>Enter fullscreen mode</title>
|
||
<path d="M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z"></path>
|
||
</svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="20px" height="20px" viewbox="0 0 24 24" class="highlight-action crayons-icon highlight-action--fullscreen-off"><title>Exit fullscreen mode</title>
|
||
<path d="M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z"></path>
|
||
</svg>
|
||
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<p>In total, there were 934 comments that were affected by the broken frontend interface. </p>
|
||
|
||
<p><a href="https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2F3csl8linhu5fzipww4gq.png" class="article-body-image-wrapper"><img src="https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2F3csl8linhu5fzipww4gq.png" alt="Honeycomb.io graph showing a total count of 934 comments over a 30 hour period" loading="lazy" width="800" height="563"></a></p>
|
||
|
||
<h3>
|
||
<a name="push-notifications" href="#push-notifications">
|
||
</a>
|
||
Push Notifications
|
||
</h3>
|
||
|
||
<p>Given DEV is early on in its mobile journey, only 0.2% of our users have registered devices that are able to receive push notifications. This means that only 0.2% of our users were affected by this incident. We consider ourselves lucky that we were able to expose some of these issues and have this incident remain very contained. </p>
|
||
|
||
<p>Since the numbers of users affected was small, we decide to proactively reach out to all of those folks via email to apologize and explain why they <em>might</em> have received erroneous notifications. </p>
|
||
|
||
<h2>
|
||
<a name="learnings" href="#learnings">
|
||
</a>
|
||
Learnings
|
||
</h2>
|
||
|
||
<h3>
|
||
<a name="large-prs" href="#large-prs">
|
||
</a>
|
||
Large PRs
|
||
</h3>
|
||
|
||
<p>Large PRs naturally come with more risk. The bug that kicked off this stream of events was a small change in a very large PR that got missed by multiple folks. For starters, the diff for the line was not particularly helpful in recognizing the change that caused the issue. In addition, there were so many lines that it was easy to overlook. Breaking down PRs is one way to prevent this. </p>
|
||
|
||
<p>However, there are times when large, wide spread changes need to be made and in those cases you have to rely on your test suite. </p>
|
||
|
||
<h3>
|
||
<a name="missing-tests" href="#missing-tests">
|
||
</a>
|
||
Missing tests
|
||
</h3>
|
||
|
||
<p>The feature that broke was completely untested. Had we been testing that feature properly I think there is a good chance the bug could have been caught. Immediately after pushing the hotfix <a href="https://github.com/forem/forem/pull/14187" target="_blank" rel="noopener noreferrer">I added a test</a> to ensure this never happens again. </p>
|
||
|
||
<h3>
|
||
<a name="technical-feature-education" href="#technical-feature-education">
|
||
</a>
|
||
Technical feature education
|
||
</h3>
|
||
|
||
<p>After fixing the bug, none of us working on the problem were well educated about how our Push Notification systems worked. Being late on a Friday, we skimmed the code and concluded we would be good to go. This ended up being the wrong assumption as we later learned Saturday that push notifications, unlike app ones, are stored and enqueued via Redis. I think it's more important than ever that we are somehow sharing and educating each other about some of these larger features as we roll them out. </p>
|
||
|
||
<p>Thankfully, we do <a href="https://docs.forem.com/backend/push-notifications/" target="_blank" rel="noopener noreferrer">have some great Push Notification documentation</a> but it was never sought out during the incident. Is there a way we can make docs like this more visible? During the incident we were all heads down in the code, should we maybe have some sort of URL link in the code to the docs?</p>
|
||
|
||
<h3>
|
||
<a name="being-more-intentional" href="#being-more-intentional">
|
||
</a>
|
||
Being more intentional
|
||
</h3>
|
||
|
||
<p>I could have very easily reached out to our mobile team on Friday evening to double check that we were in a good state for Push Notifications. Once again, the whole Friday evening and wanting to be done caused me to accept my assumptions rather than challenge and check them. </p>
|
||
|
||
<p>One easy way to prevent something like this is to use a checklist. A checklist (heavily used in aviation because it has been shown to be key in preventing incidents,) is an easy way to make sure you never miss something and are deliberate and intentional with your decisions. We have great checklists for handling Heroku incidents in our internal Gitbook. However, we don't have a general incident on-call checklist which we plan to add. </p>
|
||
|
||
<h2>
|
||
<a name="incident-response" href="#incident-response">
|
||
</a>
|
||
Incident Response
|
||
</h2>
|
||
|
||
<p>Before I dive into specifics, I want to point out that this is the first larger incident we have had in a while. Incidents being rare is GREAT! But, it also means that our incident response was a bit rusty and our documented incident processes were a bit out of date to handle this situation. Going forward we will be taking a closer look at all of these things to ensure we are keeping them relevant and useful based on the current state of our application and team. </p>
|
||
|
||
<h3>
|
||
<a name="broken-comment-creation-reporting-flow" href="#broken-comment-creation-reporting-flow">
|
||
</a>
|
||
Broken Comment Creation Reporting Flow
|
||
</h3>
|
||
|
||
<p>The initial comment creation problem was reported by a DEV user via a GitHub issue and seen about 11 hours later by our internal engineering team. GitHub Issues are naturally not a very urgent form of reporting for us since they are handled during working hours. One way we could improve this flow would be to further communicate that urgent support issues need to be emailed to <code>yo@forem.com</code>. Our customer success team is always monitoring these channels diligently and could more quickly triage and escalate an issue to the team. </p>
|
||
|
||
<p>Manual reporting aside, ideally, we should have caught this programmatically with our monitoring by alerting on the increased HTTP request errors that were detected.<br>
|
||
<a href="https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fforem.team%2Fremoteimages%2Fuploads%2Farticles%2Fianj96377wcnqd10q0l7.png" class="article-body-image-wrapper"><img src="https://media2.dev.to/dynamic/image/width=800%2Cheight=%2Cfit=scale-down%2Cgravity=auto%2Cformat=auto/https%3A%2F%2Fforem.team%2Fremoteimages%2Fuploads%2Farticles%2Fianj96377wcnqd10q0l7.png" alt="Datadog graphs showing elevated error rates" loading="lazy" width="800" height="400"></a></p>
|
||
|
||
<p>Going forward, we plan to set up a monitor to alert us for these changes so that we are the first to find out, rather than relying on users reporting the issue to us. With the proper monitor in place, we could have caught this within a couple of hours instead of over a day. </p>
|
||
|
||
<h3>
|
||
<a name="broken-push-notifications-reporting-flow" href="#broken-push-notifications-reporting-flow">
|
||
</a>
|
||
Broken Push Notifications Reporting Flow
|
||
</h3>
|
||
|
||
<p>We responded to the broken comment creation issue immediately when it was surfaced, however, the followup push notification issues were not. Our engineer correctly pinged the #emergency channel when he realized that we were having problems with notifications but did not escalate the issue and wake up the on-call dev via PagerDuty. </p>
|
||
|
||
<p>One way we could streamline this process would be to see if we could set up our #emergency Slack channel to automatically ping PageDuty when a message is posted. Another option is address some of our incident response rustiness by better educating the team on our desired incident response flows. This probably needs to be a reoccurring training that happens periodically so as to prevent processes and knowledge from getting stale as our systems evolve. </p>
|
||
|
||
<h3>
|
||
<a name="technical-remediation" href="#technical-remediation">
|
||
</a>
|
||
Technical Remediation
|
||
</h3>
|
||
|
||
<p>One delay we encountered when trying to fix this issue was that the Rpush keys were stored in the same Redis instance as our Rails cache. Given our Rails cache is massive and contains 4+ million keys, looping through all of them to find only the Rpush keys was tedious and took a while. That cache is also very active which caused us to hit some timeouts when we were trying to perform heavier operations on it. </p>
|
||
|
||
<p>We could have mitigated this friction by having either a separate Redis instance for Rpush OR by having those keys in a separate database away from our Rails cache keys. Going forward, as we grow our mobile platforms, I think we should make one of the above changes to ensure that we have easy access to these keys and records. It will also ensure that as we grow and scale push notifications we don't have to worry about impacting other systems like our core Rails app. </p>
|
||
|
||
<h2>
|
||
<a name="thanks-everyone" href="#thanks-everyone">
|
||
</a>
|
||
Thanks Everyone 🤗
|
||
</h2>
|
||
|
||
<p>A lot of people gave up their personal time to come together on Friday and Saturday to mitigate and fix these issues. I truly appreciate the amazing team we have at Forem and their dedication to this software. We learned a lot about our systems in the process which ensures that this incident will not go to waste. I hope others can learn from this as well.</p>
|
||
|
||
|
||
</div>
|
||
|
||
<div class="below-post-bb body-billboard-container" data-async-url="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/bmar11/post_body_bottom"></div>
|
||
</div>
|
||
<section id="comments" data-follow-button-container="true" data-updated-at="2026-09-12 01:47:43 UTC" class="text-padding mb-4 border-t-1 border-0 border-solid border-base-10">
|
||
<header class="relative flex justify-between items-center mb-6">
|
||
<div class="flex items-center">
|
||
<h2 class="crayons-subtitle-1">
|
||
Top comments <span class="js-comments-count" data-comments-count="9">(9)</span>
|
||
</h2>
|
||
</div>
|
||
|
||
|
||
<div id="comment-subscription" class="print-hidden">
|
||
<div class="crayons-btn-group">
|
||
<span class="crayons-btn crayons-btn--outlined">Subscribe</span>
|
||
</div>
|
||
</div>
|
||
</header>
|
||
<div id="billboard_delay_trigger"></div>
|
||
<div
|
||
id="comments-container"
|
||
data-testid="comments-container"
|
||
data-commentable-id="759150"
|
||
data-commentable-type="Article"
|
||
data-has-recent-comment-activity="false">
|
||
|
||
<div id="response-templates-data" class="hidden"></div>
|
||
|
||
|
||
<form class="comment-form print-hidden" id="new_comment" action="/comments" accept-charset="UTF-8" method="post">
|
||
|
||
<input type="hidden" name="authenticity_token" value="NOTHING" id="new_comment_authenticity_token">
|
||
|
||
<input value="759150" autocomplete="off" type="hidden" name="comment[commentable_id]" id="comment_commentable_id" />
|
||
<input value="Article" autocomplete="off" type="hidden" name="comment[commentable_type]" id="comment_commentable_type" />
|
||
|
||
|
||
<span class="crayons-avatar m:crayons-avatar--l mr-2 shrink-0">
|
||
<img src="https://media2.dev.to/dynamic/image/width=256,height=,fit=scale-down,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2F8j7kvp660rqzt99zui8e.png" width="32" height="32" alt="pic" class="crayons-avatar__image overflow-hidden" id="comment-primary-user-profile--avatar" loading="lazy" />
|
||
</span>
|
||
<div class="comment-form__inner">
|
||
<div class="comment-form__field" data-tracking-name="comment_form_textfield">
|
||
<textarea placeholder="Add to the discussion" onfocus="handleFocus(event)" onkeyup="handleKeyUp(event)" onkeydown="handleKeyDown(event)" oninput="handleChange(event)" id="text-area" required="required" class="crayons-textfield comment-textarea crayons-textfield--ghost" aria-label="Add a comment to the discussion" name="comment[body_markdown]">
|
||
</textarea>
|
||
|
||
</div>
|
||
|
||
<div class="response-templates-container crayons-card crayons-card--secondary p-4 mb-4 comment-form__templates fs-base hidden">
|
||
<header class="mb-3">
|
||
<button type="button" class="crayons-btn personal-template-button active" data-target-type="personal" data-form-id="new_comment">Personal</button>
|
||
<button type="button" class="crayons-btn moderator-template-button hidden" data-target-type="moderator" data-form-id="new_comment">Trusted User</button>
|
||
</header>
|
||
|
||
<div class="personal-responses-container">
|
||
</div>
|
||
<div class="moderator-responses-container hidden">
|
||
</div>
|
||
|
||
<a target="_blank" rel="noopener nofollow" href="/settings/response-templates">
|
||
Create template
|
||
</a>
|
||
<p>Templates let you quickly answer FAQs or store snippets for re-use.</p>
|
||
</div>
|
||
|
||
<div class="comment-form__preview text-styles text-styles--secondary" id="preview-div"></div>
|
||
|
||
<div class="comment-form__buttons mb-4 flex items-center flex-wrap gap-2">
|
||
<button type="submit" class="crayons-btn mr-2 js-btn-enable" onclick="validateField(event)" data-tracking-name="comment_submit_button" disabled>Submit</button>
|
||
<button type="button" class="preview-toggle crayons-btn crayons-btn--secondary comment-action-preview js-btn-enable mr-2" data-tracking-name="comment_preview_button" disabled>Preview</button>
|
||
<a href="/404.html" class="dismiss-edit-comment crayons-btn crayons-btn--ghost js-btn-dismiss hidden">Dismiss</a>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="code-of-conduct" id="toggle-code-of-conduct-checkbox"></div>
|
||
</form>
|
||
|
||
|
||
<div class="comments" id="comment-trees-container">
|
||
<details class="comment-wrapper js-comment-wrapper comment-wrapper--deep-0
|
||
root
|
||
" open>
|
||
<summary aria-label="Toggle this comment (and replies)" data-tracking-name="expand_comment_toggle">
|
||
<span class="m:mx-1 inline-block align-middle">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="a74lw76dv0tcbqsi0mixt0ebs3nzobm3" class="crayons-icon expanded"><title id="a74lw76dv0tcbqsi0mixt0ebs3nzobm3">Collapse</title>
|
||
<path d="M12 10.677L8 6.935 9 6l3 2.807L15 6l1 .935-4 3.742zm0 4.517L9 18l-1-.935 4-3.742 4 3.742-1 .934-3-2.805z"></path>
|
||
</svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="a6ofvr7mnkn6phpclpz9u5ot4bhp5gcv" class="crayons-icon collapsed"><title id="a6ofvr7mnkn6phpclpz9u5ot4bhp5gcv">Expand</title>
|
||
<path d="M12 18l-4-3.771 1-.943 3 2.829 3-2.829 1 .943L12 18zm0-10.115l-3 2.829-1-.943L12 6l4 3.771-1 .942-3-2.828z"></path>
|
||
</svg>
|
||
|
||
</span>
|
||
<span class="js-collapse-comment-content inline-block align-middle"></span>
|
||
</summary>
|
||
<div
|
||
id="comment-node-745008"
|
||
class="
|
||
comment single-comment-node
|
||
|
||
root
|
||
comment--deep-0
|
||
|
||
"
|
||
data-comment-id="745008"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1ga24"
|
||
data-comment-author-id="38578"
|
||
data-content-user-id="38578">
|
||
<a name="comment-1ga24" style="position: absolute; top: -8px;"> </a>
|
||
|
||
<div class="comment__inner">
|
||
<a href="https://dev.to/michaeltharrington" class="shrink-0 crayons-avatar m:crayons-avatar--l mt-4 m:mt-3">
|
||
<img class="crayons-avatar__image" width="32" height="32" src="https://media2.dev.to/dynamic/image/width=50,height=50,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F38578%2F5cadc1f1-fbca-496d-a375-667c21d9c122.jpg" alt="michaeltharrington profile image" loading="lazy" />
|
||
</a>
|
||
|
||
|
||
<div class="inner-comment comment__details">
|
||
<div class="comment__content crayons-card">
|
||
|
||
|
||
|
||
<div class="comment__header">
|
||
<a href="https://dev.to/michaeltharrington" class="crayons-link crayons-link--secondary flex items-center fw-medium m:hidden">
|
||
<span class="js-comment-username">
|
||
Michael Tharrington
|
||
</span>
|
||
</a>
|
||
<div class="profile-preview-card relative mb-4 s:mb-0 fw-medium hidden m:block">
|
||
<button id="comment-profile-preview-trigger-745008" aria-controls="comment-profile-preview-content-745008" class="profile-preview-card__trigger p-1 -my-1 -ml-1 crayons-btn crayons-btn--ghost" aria-label="Michael Tharrington profile details">
|
||
Michael Tharrington
|
||
<a style="display: inline;" href="/++"><img alt="Subscriber" class="subscription-icon" src="https://assets.dev.to/assets/subscription-icon-805dfa7ac7dd660f07ed8d654877270825b07a92a03841aa99a1093bd00431b2.png" /></a>
|
||
|
||
</button>
|
||
<div id="comment-profile-preview-content-745008" class="profile-preview-card__content p-4 pt-0 branded-7 crayons-dropdown" style="--card-color: #9fcfac; border-top-color: var(--card-color);" data-testid="profile-preview-card" data-repositioning-dropdown="true">
|
||
<div class="gap-4 grid">
|
||
<div class="-mt-4">
|
||
<a href="/michaeltharrington" class="flex">
|
||
<span class="crayons-avatar crayons-avatar--xl mr-2 shrink-0">
|
||
<img src="https://media2.dev.to/dynamic/image/width=90,height=90,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F38578%2F5cadc1f1-fbca-496d-a375-667c21d9c122.jpg" class="crayons-avatar__image" alt="" loading="lazy" />
|
||
</span>
|
||
<span class="crayons-link crayons-subtitle-2 mt-5">
|
||
Michael Tharrington
|
||
<img class="subscription-icon" src="https://assets.dev.to/assets/subscription-icon-805dfa7ac7dd660f07ed8d654877270825b07a92a03841aa99a1093bd00431b2.png" />
|
||
|
||
</span>
|
||
</a>
|
||
</div>
|
||
|
||
<div class="print-hidden">
|
||
<button name="button" type="button" data-info="{"className":"User","style":"","id":38578,"name":"Michael Tharrington"}" class="crayons-btn follow-action-button whitespace-nowrap w-100 follow-user" aria-label="Follow user: Michael Tharrington" aria-pressed="false">Follow</button>
|
||
</div>
|
||
<div class="color-base-70">
|
||
I'm a friendly, non-dev, cisgender guy from NC who enjoys playing music/making noise, hiking, eating veggies, and hanging out with my best friend/wife + our 3 kitties + 1 greyhound.
|
||
</div>
|
||
|
||
<div class="user-metadata-details">
|
||
<ul class="user-metadata-details-inner">
|
||
<li>
|
||
<div class="key">
|
||
Email
|
||
</div>
|
||
<div class="value">
|
||
<a href="mailto:mct3545@gmail.com">mct3545@gmail.com</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Location
|
||
</div>
|
||
<div class="value">
|
||
North Carolina
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Education
|
||
</div>
|
||
<div class="value">
|
||
BFA in Creative Writing
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Pronouns
|
||
</div>
|
||
<div class="value">
|
||
he/him
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Work
|
||
</div>
|
||
<div class="value">
|
||
Senior Community Manager at DEV
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Joined
|
||
</div>
|
||
<div class="value">
|
||
<time datetime="2017-10-24T14:36:06Z" class="date">Oct 24, 2017</time>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
<span class="color-base-30 px-2 m:pl-0" role="presentation">•</span>
|
||
|
||
<a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1ga24" class="comment-date crayons-link crayons-link--secondary fs-s">
|
||
<time datetime="2021-07-14T21:30:32Z" class=date-short-year>
|
||
Jul 14 '21
|
||
</time>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
<div class="comment__dropdown" data-tracking-name="comment_dropdown">
|
||
<button id="comment-dropdown-trigger-745008" aria-controls="comment-dropdown-745008" aria-expanded="false"
|
||
class="dropbtn comment__dropdown-trigger crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon "
|
||
aria-label="Toggle dropdown menu" aria-haspopup="true">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="apcwtewhiolsa2n0b3m1h6zr3puc237j" class="crayons-icon pointer-events-none"><title id="apcwtewhiolsa2n0b3m1h6zr3puc237j">Dropdown menu</title>
|
||
<path fill-rule="evenodd" clip-rule="evenodd" d="M8.25 12a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm5.25 0a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm3.75 1.5a1.5 1.5 0 100-3 1.5 1.5 0 000 3z"></path>
|
||
</svg>
|
||
|
||
</button>
|
||
<div id="comment-dropdown-745008" class="crayons-dropdown right-1 s:right-0 s:left-auto fs-base dropdown">
|
||
<ul class="m-0">
|
||
<li><a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1ga24" class="crayons-link crayons-link--block permalink-copybtn" aria-label="Copy link to Michael Tharrington's comment" data-no-instant>Copy link</a></li>
|
||
<li class="comment-actions hidden" data-user-id="38578" data-action="settings-button" data-path="https://dev.to/michaeltharrington/comment/1ga24/settings" aria-label="Go to Michael Tharrington's comment settings"></li>
|
||
<li class="comment-actions hidden" data-action="hide-button" data-commentable-user-id="119473" data-user-id="38578">
|
||
<button
|
||
class="flex justify-between crayons-link crayons-link--block w-100 bg-transparent border-0 hide-comment"
|
||
data-hide-type="hide"
|
||
data-comment-id="745008"
|
||
data-comment-url="https://dev.to/michaeltharrington/comment/1ga24"
|
||
aria-label="Hide Michael Tharrington's comment">
|
||
Hide
|
||
</button>
|
||
</li>
|
||
<li class="mod-actions hidden mod-actions-comment-button" data-path="https://dev.to/michaeltharrington/comment/1ga24/mod" aria-label="Moderate Michael Tharrington's comment"></li>
|
||
<li class="report-abuse-link-wrapper" data-path="/report-abuse?url=https://dev.to/michaeltharrington/comment/1ga24" aria-label="Report Michael Tharrington's comment as abusive or violating our code of conduct and/or terms and conditions"></li>
|
||
<li class="current-user-actions"></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<div
|
||
class="
|
||
comment__body
|
||
text-styles
|
||
text-styles--secondary
|
||
body
|
||
|
||
|
||
">
|
||
<p>I made this joke on Slack and I'll make it again here...</p>
|
||
|
||
<p><a href="https://res.cloudinary.com/practicaldev/image/fetch/s--J6GJ-e0y--/c_limit%2Cf_auto%2Cfl_progressive%2Cq_auto%2Cw_880/https://dev-to-uploads.s3.amazonaws.com/uploads/articles/22ljq94e39pqjzjym34f.png" class="article-body-image-wrapper"><img src="https://res.cloudinary.com/practicaldev/image/fetch/s--J6GJ-e0y--/c_limit%2Cf_auto%2Cfl_progressive%2Cq_auto%2Cw_880/https://dev-to-uploads.s3.amazonaws.com/uploads/articles/22ljq94e39pqjzjym34f.png" alt="https://dev-to-uploads.s3.amazonaws.com/uploads/articles/22ljq94e39pqjzjym34f.png" loading="lazy"></a></p>
|
||
|
||
<p>...is this a <a href="https://www.denofgeek.com/tv/loki-what-is-a-nexus-event/" rel="nofollow">Nexus event</a>?</p>
|
||
|
||
<p>Loving Loki! </p>
|
||
|
||
<p><a href="https://i.giphy.com/media/jxsXCHBt4ZaEGYajmD/source.gif" class="article-body-image-wrapper"><img src="https://i.giphy.com/media/jxsXCHBt4ZaEGYajmD/source.gif" alt="Mobius says "You're my favorite"" loading="lazy"></a></p>
|
||
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
<script>
|
||
</script>
|
||
|
||
<footer class="comment__footer print-hidden">
|
||
<button
|
||
class="crayons-tooltip__activator relative crayons-btn crayons-btn--ghost crayons-btn--icon-left crayons-btn--s mr-1 reaction-like inline-flex reaction-button"
|
||
id="button-for-comment-745008"
|
||
data-comment-id="745008"
|
||
aria-label="like"
|
||
data-tracking-name="comment_heart_button">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="a9ak6e7hvme6yb7ptnrdb7va1gglw7si" class="crayons-icon reaction-icon not-reacted"><title id="a9ak6e7hvme6yb7ptnrdb7va1gglw7si">Like comment: </title><path d="M18.884 12.595l.01.011L12 19.5l-6.894-6.894.01-.01A4.875 4.875 0 0112 5.73a4.875 4.875 0 016.884 6.865zM6.431 7.037a3.375 3.375 0 000 4.773L12 17.38l5.569-5.569a3.375 3.375 0 10-4.773-4.773L9.613 10.22l-1.06-1.062 2.371-2.372a3.375 3.375 0 00-4.492.25v.001z"></path></svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="a8jkm5qv03xr5xxeyhnk3vpj6qadmhoq" class="crayons-icon crayons-icon reaction-icon--like reaction-icon reacted"><title id="a8jkm5qv03xr5xxeyhnk3vpj6qadmhoq">Like comment: </title>
|
||
<path d="M5.116 12.595a4.875 4.875 0 015.56-7.68h-.002L7.493 8.098l1.06 1.061 3.181-3.182a4.875 4.875 0 016.895 6.894L12 19.5l-6.894-6.894.01-.01z"></path>
|
||
</svg>
|
||
|
||
<span class="reactions-count">8</span><span class="reactions-label hidden m:inline-block"> likes</span>
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Like
|
||
</span>
|
||
</button>
|
||
|
||
<span class="favorite-control-root" data-favorite-control="true" data-variant="comment" data-favoritable-type="Comment" data-favoritable-id="745008" data-favoritable-user-id="38578" data-favorited="false" data-label-favorite="Pick as gem" data-label-favorited="Picked as gem" data-label-favorited-by-you="Picked as gem by you" data-modal-title="Gem Picked!" data-modal-body="You've picked this as a community gem!" data-modal-remaining-zero="You have no more gems left to give out today." data-modal-remaining-one="You have 1 more gem left to give out today." data-modal-remaining-other="You have %{count} more gems left to give out today." data-modal-remaining-unlimited="You have unlimited gems to give out." data-modal-close="Got it" data-modal-exhausted-title="Out of Gems" data-modal-exhausted-body="You have no more gems left to give out today. Your allowance will refresh soon!" data-modal-exhausted-close="Got it"></span>
|
||
|
||
<button
|
||
class="actions crayons-btn crayons-btn--ghost crayons-btn--s crayons-btn--icon-left toggle-reply-form mr-1 inline-flex"
|
||
data-comment-id="745008"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1ga24"
|
||
data-tracking-name="comment_reply_button"
|
||
data-testid="reply-button-745008"
|
||
rel="nofollow">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="an319lb9eeheg8kedx1huf7t91ke6xa9" class="crayons-icon reaction-icon not-reacted"><title id="an319lb9eeheg8kedx1huf7t91ke6xa9">Comment button</title><path d="M10.5 5h3a6 6 0 110 12v2.625c-3.75-1.5-9-3.75-9-8.625a6 6 0 016-6zM12 15.5h1.5a4.501 4.501 0 001.722-8.657A4.5 4.5 0 0013.5 6.5h-3A4.5 4.5 0 006 11c0 2.707 1.846 4.475 6 6.36V15.5z"></path></svg>
|
||
|
||
<span class="hidden m:inline-block">Reply</span>
|
||
</button>
|
||
|
||
</footer>
|
||
|
||
</div>
|
||
</div>
|
||
<details class="comment-wrapper js-comment-wrapper comment-wrapper--deep-1
|
||
child
|
||
" open>
|
||
<summary aria-label="Toggle this comment (and replies)" data-tracking-name="expand_comment_toggle">
|
||
<span class="mx-0 inline-block align-middle">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="akr5f8pp38y1l0a2ze6geq15mx3skq75" class="crayons-icon expanded"><title id="akr5f8pp38y1l0a2ze6geq15mx3skq75">Collapse</title>
|
||
<path d="M12 10.677L8 6.935 9 6l3 2.807L15 6l1 .935-4 3.742zm0 4.517L9 18l-1-.935 4-3.742 4 3.742-1 .934-3-2.805z"></path>
|
||
</svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="an0eu79svfev7ougukxblhpktjde5pwb" class="crayons-icon collapsed"><title id="an0eu79svfev7ougukxblhpktjde5pwb">Expand</title>
|
||
<path d="M12 18l-4-3.771 1-.943 3 2.829 3-2.829 1 .943L12 18zm0-10.115l-3 2.829-1-.943L12 6l4 3.771-1 .942-3-2.828z"></path>
|
||
</svg>
|
||
|
||
</span>
|
||
<span class="js-collapse-comment-content inline-block align-middle"></span>
|
||
</summary>
|
||
<div
|
||
id="comment-node-745012"
|
||
class="
|
||
comment single-comment-node
|
||
|
||
child
|
||
comment--deep-1
|
||
|
||
"
|
||
data-comment-id="745012"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1ga28"
|
||
data-comment-author-id="119473"
|
||
data-content-user-id="119473">
|
||
<a name="comment-1ga28" style="position: absolute; top: -8px;"> </a>
|
||
|
||
<div class="comment__inner">
|
||
<a href="https://dev.to/molly" class="shrink-0 crayons-avatar mt-4">
|
||
<img class="crayons-avatar__image" width="32" height="32" src="https://media2.dev.to/dynamic/image/width=50,height=50,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F119473%2F4fe2a414-c5d4-4cfe-b9da-8b9da90fb5e6.jpg" alt="molly profile image" loading="lazy" />
|
||
</a>
|
||
|
||
|
||
<div class="inner-comment comment__details">
|
||
<div class="comment__content crayons-card">
|
||
|
||
|
||
|
||
<div class="comment__header">
|
||
<a href="https://dev.to/molly" class="crayons-link crayons-link--secondary flex items-center fw-medium m:hidden">
|
||
<span class="js-comment-username">
|
||
Molly Struve (she/her)
|
||
</span>
|
||
<span class="crayons-hover-tooltip inline-block spec-op-author -mr-2" data-tooltip="Author">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" class="crayons-icon">
|
||
<path d="M12 8.25a6 6 0 110 12 6 6 0 010-12zm0 2.625l-.992 2.01-2.218.322 1.605 1.564-.379 2.21L12 15.937l1.984 1.043-.379-2.209 1.605-1.564-2.218-.323L12 10.875zm.75-6.376l3.75.001v2.25l-1.022.854a7.45 7.45 0 00-2.728-.817V4.5zm-1.5 0v2.288a7.451 7.451 0 00-2.727.816L7.5 6.75V4.5h3.75z"></path>
|
||
</svg>
|
||
|
||
</span>
|
||
</a>
|
||
<div class="profile-preview-card relative mb-4 s:mb-0 fw-medium hidden m:block">
|
||
<button id="comment-profile-preview-trigger-745012" aria-controls="comment-profile-preview-content-745012" class="profile-preview-card__trigger p-1 -my-1 -ml-1 crayons-btn crayons-btn--ghost" aria-label="Molly Struve (she/her) profile details">
|
||
Molly Struve (she/her)
|
||
|
||
|
||
</button>
|
||
<div id="comment-profile-preview-content-745012" class="profile-preview-card__content p-4 pt-0 branded-7 crayons-dropdown" style="--card-color: #551029; border-top-color: var(--card-color);" data-testid="profile-preview-card" data-repositioning-dropdown="true">
|
||
<div class="gap-4 grid">
|
||
<div class="-mt-4">
|
||
<a href="/molly" class="flex">
|
||
<span class="crayons-avatar crayons-avatar--xl mr-2 shrink-0">
|
||
<img src="https://media2.dev.to/dynamic/image/width=90,height=90,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F119473%2F4fe2a414-c5d4-4cfe-b9da-8b9da90fb5e6.jpg" class="crayons-avatar__image" alt="" loading="lazy" />
|
||
</span>
|
||
<span class="crayons-link crayons-subtitle-2 mt-5">
|
||
Molly Struve (she/her)
|
||
|
||
|
||
</span>
|
||
</a>
|
||
</div>
|
||
|
||
<div class="print-hidden">
|
||
<button name="button" type="button" data-info="{"className":"User","style":"","id":119473,"name":"Molly Struve (she/her)"}" class="crayons-btn follow-action-button whitespace-nowrap w-100 follow-user" aria-label="Follow user: Molly Struve (she/her)" aria-pressed="false">Follow</button>
|
||
</div>
|
||
<div class="color-base-70">
|
||
International Speaker 🗣Runner 🏃♀️ Always Ambitious. Never Satisfied. I ride 🦄's IRL
|
||
</div>
|
||
|
||
<div class="user-metadata-details">
|
||
<ul class="user-metadata-details-inner">
|
||
<li>
|
||
<div class="key">
|
||
Location
|
||
</div>
|
||
<div class="value">
|
||
Ocala, FL
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Education
|
||
</div>
|
||
<div class="value">
|
||
MIT '10 Aerospace Engineering
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Pronouns
|
||
</div>
|
||
<div class="value">
|
||
she/her
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Work
|
||
</div>
|
||
<div class="value">
|
||
Staff Site Reliability Engineer at Netflix
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Joined
|
||
</div>
|
||
<div class="value">
|
||
<time datetime="2018-12-04T02:00:27Z" class="date">Dec 4, 2018</time>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<span class="crayons-hover-tooltip inline-block spec-op-author -ml-2" data-tooltip="Author">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" class="crayons-icon">
|
||
<path d="M12 8.25a6 6 0 110 12 6 6 0 010-12zm0 2.625l-.992 2.01-2.218.322 1.605 1.564-.379 2.21L12 15.937l1.984 1.043-.379-2.209 1.605-1.564-2.218-.323L12 10.875zm.75-6.376l3.75.001v2.25l-1.022.854a7.45 7.45 0 00-2.728-.817V4.5zm-1.5 0v2.288a7.451 7.451 0 00-2.727.816L7.5 6.75V4.5h3.75z"></path>
|
||
</svg>
|
||
|
||
</span>
|
||
|
||
</div>
|
||
|
||
<span class="color-base-30 px-2 m:pl-0" role="presentation">•</span>
|
||
|
||
<a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1ga28" class="comment-date crayons-link crayons-link--secondary fs-s">
|
||
<time datetime="2021-07-14T21:41:22Z" class=date-short-year>
|
||
Jul 14 '21
|
||
</time>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
<div class="comment__dropdown" data-tracking-name="comment_dropdown">
|
||
<button id="comment-dropdown-trigger-745012" aria-controls="comment-dropdown-745012" aria-expanded="false"
|
||
class="dropbtn comment__dropdown-trigger crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon "
|
||
aria-label="Toggle dropdown menu" aria-haspopup="true">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="anmwic1ybs47zccgh3ysyajlx5jvry0w" class="crayons-icon pointer-events-none"><title id="anmwic1ybs47zccgh3ysyajlx5jvry0w">Dropdown menu</title>
|
||
<path fill-rule="evenodd" clip-rule="evenodd" d="M8.25 12a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm5.25 0a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm3.75 1.5a1.5 1.5 0 100-3 1.5 1.5 0 000 3z"></path>
|
||
</svg>
|
||
|
||
</button>
|
||
<div id="comment-dropdown-745012" class="crayons-dropdown right-1 s:right-0 s:left-auto fs-base dropdown">
|
||
<ul class="m-0">
|
||
<li><a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1ga28" class="crayons-link crayons-link--block permalink-copybtn" aria-label="Copy link to Molly Struve (she/her)'s comment" data-no-instant>Copy link</a></li>
|
||
<li class="comment-actions hidden" data-user-id="119473" data-action="settings-button" data-path="https://dev.to/molly/comment/1ga28/settings" aria-label="Go to Molly Struve (she/her)'s comment settings"></li>
|
||
<li class="comment-actions hidden" data-action="hide-button" data-commentable-user-id="119473" data-user-id="119473">
|
||
<button
|
||
class="flex justify-between crayons-link crayons-link--block w-100 bg-transparent border-0 hide-comment"
|
||
data-hide-type="hide"
|
||
data-comment-id="745012"
|
||
data-comment-url="https://dev.to/molly/comment/1ga28"
|
||
aria-label="Hide Molly Struve (she/her)'s comment">
|
||
Hide
|
||
</button>
|
||
</li>
|
||
<li class="mod-actions hidden mod-actions-comment-button" data-path="https://dev.to/molly/comment/1ga28/mod" aria-label="Moderate Molly Struve (she/her)'s comment"></li>
|
||
<li class="report-abuse-link-wrapper" data-path="/report-abuse?url=https://dev.to/molly/comment/1ga28" aria-label="Report Molly Struve (she/her)'s comment as abusive or violating our code of conduct and/or terms and conditions"></li>
|
||
<li class="current-user-actions"></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<div
|
||
class="
|
||
comment__body
|
||
text-styles
|
||
text-styles--secondary
|
||
body
|
||
|
||
|
||
">
|
||
<p>I just started watching this last night!!!! LOL 😂</p>
|
||
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
<script>
|
||
</script>
|
||
|
||
<footer class="comment__footer print-hidden">
|
||
<button
|
||
class="crayons-tooltip__activator relative crayons-btn crayons-btn--ghost crayons-btn--icon-left crayons-btn--s mr-1 reaction-like inline-flex reaction-button"
|
||
id="button-for-comment-745012"
|
||
data-comment-id="745012"
|
||
aria-label="like"
|
||
data-tracking-name="comment_heart_button">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="ast6jp2j877il3b5otpfzl76lx3i8iiv" class="crayons-icon reaction-icon not-reacted"><title id="ast6jp2j877il3b5otpfzl76lx3i8iiv">Like comment: </title><path d="M18.884 12.595l.01.011L12 19.5l-6.894-6.894.01-.01A4.875 4.875 0 0112 5.73a4.875 4.875 0 016.884 6.865zM6.431 7.037a3.375 3.375 0 000 4.773L12 17.38l5.569-5.569a3.375 3.375 0 10-4.773-4.773L9.613 10.22l-1.06-1.062 2.371-2.372a3.375 3.375 0 00-4.492.25v.001z"></path></svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="an9cg05wkm8kvw8w80jkbfcydo0v6zmk" class="crayons-icon crayons-icon reaction-icon--like reaction-icon reacted"><title id="an9cg05wkm8kvw8w80jkbfcydo0v6zmk">Like comment: </title>
|
||
<path d="M5.116 12.595a4.875 4.875 0 015.56-7.68h-.002L7.493 8.098l1.06 1.061 3.181-3.182a4.875 4.875 0 016.895 6.894L12 19.5l-6.894-6.894.01-.01z"></path>
|
||
</svg>
|
||
|
||
<span class="reactions-count">3</span><span class="reactions-label hidden m:inline-block"> likes</span>
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Like
|
||
</span>
|
||
</button>
|
||
|
||
<span class="favorite-control-root" data-favorite-control="true" data-variant="comment" data-favoritable-type="Comment" data-favoritable-id="745012" data-favoritable-user-id="119473" data-favorited="false" data-label-favorite="Pick as gem" data-label-favorited="Picked as gem" data-label-favorited-by-you="Picked as gem by you" data-modal-title="Gem Picked!" data-modal-body="You've picked this as a community gem!" data-modal-remaining-zero="You have no more gems left to give out today." data-modal-remaining-one="You have 1 more gem left to give out today." data-modal-remaining-other="You have %{count} more gems left to give out today." data-modal-remaining-unlimited="You have unlimited gems to give out." data-modal-close="Got it" data-modal-exhausted-title="Out of Gems" data-modal-exhausted-body="You have no more gems left to give out today. Your allowance will refresh soon!" data-modal-exhausted-close="Got it"></span>
|
||
|
||
<button
|
||
class="actions crayons-btn crayons-btn--ghost crayons-btn--s crayons-btn--icon-left toggle-reply-form mr-1 inline-flex"
|
||
data-comment-id="745012"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1ga28"
|
||
data-tracking-name="comment_reply_button"
|
||
data-testid="reply-button-745012"
|
||
rel="nofollow">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="am6ae8zp3x3aid3lwmspwbnvz35u780y" class="crayons-icon reaction-icon not-reacted"><title id="am6ae8zp3x3aid3lwmspwbnvz35u780y">Comment button</title><path d="M10.5 5h3a6 6 0 110 12v2.625c-3.75-1.5-9-3.75-9-8.625a6 6 0 016-6zM12 15.5h1.5a4.501 4.501 0 001.722-8.657A4.5 4.5 0 0013.5 6.5h-3A4.5 4.5 0 006 11c0 2.707 1.846 4.475 6 6.36V15.5z"></path></svg>
|
||
|
||
<span class="hidden m:inline-block">Reply</span>
|
||
</button>
|
||
|
||
</footer>
|
||
|
||
</div>
|
||
</div>
|
||
<details class="comment-wrapper js-comment-wrapper comment-wrapper--deep-2
|
||
child
|
||
" open>
|
||
<summary aria-label="Toggle this comment (and replies)" data-tracking-name="expand_comment_toggle">
|
||
<span class="mx-0 inline-block align-middle">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="a1d0mnfx6djrjddhgnh3nj7a2zzzxhjx" class="crayons-icon expanded"><title id="a1d0mnfx6djrjddhgnh3nj7a2zzzxhjx">Collapse</title>
|
||
<path d="M12 10.677L8 6.935 9 6l3 2.807L15 6l1 .935-4 3.742zm0 4.517L9 18l-1-.935 4-3.742 4 3.742-1 .934-3-2.805z"></path>
|
||
</svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="a7e75g9i1sxzvlp5ishvyauvhp2tzsd5" class="crayons-icon collapsed"><title id="a7e75g9i1sxzvlp5ishvyauvhp2tzsd5">Expand</title>
|
||
<path d="M12 18l-4-3.771 1-.943 3 2.829 3-2.829 1 .943L12 18zm0-10.115l-3 2.829-1-.943L12 6l4 3.771-1 .942-3-2.828z"></path>
|
||
</svg>
|
||
|
||
</span>
|
||
<span class="js-collapse-comment-content inline-block align-middle"></span>
|
||
</summary>
|
||
<div
|
||
id="comment-node-745374"
|
||
class="
|
||
comment single-comment-node
|
||
|
||
child
|
||
comment--deep-2
|
||
|
||
"
|
||
data-comment-id="745374"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1gag6"
|
||
data-comment-author-id="38578"
|
||
data-content-user-id="38578">
|
||
<a name="comment-1gag6" style="position: absolute; top: -8px;"> </a>
|
||
|
||
<div class="comment__inner">
|
||
<a href="https://dev.to/michaeltharrington" class="shrink-0 crayons-avatar mt-4">
|
||
<img class="crayons-avatar__image" width="32" height="32" src="https://media2.dev.to/dynamic/image/width=50,height=50,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F38578%2F5cadc1f1-fbca-496d-a375-667c21d9c122.jpg" alt="michaeltharrington profile image" loading="lazy" />
|
||
</a>
|
||
|
||
|
||
<div class="inner-comment comment__details">
|
||
<div class="comment__content crayons-card">
|
||
|
||
|
||
|
||
<div class="comment__header">
|
||
<a href="https://dev.to/michaeltharrington" class="crayons-link crayons-link--secondary flex items-center fw-medium m:hidden">
|
||
<span class="js-comment-username">
|
||
Michael Tharrington
|
||
</span>
|
||
</a>
|
||
<div class="profile-preview-card relative mb-4 s:mb-0 fw-medium hidden m:block">
|
||
<button id="comment-profile-preview-trigger-745374" aria-controls="comment-profile-preview-content-745374" class="profile-preview-card__trigger p-1 -my-1 -ml-1 crayons-btn crayons-btn--ghost" aria-label="Michael Tharrington profile details">
|
||
Michael Tharrington
|
||
<a style="display: inline;" href="/++"><img alt="Subscriber" class="subscription-icon" src="https://assets.dev.to/assets/subscription-icon-805dfa7ac7dd660f07ed8d654877270825b07a92a03841aa99a1093bd00431b2.png" /></a>
|
||
|
||
</button>
|
||
<div id="comment-profile-preview-content-745374" class="profile-preview-card__content p-4 pt-0 branded-7 crayons-dropdown" style="--card-color: #9fcfac; border-top-color: var(--card-color);" data-testid="profile-preview-card" data-repositioning-dropdown="true">
|
||
<div class="gap-4 grid">
|
||
<div class="-mt-4">
|
||
<a href="/michaeltharrington" class="flex">
|
||
<span class="crayons-avatar crayons-avatar--xl mr-2 shrink-0">
|
||
<img src="https://media2.dev.to/dynamic/image/width=90,height=90,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F38578%2F5cadc1f1-fbca-496d-a375-667c21d9c122.jpg" class="crayons-avatar__image" alt="" loading="lazy" />
|
||
</span>
|
||
<span class="crayons-link crayons-subtitle-2 mt-5">
|
||
Michael Tharrington
|
||
<img class="subscription-icon" src="https://assets.dev.to/assets/subscription-icon-805dfa7ac7dd660f07ed8d654877270825b07a92a03841aa99a1093bd00431b2.png" />
|
||
|
||
</span>
|
||
</a>
|
||
</div>
|
||
|
||
<div class="print-hidden">
|
||
<button name="button" type="button" data-info="{"className":"User","style":"","id":38578,"name":"Michael Tharrington"}" class="crayons-btn follow-action-button whitespace-nowrap w-100 follow-user" aria-label="Follow user: Michael Tharrington" aria-pressed="false">Follow</button>
|
||
</div>
|
||
<div class="color-base-70">
|
||
I'm a friendly, non-dev, cisgender guy from NC who enjoys playing music/making noise, hiking, eating veggies, and hanging out with my best friend/wife + our 3 kitties + 1 greyhound.
|
||
</div>
|
||
|
||
<div class="user-metadata-details">
|
||
<ul class="user-metadata-details-inner">
|
||
<li>
|
||
<div class="key">
|
||
Email
|
||
</div>
|
||
<div class="value">
|
||
<a href="mailto:mct3545@gmail.com">mct3545@gmail.com</a>
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Location
|
||
</div>
|
||
<div class="value">
|
||
North Carolina
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Education
|
||
</div>
|
||
<div class="value">
|
||
BFA in Creative Writing
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Pronouns
|
||
</div>
|
||
<div class="value">
|
||
he/him
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Work
|
||
</div>
|
||
<div class="value">
|
||
Senior Community Manager at DEV
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Joined
|
||
</div>
|
||
<div class="value">
|
||
<time datetime="2017-10-24T14:36:06Z" class="date">Oct 24, 2017</time>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
<span class="color-base-30 px-2 m:pl-0" role="presentation">•</span>
|
||
|
||
<a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1gag6" class="comment-date crayons-link crayons-link--secondary fs-s">
|
||
<time datetime="2021-07-15T13:38:36Z" class=date-short-year>
|
||
Jul 15 '21
|
||
</time>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
<div class="comment__dropdown" data-tracking-name="comment_dropdown">
|
||
<button id="comment-dropdown-trigger-745374" aria-controls="comment-dropdown-745374" aria-expanded="false"
|
||
class="dropbtn comment__dropdown-trigger crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon "
|
||
aria-label="Toggle dropdown menu" aria-haspopup="true">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="ai56o4rwtg8p0h9pqx1gr49b4pc2eptv" class="crayons-icon pointer-events-none"><title id="ai56o4rwtg8p0h9pqx1gr49b4pc2eptv">Dropdown menu</title>
|
||
<path fill-rule="evenodd" clip-rule="evenodd" d="M8.25 12a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm5.25 0a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm3.75 1.5a1.5 1.5 0 100-3 1.5 1.5 0 000 3z"></path>
|
||
</svg>
|
||
|
||
</button>
|
||
<div id="comment-dropdown-745374" class="crayons-dropdown right-1 s:right-0 s:left-auto fs-base dropdown">
|
||
<ul class="m-0">
|
||
<li><a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1gag6" class="crayons-link crayons-link--block permalink-copybtn" aria-label="Copy link to Michael Tharrington's comment" data-no-instant>Copy link</a></li>
|
||
<li class="comment-actions hidden" data-user-id="38578" data-action="settings-button" data-path="https://dev.to/michaeltharrington/comment/1gag6/settings" aria-label="Go to Michael Tharrington's comment settings"></li>
|
||
<li class="comment-actions hidden" data-action="hide-button" data-commentable-user-id="119473" data-user-id="38578">
|
||
<button
|
||
class="flex justify-between crayons-link crayons-link--block w-100 bg-transparent border-0 hide-comment"
|
||
data-hide-type="hide"
|
||
data-comment-id="745374"
|
||
data-comment-url="https://dev.to/michaeltharrington/comment/1gag6"
|
||
aria-label="Hide Michael Tharrington's comment">
|
||
Hide
|
||
</button>
|
||
</li>
|
||
<li class="mod-actions hidden mod-actions-comment-button" data-path="https://dev.to/michaeltharrington/comment/1gag6/mod" aria-label="Moderate Michael Tharrington's comment"></li>
|
||
<li class="report-abuse-link-wrapper" data-path="/report-abuse?url=https://dev.to/michaeltharrington/comment/1gag6" aria-label="Report Michael Tharrington's comment as abusive or violating our code of conduct and/or terms and conditions"></li>
|
||
<li class="current-user-actions"></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<div
|
||
class="
|
||
comment__body
|
||
text-styles
|
||
text-styles--secondary
|
||
body
|
||
|
||
|
||
">
|
||
<p>Haha! Ooooo you're in for a treat, Molly. It's good times. I still need to watch the most recent, last episode.</p>
|
||
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
<script>
|
||
</script>
|
||
|
||
<footer class="comment__footer print-hidden">
|
||
<button
|
||
class="crayons-tooltip__activator relative crayons-btn crayons-btn--ghost crayons-btn--icon-left crayons-btn--s mr-1 reaction-like inline-flex reaction-button"
|
||
id="button-for-comment-745374"
|
||
data-comment-id="745374"
|
||
aria-label="like"
|
||
data-tracking-name="comment_heart_button">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="acsj9mw8w2e1ieajaeexlxxwpau8siux" class="crayons-icon reaction-icon not-reacted"><title id="acsj9mw8w2e1ieajaeexlxxwpau8siux">Like comment: </title><path d="M18.884 12.595l.01.011L12 19.5l-6.894-6.894.01-.01A4.875 4.875 0 0112 5.73a4.875 4.875 0 016.884 6.865zM6.431 7.037a3.375 3.375 0 000 4.773L12 17.38l5.569-5.569a3.375 3.375 0 10-4.773-4.773L9.613 10.22l-1.06-1.062 2.371-2.372a3.375 3.375 0 00-4.492.25v.001z"></path></svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="a9yw05ulvcgbrz7vxj167kdzrc1dh4kr" class="crayons-icon crayons-icon reaction-icon--like reaction-icon reacted"><title id="a9yw05ulvcgbrz7vxj167kdzrc1dh4kr">Like comment: </title>
|
||
<path d="M5.116 12.595a4.875 4.875 0 015.56-7.68h-.002L7.493 8.098l1.06 1.061 3.181-3.182a4.875 4.875 0 016.895 6.894L12 19.5l-6.894-6.894.01-.01z"></path>
|
||
</svg>
|
||
|
||
<span class="reactions-count">2</span><span class="reactions-label hidden m:inline-block"> likes</span>
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Like
|
||
</span>
|
||
</button>
|
||
|
||
<span class="favorite-control-root" data-favorite-control="true" data-variant="comment" data-favoritable-type="Comment" data-favoritable-id="745374" data-favoritable-user-id="38578" data-favorited="false" data-label-favorite="Pick as gem" data-label-favorited="Picked as gem" data-label-favorited-by-you="Picked as gem by you" data-modal-title="Gem Picked!" data-modal-body="You've picked this as a community gem!" data-modal-remaining-zero="You have no more gems left to give out today." data-modal-remaining-one="You have 1 more gem left to give out today." data-modal-remaining-other="You have %{count} more gems left to give out today." data-modal-remaining-unlimited="You have unlimited gems to give out." data-modal-close="Got it" data-modal-exhausted-title="Out of Gems" data-modal-exhausted-body="You have no more gems left to give out today. Your allowance will refresh soon!" data-modal-exhausted-close="Got it"></span>
|
||
|
||
<button
|
||
class="actions crayons-btn crayons-btn--ghost crayons-btn--s crayons-btn--icon-left toggle-reply-form mr-1 inline-flex"
|
||
data-comment-id="745374"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1gag6"
|
||
data-tracking-name="comment_reply_button"
|
||
data-testid="reply-button-745374"
|
||
rel="nofollow">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="atvw8nwbcxyn8g83eco4hfru8fk3j8ix" class="crayons-icon reaction-icon not-reacted"><title id="atvw8nwbcxyn8g83eco4hfru8fk3j8ix">Comment button</title><path d="M10.5 5h3a6 6 0 110 12v2.625c-3.75-1.5-9-3.75-9-8.625a6 6 0 016-6zM12 15.5h1.5a4.501 4.501 0 001.722-8.657A4.5 4.5 0 0013.5 6.5h-3A4.5 4.5 0 006 11c0 2.707 1.846 4.475 6 6.36V15.5z"></path></svg>
|
||
|
||
<span class="hidden m:inline-block">Reply</span>
|
||
</button>
|
||
|
||
</footer>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
</details>
|
||
|
||
|
||
</div>
|
||
</details>
|
||
|
||
|
||
</div>
|
||
</details>
|
||
|
||
<details class="comment-wrapper js-comment-wrapper comment-wrapper--deep-0
|
||
root
|
||
" open>
|
||
<summary aria-label="Toggle this comment (and replies)" data-tracking-name="expand_comment_toggle">
|
||
<span class="m:mx-1 inline-block align-middle">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="acoeq8udda3uzc57l06g7rr6tsrksqvx" class="crayons-icon expanded"><title id="acoeq8udda3uzc57l06g7rr6tsrksqvx">Collapse</title>
|
||
<path d="M12 10.677L8 6.935 9 6l3 2.807L15 6l1 .935-4 3.742zm0 4.517L9 18l-1-.935 4-3.742 4 3.742-1 .934-3-2.805z"></path>
|
||
</svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="a32hxpxguoqe9cknjgzbzjtexa42uepf" class="crayons-icon collapsed"><title id="a32hxpxguoqe9cknjgzbzjtexa42uepf">Expand</title>
|
||
<path d="M12 18l-4-3.771 1-.943 3 2.829 3-2.829 1 .943L12 18zm0-10.115l-3 2.829-1-.943L12 6l4 3.771-1 .942-3-2.828z"></path>
|
||
</svg>
|
||
|
||
</span>
|
||
<span class="js-collapse-comment-content inline-block align-middle"></span>
|
||
</summary>
|
||
<div
|
||
id="comment-node-746616"
|
||
class="
|
||
comment single-comment-node
|
||
|
||
root
|
||
comment--deep-0
|
||
|
||
"
|
||
data-comment-id="746616"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1gcc0"
|
||
data-comment-author-id="707"
|
||
data-content-user-id="707">
|
||
<a name="comment-1gcc0" style="position: absolute; top: -8px;"> </a>
|
||
|
||
<div class="comment__inner">
|
||
<a href="https://dev.to/joshcheek" class="shrink-0 crayons-avatar m:crayons-avatar--l mt-4 m:mt-3">
|
||
<img class="crayons-avatar__image" width="32" height="32" src="https://media2.dev.to/dynamic/image/width=50,height=50,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F707%2F77495.jpeg" alt="joshcheek profile image" loading="lazy" />
|
||
</a>
|
||
|
||
|
||
<div class="inner-comment comment__details">
|
||
<div class="comment__content crayons-card">
|
||
|
||
|
||
|
||
<div class="comment__header">
|
||
<a href="https://dev.to/joshcheek" class="crayons-link crayons-link--secondary flex items-center fw-medium m:hidden">
|
||
<span class="js-comment-username">
|
||
Josh Cheek
|
||
</span>
|
||
</a>
|
||
<div class="profile-preview-card relative mb-4 s:mb-0 fw-medium hidden m:block">
|
||
<button id="comment-profile-preview-trigger-746616" aria-controls="comment-profile-preview-content-746616" class="profile-preview-card__trigger p-1 -my-1 -ml-1 crayons-btn crayons-btn--ghost" aria-label="Josh Cheek profile details">
|
||
Josh Cheek
|
||
|
||
|
||
</button>
|
||
<div id="comment-profile-preview-content-746616" class="profile-preview-card__content p-4 pt-0 branded-7 crayons-dropdown" style="--card-color: #002a9e; border-top-color: var(--card-color);" data-testid="profile-preview-card" data-repositioning-dropdown="true">
|
||
<div class="gap-4 grid">
|
||
<div class="-mt-4">
|
||
<a href="/joshcheek" class="flex">
|
||
<span class="crayons-avatar crayons-avatar--xl mr-2 shrink-0">
|
||
<img src="https://media2.dev.to/dynamic/image/width=90,height=90,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F707%2F77495.jpeg" class="crayons-avatar__image" alt="" loading="lazy" />
|
||
</span>
|
||
<span class="crayons-link crayons-subtitle-2 mt-5">
|
||
Josh Cheek
|
||
|
||
|
||
</span>
|
||
</a>
|
||
</div>
|
||
|
||
<div class="print-hidden">
|
||
<button name="button" type="button" data-info="{"className":"User","style":"","id":707,"name":"Josh Cheek"}" class="crayons-btn follow-action-button whitespace-nowrap w-100 follow-user" aria-label="Follow user: Josh Cheek" aria-pressed="false">Follow</button>
|
||
</div>
|
||
|
||
<div class="user-metadata-details">
|
||
<ul class="user-metadata-details-inner">
|
||
<li>
|
||
<div class="key">
|
||
Joined
|
||
</div>
|
||
<div class="value">
|
||
<time datetime="2016-12-12T06:22:36Z" class="date">Dec 12, 2016</time>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
<span class="color-base-30 px-2 m:pl-0" role="presentation">•</span>
|
||
|
||
<a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1gcc0" class="comment-date crayons-link crayons-link--secondary fs-s">
|
||
<time datetime="2021-07-18T02:16:50Z" class=date-short-year>
|
||
Jul 18 '21
|
||
</time>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
<div class="comment__dropdown" data-tracking-name="comment_dropdown">
|
||
<button id="comment-dropdown-trigger-746616" aria-controls="comment-dropdown-746616" aria-expanded="false"
|
||
class="dropbtn comment__dropdown-trigger crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon "
|
||
aria-label="Toggle dropdown menu" aria-haspopup="true">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="aspsx3dvyybh8148qd5e5r161u988iag" class="crayons-icon pointer-events-none"><title id="aspsx3dvyybh8148qd5e5r161u988iag">Dropdown menu</title>
|
||
<path fill-rule="evenodd" clip-rule="evenodd" d="M8.25 12a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm5.25 0a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm3.75 1.5a1.5 1.5 0 100-3 1.5 1.5 0 000 3z"></path>
|
||
</svg>
|
||
|
||
</button>
|
||
<div id="comment-dropdown-746616" class="crayons-dropdown right-1 s:right-0 s:left-auto fs-base dropdown">
|
||
<ul class="m-0">
|
||
<li><a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1gcc0" class="crayons-link crayons-link--block permalink-copybtn" aria-label="Copy link to Josh Cheek's comment" data-no-instant>Copy link</a></li>
|
||
<li class="comment-actions hidden" data-user-id="707" data-action="settings-button" data-path="https://dev.to/joshcheek/comment/1gcc0/settings" aria-label="Go to Josh Cheek's comment settings"></li>
|
||
<li class="comment-actions hidden" data-action="hide-button" data-commentable-user-id="119473" data-user-id="707">
|
||
<button
|
||
class="flex justify-between crayons-link crayons-link--block w-100 bg-transparent border-0 hide-comment"
|
||
data-hide-type="hide"
|
||
data-comment-id="746616"
|
||
data-comment-url="https://dev.to/joshcheek/comment/1gcc0"
|
||
aria-label="Hide Josh Cheek's comment">
|
||
Hide
|
||
</button>
|
||
</li>
|
||
<li class="mod-actions hidden mod-actions-comment-button" data-path="https://dev.to/joshcheek/comment/1gcc0/mod" aria-label="Moderate Josh Cheek's comment"></li>
|
||
<li class="report-abuse-link-wrapper" data-path="/report-abuse?url=https://dev.to/joshcheek/comment/1gcc0" aria-label="Report Josh Cheek's comment as abusive or violating our code of conduct and/or terms and conditions"></li>
|
||
<li class="current-user-actions"></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<div
|
||
class="
|
||
comment__body
|
||
text-styles
|
||
text-styles--secondary
|
||
body
|
||
|
||
|
||
">
|
||
<p>Looks like you can do <code>redis.keys('rpush:notifications:*')</code> in place of the first script. It should be much quicker, because it doesn't send anything over the wire which doesn't match the pattern. If you want to do it in a paginated way, you could do: <code>r.scan_each(match: 'rpush:notifications:*') { |key| }</code> which basically does what your script did, except that the <code>match</code> is executed on the server, so it doesn't send nonmatching keys over the wire. You can also pass it a <code>count</code> option, but in my experiments, that was always slower (this will tell redis how many keys you want it to scan).</p>
|
||
|
||
<p>Locally I added 1.2 million matching keys and 1.2 million nonmatching keys, and then played around with it for a bit:<br>
|
||
</p>
|
||
|
||
<div class="highlight js-code-highlight">
|
||
<pre class="highlight ruby"><code><span class="k">def</span> <span class="nf">time</span>
|
||
<span class="n">start</span> <span class="o">=</span> <span class="no">Time</span><span class="p">.</span><span class="nf">new</span>
|
||
<span class="k">yield</span><span class="p">.</span><span class="nf">tap</span> <span class="p">{</span> <span class="vg">$stderr</span><span class="p">.</span><span class="nf">printf</span> <span class="s2">"%.2f seconds</span><span class="se">\n</span><span class="s2">"</span><span class="p">,</span> <span class="no">Time</span><span class="p">.</span><span class="nf">now</span><span class="o">-</span><span class="n">start</span> <span class="p">}</span>
|
||
<span class="k">end</span>
|
||
|
||
<span class="n">r</span> <span class="o">=</span> <span class="no">Redis</span><span class="p">.</span><span class="nf">new</span>
|
||
|
||
<span class="c1"># Seed it with records</span>
|
||
<span class="mi">1_200_000</span><span class="p">.</span><span class="nf">times</span><span class="p">.</span><span class="nf">each</span> <span class="k">do</span> <span class="o">|</span><span class="n">i</span><span class="o">|</span>
|
||
<span class="n">r</span><span class="p">.</span><span class="nf">set</span> <span class="s2">"rpush:notifications:</span><span class="si">#{</span><span class="n">i</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> <span class="n">i</span>
|
||
<span class="n">r</span><span class="p">.</span><span class="nf">set</span> <span class="s2">"other:thing:</span><span class="si">#{</span><span class="n">i</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> <span class="n">i</span>
|
||
<span class="k">end</span>
|
||
|
||
<span class="c1"># Check total number of keys</span>
|
||
<span class="n">r</span><span class="p">.</span><span class="nf">dbsize</span> <span class="c1"># => 1200014</span>
|
||
|
||
<span class="c1"># Check how many match our pattern (this is a lua script that calls the `keys</span>
|
||
<span class="c1"># command on the redis server, and then returns its size) It's much faster</span>
|
||
<span class="c1"># just b/c it doesn't need to ship all the keys over the wire</span>
|
||
<span class="n">r</span><span class="p">.</span><span class="nf">eval</span> <span class="s2">"return #redis.call('keys', 'rpush:notifications:*')"</span><span class="p">,</span> <span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="c1"># => 1200000</span>
|
||
|
||
<span class="c1"># Get the list of keys using https://redis.io/commands/keys</span>
|
||
<span class="n">keys</span> <span class="o">=</span> <span class="n">time</span> <span class="p">{</span> <span class="n">r</span><span class="p">.</span><span class="nf">keys</span><span class="p">(</span><span class="s1">'rpush:notifications:*'</span><span class="p">)</span> <span class="p">}</span> <span class="c1"># 7.77 seconds</span>
|
||
<span class="n">keys</span><span class="p">.</span><span class="nf">size</span> <span class="c1"># => 1200000</span>
|
||
|
||
<span class="c1"># Iterate over each key with successive scans (here we just increment a counter)</span>
|
||
<span class="n">time</span> <span class="p">{</span> <span class="n">r</span><span class="p">.</span><span class="nf">scan_each</span><span class="p">(</span><span class="ss">match: </span><span class="s1">'rpush:notifications:*'</span><span class="p">).</span><span class="nf">count</span> <span class="p">}</span> <span class="c1"># 19.16 seconds</span>
|
||
<span class="c1"># => 1200000</span>
|
||
|
||
<span class="c1"># Delete the keys in batches of 10k</span>
|
||
<span class="n">time</span> <span class="p">{</span> <span class="n">r</span><span class="p">.</span><span class="nf">keys</span><span class="p">(</span><span class="s1">'rpush:notifications:*'</span><span class="p">).</span><span class="nf">each_slice</span><span class="p">(</span><span class="mi">10_000</span><span class="p">)</span> <span class="p">{</span> <span class="o">|</span><span class="n">s</span><span class="o">|</span> <span class="nb">p</span> <span class="n">r</span><span class="p">.</span><span class="nf">del</span> <span class="n">s</span> <span class="p">}</span> <span class="p">}</span> <span class="c1"># 32.90s</span>
|
||
|
||
<span class="c1"># They're deleted</span>
|
||
<span class="n">time</span> <span class="p">{</span> <span class="n">r</span><span class="p">.</span><span class="nf">keys</span><span class="p">(</span><span class="s1">'rpush:notifications:*'</span><span class="p">).</span><span class="nf">size</span> <span class="p">}</span> <span class="c1"># 0.57s</span>
|
||
<span class="c1"># => 0</span>
|
||
</code></pre>
|
||
<div class="highlight__panel js-actions-panel">
|
||
<div class="highlight__panel-action js-fullscreen-code-action">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="20px" height="20px" viewbox="0 0 24 24" class="highlight-action crayons-icon highlight-action--fullscreen-on"><title>Enter fullscreen mode</title>
|
||
<path d="M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z"></path>
|
||
</svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="20px" height="20px" viewbox="0 0 24 24" class="highlight-action crayons-icon highlight-action--fullscreen-off"><title>Exit fullscreen mode</title>
|
||
<path d="M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z"></path>
|
||
</svg>
|
||
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
<script>
|
||
</script>
|
||
|
||
<footer class="comment__footer print-hidden">
|
||
<button
|
||
class="crayons-tooltip__activator relative crayons-btn crayons-btn--ghost crayons-btn--icon-left crayons-btn--s mr-1 reaction-like inline-flex reaction-button"
|
||
id="button-for-comment-746616"
|
||
data-comment-id="746616"
|
||
aria-label="like"
|
||
data-tracking-name="comment_heart_button">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="a6dsjkq676xjt365maaaa4s3lz7y2o0z" class="crayons-icon reaction-icon not-reacted"><title id="a6dsjkq676xjt365maaaa4s3lz7y2o0z">Like comment: </title><path d="M18.884 12.595l.01.011L12 19.5l-6.894-6.894.01-.01A4.875 4.875 0 0112 5.73a4.875 4.875 0 016.884 6.865zM6.431 7.037a3.375 3.375 0 000 4.773L12 17.38l5.569-5.569a3.375 3.375 0 10-4.773-4.773L9.613 10.22l-1.06-1.062 2.371-2.372a3.375 3.375 0 00-4.492.25v.001z"></path></svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="asux0yjje2fm5609uqv4id5cch9tnze5" class="crayons-icon crayons-icon reaction-icon--like reaction-icon reacted"><title id="asux0yjje2fm5609uqv4id5cch9tnze5">Like comment: </title>
|
||
<path d="M5.116 12.595a4.875 4.875 0 015.56-7.68h-.002L7.493 8.098l1.06 1.061 3.181-3.182a4.875 4.875 0 016.895 6.894L12 19.5l-6.894-6.894.01-.01z"></path>
|
||
</svg>
|
||
|
||
<span class="reactions-count">3</span><span class="reactions-label hidden m:inline-block"> likes</span>
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Like
|
||
</span>
|
||
</button>
|
||
|
||
<span class="favorite-control-root" data-favorite-control="true" data-variant="comment" data-favoritable-type="Comment" data-favoritable-id="746616" data-favoritable-user-id="707" data-favorited="false" data-label-favorite="Pick as gem" data-label-favorited="Picked as gem" data-label-favorited-by-you="Picked as gem by you" data-modal-title="Gem Picked!" data-modal-body="You've picked this as a community gem!" data-modal-remaining-zero="You have no more gems left to give out today." data-modal-remaining-one="You have 1 more gem left to give out today." data-modal-remaining-other="You have %{count} more gems left to give out today." data-modal-remaining-unlimited="You have unlimited gems to give out." data-modal-close="Got it" data-modal-exhausted-title="Out of Gems" data-modal-exhausted-body="You have no more gems left to give out today. Your allowance will refresh soon!" data-modal-exhausted-close="Got it"></span>
|
||
|
||
<button
|
||
class="actions crayons-btn crayons-btn--ghost crayons-btn--s crayons-btn--icon-left toggle-reply-form mr-1 inline-flex"
|
||
data-comment-id="746616"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1gcc0"
|
||
data-tracking-name="comment_reply_button"
|
||
data-testid="reply-button-746616"
|
||
rel="nofollow">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="al9azpukuzufow0gid9xyws0x00jkbac" class="crayons-icon reaction-icon not-reacted"><title id="al9azpukuzufow0gid9xyws0x00jkbac">Comment button</title><path d="M10.5 5h3a6 6 0 110 12v2.625c-3.75-1.5-9-3.75-9-8.625a6 6 0 016-6zM12 15.5h1.5a4.501 4.501 0 001.722-8.657A4.5 4.5 0 0013.5 6.5h-3A4.5 4.5 0 006 11c0 2.707 1.846 4.475 6 6.36V15.5z"></path></svg>
|
||
|
||
<span class="hidden m:inline-block">Reply</span>
|
||
</button>
|
||
|
||
</footer>
|
||
|
||
</div>
|
||
</div>
|
||
<details class="comment-wrapper js-comment-wrapper comment-wrapper--deep-1
|
||
child
|
||
" open>
|
||
<summary aria-label="Toggle this comment (and replies)" data-tracking-name="expand_comment_toggle">
|
||
<span class="mx-0 inline-block align-middle">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="abo16iadjub9lcp1lq6xrovyqcnat1d" class="crayons-icon expanded"><title id="abo16iadjub9lcp1lq6xrovyqcnat1d">Collapse</title>
|
||
<path d="M12 10.677L8 6.935 9 6l3 2.807L15 6l1 .935-4 3.742zm0 4.517L9 18l-1-.935 4-3.742 4 3.742-1 .934-3-2.805z"></path>
|
||
</svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="af0pmn0sdqfe7fhqgsefd9ua52hbcrae" class="crayons-icon collapsed"><title id="af0pmn0sdqfe7fhqgsefd9ua52hbcrae">Expand</title>
|
||
<path d="M12 18l-4-3.771 1-.943 3 2.829 3-2.829 1 .943L12 18zm0-10.115l-3 2.829-1-.943L12 6l4 3.771-1 .942-3-2.828z"></path>
|
||
</svg>
|
||
|
||
</span>
|
||
<span class="js-collapse-comment-content inline-block align-middle"></span>
|
||
</summary>
|
||
<div
|
||
id="comment-node-748108"
|
||
class="
|
||
comment single-comment-node
|
||
|
||
child
|
||
comment--deep-1
|
||
|
||
"
|
||
data-comment-id="748108"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1geha"
|
||
data-comment-author-id="17908"
|
||
data-content-user-id="17908">
|
||
<a name="comment-1geha" style="position: absolute; top: -8px;"> </a>
|
||
|
||
<div class="comment__inner">
|
||
<a href="https://dev.to/jgaskins" class="shrink-0 crayons-avatar mt-4">
|
||
<img class="crayons-avatar__image" width="32" height="32" src="https://media2.dev.to/dynamic/image/width=50,height=50,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F17908%2Fdf4d4ab2-af2a-4eaa-8b31-311af5306876.jpeg" alt="jgaskins profile image" loading="lazy" />
|
||
</a>
|
||
|
||
|
||
<div class="inner-comment comment__details">
|
||
<div class="comment__content crayons-card">
|
||
|
||
|
||
|
||
<div class="comment__header">
|
||
<a href="https://dev.to/jgaskins" class="crayons-link crayons-link--secondary flex items-center fw-medium m:hidden">
|
||
<span class="js-comment-username">
|
||
Jamie Gaskins
|
||
</span>
|
||
</a>
|
||
<div class="profile-preview-card relative mb-4 s:mb-0 fw-medium hidden m:block">
|
||
<button id="comment-profile-preview-trigger-748108" aria-controls="comment-profile-preview-content-748108" class="profile-preview-card__trigger p-1 -my-1 -ml-1 crayons-btn crayons-btn--ghost" aria-label="Jamie Gaskins profile details">
|
||
Jamie Gaskins
|
||
|
||
|
||
</button>
|
||
<div id="comment-profile-preview-content-748108" class="profile-preview-card__content p-4 pt-0 branded-7 crayons-dropdown" style="--card-color: #131313; border-top-color: var(--card-color);" data-testid="profile-preview-card" data-repositioning-dropdown="true">
|
||
<div class="gap-4 grid">
|
||
<div class="-mt-4">
|
||
<a href="/jgaskins" class="flex">
|
||
<span class="crayons-avatar crayons-avatar--xl mr-2 shrink-0">
|
||
<img src="https://media2.dev.to/dynamic/image/width=90,height=90,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F17908%2Fdf4d4ab2-af2a-4eaa-8b31-311af5306876.jpeg" class="crayons-avatar__image" alt="" loading="lazy" />
|
||
</span>
|
||
<span class="crayons-link crayons-subtitle-2 mt-5">
|
||
Jamie Gaskins
|
||
|
||
|
||
</span>
|
||
</a>
|
||
</div>
|
||
|
||
<div class="print-hidden">
|
||
<button name="button" type="button" data-info="{"className":"User","style":"","id":17908,"name":"Jamie Gaskins"}" class="crayons-btn follow-action-button whitespace-nowrap w-100 follow-user" aria-label="Follow user: Jamie Gaskins" aria-pressed="false">Follow</button>
|
||
</div>
|
||
<div class="color-base-70">
|
||
I put the bits into the computers and they give me different bits back. I'm still working out the details of what happens in between.
|
||
</div>
|
||
|
||
<div class="user-metadata-details">
|
||
<ul class="user-metadata-details-inner">
|
||
<li>
|
||
<div class="key">
|
||
Location
|
||
</div>
|
||
<div class="value">
|
||
Baltimore, MD
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Work
|
||
</div>
|
||
<div class="value">
|
||
Shopify
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Joined
|
||
</div>
|
||
<div class="value">
|
||
<time datetime="2017-05-02T11:56:42Z" class="date">May 2, 2017</time>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
<span class="color-base-30 px-2 m:pl-0" role="presentation">•</span>
|
||
|
||
<a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1geha" class="comment-date crayons-link crayons-link--secondary fs-s">
|
||
<time datetime="2021-07-21T02:04:25Z" class=date-short-year>
|
||
Jul 21 '21
|
||
</time>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
<div class="comment__dropdown" data-tracking-name="comment_dropdown">
|
||
<button id="comment-dropdown-trigger-748108" aria-controls="comment-dropdown-748108" aria-expanded="false"
|
||
class="dropbtn comment__dropdown-trigger crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon "
|
||
aria-label="Toggle dropdown menu" aria-haspopup="true">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="a60oburssujr58ylis9dee04uuw3izat" class="crayons-icon pointer-events-none"><title id="a60oburssujr58ylis9dee04uuw3izat">Dropdown menu</title>
|
||
<path fill-rule="evenodd" clip-rule="evenodd" d="M8.25 12a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm5.25 0a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm3.75 1.5a1.5 1.5 0 100-3 1.5 1.5 0 000 3z"></path>
|
||
</svg>
|
||
|
||
</button>
|
||
<div id="comment-dropdown-748108" class="crayons-dropdown right-1 s:right-0 s:left-auto fs-base dropdown">
|
||
<ul class="m-0">
|
||
<li><a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1geha" class="crayons-link crayons-link--block permalink-copybtn" aria-label="Copy link to Jamie Gaskins's comment" data-no-instant>Copy link</a></li>
|
||
<li class="comment-actions hidden" data-user-id="17908" data-action="settings-button" data-path="https://dev.to/jgaskins/comment/1geha/settings" aria-label="Go to Jamie Gaskins's comment settings"></li>
|
||
<li class="comment-actions hidden" data-action="hide-button" data-commentable-user-id="119473" data-user-id="17908">
|
||
<button
|
||
class="flex justify-between crayons-link crayons-link--block w-100 bg-transparent border-0 hide-comment"
|
||
data-hide-type="hide"
|
||
data-comment-id="748108"
|
||
data-comment-url="https://dev.to/jgaskins/comment/1geha"
|
||
aria-label="Hide Jamie Gaskins's comment">
|
||
Hide
|
||
</button>
|
||
</li>
|
||
<li class="mod-actions hidden mod-actions-comment-button" data-path="https://dev.to/jgaskins/comment/1geha/mod" aria-label="Moderate Jamie Gaskins's comment"></li>
|
||
<li class="report-abuse-link-wrapper" data-path="/report-abuse?url=https://dev.to/jgaskins/comment/1geha" aria-label="Report Jamie Gaskins's comment as abusive or violating our code of conduct and/or terms and conditions"></li>
|
||
<li class="current-user-actions"></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<div
|
||
class="
|
||
comment__body
|
||
text-styles
|
||
text-styles--secondary
|
||
body
|
||
|
||
|
||
">
|
||
<p>I like that you enjoy optimizing things and golfing code down to fit in tweets. 🙂 However, like many optimizations, this one is more complex than simple latency. <code>KEYS</code> would indeed have had lower latency, but it locks the Redis event loop for the duration, so we avoid that command wherever possible. To put it into perspective, your <code>KEYS</code> query took 7 seconds in your benchmark (we had almost 5M keys in Redis, so it would've actually taken longer, but I'll work with your numbers for the moment). In that amount of time we normally execute over 10,000 Redis queries, so running <code>KEYS</code> would introduce a pretty sizable head-of-line-blocking issue. On top of that, every HTTP request that makes it past Fastly to our backend hits this Redis instance at least once, so blocking Redis means blocking all HTTP requests. Since we don't want to do that, Redis needs to be <em>available af</em>. We also don't allow other <code>O(n)</code> calls where <code>n</code> can ever be nontrivial for that reason, so nothing like <code>LRANGE my-list 0 -1</code>, either.</p>
|
||
|
||
<p>In fairness, it almost certainly doesn't lock the main Redis thread for the <em>full</em> amount of time it takes the Ruby app to receive the entire list of keys (we're using Redis 6, which <a href="http://antirez.com/news/132" rel="nofollow">does I/O in a separate thread</a>), but without any insight into how much of that time is spent in the main Redis event loop, we don't take chances. Since this issue impacted only a tiny fraction of our total traffic, we needed to continue optimizing for availability of Redis over shaving a few seconds off of the latency of the solution.</p>
|
||
|
||
<p>The code Molly posted was adapted from <a href="https://github.com/forem/forem/blob/94171f0cfbe16d9f320753ac6e67b0169ef586cc/app/workers/push_notifications/cleanup_worker.rb#L10-L18" rel="nofollow">this worker</a>. I didn't know about <code>Redis#scan_each</code>, though. That completely obviates our manual tracking of the cursor, which I <em>really</em> like.</p>
|
||
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
<script>
|
||
</script>
|
||
|
||
<footer class="comment__footer print-hidden">
|
||
<button
|
||
class="crayons-tooltip__activator relative crayons-btn crayons-btn--ghost crayons-btn--icon-left crayons-btn--s mr-1 reaction-like inline-flex reaction-button"
|
||
id="button-for-comment-748108"
|
||
data-comment-id="748108"
|
||
aria-label="like"
|
||
data-tracking-name="comment_heart_button">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="afodcceieiqlf5xd05p68ck5p6nzalgf" class="crayons-icon reaction-icon not-reacted"><title id="afodcceieiqlf5xd05p68ck5p6nzalgf">Like comment: </title><path d="M18.884 12.595l.01.011L12 19.5l-6.894-6.894.01-.01A4.875 4.875 0 0112 5.73a4.875 4.875 0 016.884 6.865zM6.431 7.037a3.375 3.375 0 000 4.773L12 17.38l5.569-5.569a3.375 3.375 0 10-4.773-4.773L9.613 10.22l-1.06-1.062 2.371-2.372a3.375 3.375 0 00-4.492.25v.001z"></path></svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="ap9lfetnx1vu3c6u931czseicr95329o" class="crayons-icon crayons-icon reaction-icon--like reaction-icon reacted"><title id="ap9lfetnx1vu3c6u931czseicr95329o">Like comment: </title>
|
||
<path d="M5.116 12.595a4.875 4.875 0 015.56-7.68h-.002L7.493 8.098l1.06 1.061 3.181-3.182a4.875 4.875 0 016.895 6.894L12 19.5l-6.894-6.894.01-.01z"></path>
|
||
</svg>
|
||
|
||
<span class="reactions-count">2</span><span class="reactions-label hidden m:inline-block"> likes</span>
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Like
|
||
</span>
|
||
</button>
|
||
|
||
<span class="favorite-control-root" data-favorite-control="true" data-variant="comment" data-favoritable-type="Comment" data-favoritable-id="748108" data-favoritable-user-id="17908" data-favorited="false" data-label-favorite="Pick as gem" data-label-favorited="Picked as gem" data-label-favorited-by-you="Picked as gem by you" data-modal-title="Gem Picked!" data-modal-body="You've picked this as a community gem!" data-modal-remaining-zero="You have no more gems left to give out today." data-modal-remaining-one="You have 1 more gem left to give out today." data-modal-remaining-other="You have %{count} more gems left to give out today." data-modal-remaining-unlimited="You have unlimited gems to give out." data-modal-close="Got it" data-modal-exhausted-title="Out of Gems" data-modal-exhausted-body="You have no more gems left to give out today. Your allowance will refresh soon!" data-modal-exhausted-close="Got it"></span>
|
||
|
||
<button
|
||
class="actions crayons-btn crayons-btn--ghost crayons-btn--s crayons-btn--icon-left toggle-reply-form mr-1 inline-flex"
|
||
data-comment-id="748108"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1geha"
|
||
data-tracking-name="comment_reply_button"
|
||
data-testid="reply-button-748108"
|
||
rel="nofollow">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="ar2gkydt4sio3zuqavihycjolu8c3rvi" class="crayons-icon reaction-icon not-reacted"><title id="ar2gkydt4sio3zuqavihycjolu8c3rvi">Comment button</title><path d="M10.5 5h3a6 6 0 110 12v2.625c-3.75-1.5-9-3.75-9-8.625a6 6 0 016-6zM12 15.5h1.5a4.501 4.501 0 001.722-8.657A4.5 4.5 0 0013.5 6.5h-3A4.5 4.5 0 006 11c0 2.707 1.846 4.475 6 6.36V15.5z"></path></svg>
|
||
|
||
<span class="hidden m:inline-block">Reply</span>
|
||
</button>
|
||
|
||
</footer>
|
||
|
||
</div>
|
||
</div>
|
||
<details class="comment-wrapper js-comment-wrapper comment-wrapper--deep-2
|
||
child
|
||
" open>
|
||
<summary aria-label="Toggle this comment (and replies)" data-tracking-name="expand_comment_toggle">
|
||
<span class="mx-0 inline-block align-middle">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="a1763guvt6myatq3a6hjo8vyurvdl2dn" class="crayons-icon expanded"><title id="a1763guvt6myatq3a6hjo8vyurvdl2dn">Collapse</title>
|
||
<path d="M12 10.677L8 6.935 9 6l3 2.807L15 6l1 .935-4 3.742zm0 4.517L9 18l-1-.935 4-3.742 4 3.742-1 .934-3-2.805z"></path>
|
||
</svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="agel2jj1f4cc333g8u0b4wttv3h7aq1g" class="crayons-icon collapsed"><title id="agel2jj1f4cc333g8u0b4wttv3h7aq1g">Expand</title>
|
||
<path d="M12 18l-4-3.771 1-.943 3 2.829 3-2.829 1 .943L12 18zm0-10.115l-3 2.829-1-.943L12 6l4 3.771-1 .942-3-2.828z"></path>
|
||
</svg>
|
||
|
||
</span>
|
||
<span class="js-collapse-comment-content inline-block align-middle"></span>
|
||
</summary>
|
||
<div
|
||
id="comment-node-748769"
|
||
class="
|
||
comment single-comment-node
|
||
|
||
child
|
||
comment--deep-2
|
||
|
||
"
|
||
data-comment-id="748769"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1gfgl"
|
||
data-comment-author-id="707"
|
||
data-content-user-id="707">
|
||
<a name="comment-1gfgl" style="position: absolute; top: -8px;"> </a>
|
||
|
||
<div class="comment__inner">
|
||
<a href="https://dev.to/joshcheek" class="shrink-0 crayons-avatar mt-4">
|
||
<img class="crayons-avatar__image" width="32" height="32" src="https://media2.dev.to/dynamic/image/width=50,height=50,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F707%2F77495.jpeg" alt="joshcheek profile image" loading="lazy" />
|
||
</a>
|
||
|
||
|
||
<div class="inner-comment comment__details">
|
||
<div class="comment__content crayons-card">
|
||
|
||
|
||
|
||
<div class="comment__header">
|
||
<a href="https://dev.to/joshcheek" class="crayons-link crayons-link--secondary flex items-center fw-medium m:hidden">
|
||
<span class="js-comment-username">
|
||
Josh Cheek
|
||
</span>
|
||
</a>
|
||
<div class="profile-preview-card relative mb-4 s:mb-0 fw-medium hidden m:block">
|
||
<button id="comment-profile-preview-trigger-748769" aria-controls="comment-profile-preview-content-748769" class="profile-preview-card__trigger p-1 -my-1 -ml-1 crayons-btn crayons-btn--ghost" aria-label="Josh Cheek profile details">
|
||
Josh Cheek
|
||
|
||
|
||
</button>
|
||
<div id="comment-profile-preview-content-748769" class="profile-preview-card__content p-4 pt-0 branded-7 crayons-dropdown" style="--card-color: #002a9e; border-top-color: var(--card-color);" data-testid="profile-preview-card" data-repositioning-dropdown="true">
|
||
<div class="gap-4 grid">
|
||
<div class="-mt-4">
|
||
<a href="/joshcheek" class="flex">
|
||
<span class="crayons-avatar crayons-avatar--xl mr-2 shrink-0">
|
||
<img src="https://media2.dev.to/dynamic/image/width=90,height=90,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F707%2F77495.jpeg" class="crayons-avatar__image" alt="" loading="lazy" />
|
||
</span>
|
||
<span class="crayons-link crayons-subtitle-2 mt-5">
|
||
Josh Cheek
|
||
|
||
|
||
</span>
|
||
</a>
|
||
</div>
|
||
|
||
<div class="print-hidden">
|
||
<button name="button" type="button" data-info="{"className":"User","style":"","id":707,"name":"Josh Cheek"}" class="crayons-btn follow-action-button whitespace-nowrap w-100 follow-user" aria-label="Follow user: Josh Cheek" aria-pressed="false">Follow</button>
|
||
</div>
|
||
|
||
<div class="user-metadata-details">
|
||
<ul class="user-metadata-details-inner">
|
||
<li>
|
||
<div class="key">
|
||
Joined
|
||
</div>
|
||
<div class="value">
|
||
<time datetime="2016-12-12T06:22:36Z" class="date">Dec 12, 2016</time>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
<span class="color-base-30 px-2 m:pl-0" role="presentation">•</span>
|
||
|
||
<a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1gfgl" class="comment-date crayons-link crayons-link--secondary fs-s">
|
||
<time datetime="2021-07-22T08:57:32Z" class=date-short-year>
|
||
Jul 22 '21
|
||
</time>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
<div class="comment__dropdown" data-tracking-name="comment_dropdown">
|
||
<button id="comment-dropdown-trigger-748769" aria-controls="comment-dropdown-748769" aria-expanded="false"
|
||
class="dropbtn comment__dropdown-trigger crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon "
|
||
aria-label="Toggle dropdown menu" aria-haspopup="true">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="amafu00tkiskodxbladm1alsl6qz2r7g" class="crayons-icon pointer-events-none"><title id="amafu00tkiskodxbladm1alsl6qz2r7g">Dropdown menu</title>
|
||
<path fill-rule="evenodd" clip-rule="evenodd" d="M8.25 12a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm5.25 0a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm3.75 1.5a1.5 1.5 0 100-3 1.5 1.5 0 000 3z"></path>
|
||
</svg>
|
||
|
||
</button>
|
||
<div id="comment-dropdown-748769" class="crayons-dropdown right-1 s:right-0 s:left-auto fs-base dropdown">
|
||
<ul class="m-0">
|
||
<li><a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1gfgl" class="crayons-link crayons-link--block permalink-copybtn" aria-label="Copy link to Josh Cheek's comment" data-no-instant>Copy link</a></li>
|
||
<li class="comment-actions hidden" data-user-id="707" data-action="settings-button" data-path="https://dev.to/joshcheek/comment/1gfgl/settings" aria-label="Go to Josh Cheek's comment settings"></li>
|
||
<li class="comment-actions hidden" data-action="hide-button" data-commentable-user-id="119473" data-user-id="707">
|
||
<button
|
||
class="flex justify-between crayons-link crayons-link--block w-100 bg-transparent border-0 hide-comment"
|
||
data-hide-type="hide"
|
||
data-comment-id="748769"
|
||
data-comment-url="https://dev.to/joshcheek/comment/1gfgl"
|
||
aria-label="Hide Josh Cheek's comment">
|
||
Hide
|
||
</button>
|
||
</li>
|
||
<li class="mod-actions hidden mod-actions-comment-button" data-path="https://dev.to/joshcheek/comment/1gfgl/mod" aria-label="Moderate Josh Cheek's comment"></li>
|
||
<li class="report-abuse-link-wrapper" data-path="/report-abuse?url=https://dev.to/joshcheek/comment/1gfgl" aria-label="Report Josh Cheek's comment as abusive or violating our code of conduct and/or terms and conditions"></li>
|
||
<li class="current-user-actions"></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<div
|
||
class="
|
||
comment__body
|
||
text-styles
|
||
text-styles--secondary
|
||
body
|
||
|
||
|
||
">
|
||
<blockquote>
|
||
<p>I like that you enjoy optimizing things and golfing code down to fit in tweets.</p>
|
||
</blockquote>
|
||
|
||
<p>Thanks ^_^</p>
|
||
|
||
<blockquote>
|
||
<p>[comment about keys blocking]</p>
|
||
</blockquote>
|
||
|
||
<p>I agree with the analysis and conclusions here.</p>
|
||
|
||
<blockquote>
|
||
<p>The code Molly posted was adapted from <a href="https://github.com/forem/forem/blob/94171f0cfbe16d9f320753ac6e67b0169ef586cc/app/workers/push_notifications/cleanup_worker.rb#L10-L18" rel="nofollow">this worker</a>. I didn't know about Redis#scan_each, though. That completely obviates our manual tracking of the cursor, which I really like.</p>
|
||
</blockquote>
|
||
|
||
<p>Aye, +1 for <code>scan_each</code>. Also, note that on the last iteration, when the cursor is <code>"0"</code>, there can be keys that were returned with the cursor, which the worker won't expire, b/c it won't execute the loop body on that last response.</p>
|
||
|
||
<p>eg:<br>
|
||
</p>
|
||
|
||
<div class="highlight js-code-highlight">
|
||
<pre class="highlight shell"><code><span class="nv">$ </span>ruby <span class="nt">-r</span> redis <span class="nt">-e</span> <span class="s1">'
|
||
redis = Redis.new
|
||
|
||
p count_from_keys: redis.eval(%(return #redis.call("keys", "rpush:notifications:*")), [0])
|
||
|
||
cursor = count = 0
|
||
until (cursor, keys = redis.scan(cursor, match: "rpush:notifications:*")).first.to_i.zero?
|
||
count += keys.size
|
||
end
|
||
p count_from_loop: count, final_key_size: keys.size
|
||
'</span>
|
||
<span class="o">{</span>:count_from_keys<span class="o">=></span>1200000<span class="o">}</span>
|
||
<span class="o">{</span>:count_from_loop<span class="o">=></span>1199998, :final_key_size<span class="o">=></span>2<span class="o">}</span>
|
||
</code></pre>
|
||
<div class="highlight__panel js-actions-panel">
|
||
<div class="highlight__panel-action js-fullscreen-code-action">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="20px" height="20px" viewbox="0 0 24 24" class="highlight-action crayons-icon highlight-action--fullscreen-on"><title>Enter fullscreen mode</title>
|
||
<path d="M16 3h6v6h-2V5h-4V3zM2 3h6v2H4v4H2V3zm18 16v-4h2v6h-6v-2h4zM4 19h4v2H2v-6h2v4z"></path>
|
||
</svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="20px" height="20px" viewbox="0 0 24 24" class="highlight-action crayons-icon highlight-action--fullscreen-off"><title>Exit fullscreen mode</title>
|
||
<path d="M18 7h4v2h-6V3h2v4zM8 9H2V7h4V3h2v6zm10 8v4h-2v-6h6v2h-4zM8 15v6H6v-4H2v-2h6z"></path>
|
||
</svg>
|
||
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
<script>
|
||
</script>
|
||
|
||
<footer class="comment__footer print-hidden">
|
||
<button
|
||
class="crayons-tooltip__activator relative crayons-btn crayons-btn--ghost crayons-btn--icon-left crayons-btn--s mr-1 reaction-like inline-flex reaction-button"
|
||
id="button-for-comment-748769"
|
||
data-comment-id="748769"
|
||
aria-label="like"
|
||
data-tracking-name="comment_heart_button">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="al3cm7kkq7vor930ipqx40lmambwm7bu" class="crayons-icon reaction-icon not-reacted"><title id="al3cm7kkq7vor930ipqx40lmambwm7bu">Like comment: </title><path d="M18.884 12.595l.01.011L12 19.5l-6.894-6.894.01-.01A4.875 4.875 0 0112 5.73a4.875 4.875 0 016.884 6.865zM6.431 7.037a3.375 3.375 0 000 4.773L12 17.38l5.569-5.569a3.375 3.375 0 10-4.773-4.773L9.613 10.22l-1.06-1.062 2.371-2.372a3.375 3.375 0 00-4.492.25v.001z"></path></svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="a97uaavcdx22ahc68ursujw2jjn3ga55" class="crayons-icon crayons-icon reaction-icon--like reaction-icon reacted"><title id="a97uaavcdx22ahc68ursujw2jjn3ga55">Like comment: </title>
|
||
<path d="M5.116 12.595a4.875 4.875 0 015.56-7.68h-.002L7.493 8.098l1.06 1.061 3.181-3.182a4.875 4.875 0 016.895 6.894L12 19.5l-6.894-6.894.01-.01z"></path>
|
||
</svg>
|
||
|
||
<span class="reactions-count">2</span><span class="reactions-label hidden m:inline-block"> likes</span>
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Like
|
||
</span>
|
||
</button>
|
||
|
||
<span class="favorite-control-root" data-favorite-control="true" data-variant="comment" data-favoritable-type="Comment" data-favoritable-id="748769" data-favoritable-user-id="707" data-favorited="false" data-label-favorite="Pick as gem" data-label-favorited="Picked as gem" data-label-favorited-by-you="Picked as gem by you" data-modal-title="Gem Picked!" data-modal-body="You've picked this as a community gem!" data-modal-remaining-zero="You have no more gems left to give out today." data-modal-remaining-one="You have 1 more gem left to give out today." data-modal-remaining-other="You have %{count} more gems left to give out today." data-modal-remaining-unlimited="You have unlimited gems to give out." data-modal-close="Got it" data-modal-exhausted-title="Out of Gems" data-modal-exhausted-body="You have no more gems left to give out today. Your allowance will refresh soon!" data-modal-exhausted-close="Got it"></span>
|
||
|
||
<span class="fs-s inline-flex items-center fs-italic color-base-50 pl-1">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="alid5l3s5zyheqmbqhmtj37voa0rohth" class="crayons-icon"><title id="alid5l3s5zyheqmbqhmtj37voa0rohth">Thread</title>
|
||
<path d="M17 13l-5 6-5-6h3.125c0-3.314 2.798-6 6.25-6 .17 0 .34.006.506.02-1.787.904-3.006 2.705-3.006 4.78V13H17z"></path>
|
||
</svg>
|
||
|
||
Thread
|
||
</span>
|
||
|
||
</footer>
|
||
|
||
</div>
|
||
</div>
|
||
<div
|
||
id="comment-node-748895"
|
||
class="
|
||
comment single-comment-node
|
||
|
||
child
|
||
comment--deep-3
|
||
|
||
"
|
||
data-comment-id="748895"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1gflh"
|
||
data-comment-author-id="17908"
|
||
data-content-user-id="17908">
|
||
<a name="comment-1gflh" style="position: absolute; top: -8px;"> </a>
|
||
|
||
<div class="comment__inner">
|
||
<a href="https://dev.to/jgaskins" class="shrink-0 crayons-avatar mt-4">
|
||
<img class="crayons-avatar__image" width="32" height="32" src="https://media2.dev.to/dynamic/image/width=50,height=50,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F17908%2Fdf4d4ab2-af2a-4eaa-8b31-311af5306876.jpeg" alt="jgaskins profile image" loading="lazy" />
|
||
</a>
|
||
|
||
|
||
<div class="inner-comment comment__details">
|
||
<div class="comment__content crayons-card">
|
||
|
||
|
||
|
||
<div class="comment__header">
|
||
<a href="https://dev.to/jgaskins" class="crayons-link crayons-link--secondary flex items-center fw-medium m:hidden">
|
||
<span class="js-comment-username">
|
||
Jamie Gaskins
|
||
</span>
|
||
</a>
|
||
<div class="profile-preview-card relative mb-4 s:mb-0 fw-medium hidden m:block">
|
||
<button id="comment-profile-preview-trigger-748895" aria-controls="comment-profile-preview-content-748895" class="profile-preview-card__trigger p-1 -my-1 -ml-1 crayons-btn crayons-btn--ghost" aria-label="Jamie Gaskins profile details">
|
||
Jamie Gaskins
|
||
|
||
|
||
</button>
|
||
<div id="comment-profile-preview-content-748895" class="profile-preview-card__content p-4 pt-0 branded-7 crayons-dropdown" style="--card-color: #131313; border-top-color: var(--card-color);" data-testid="profile-preview-card" data-repositioning-dropdown="true">
|
||
<div class="gap-4 grid">
|
||
<div class="-mt-4">
|
||
<a href="/jgaskins" class="flex">
|
||
<span class="crayons-avatar crayons-avatar--xl mr-2 shrink-0">
|
||
<img src="https://media2.dev.to/dynamic/image/width=90,height=90,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F17908%2Fdf4d4ab2-af2a-4eaa-8b31-311af5306876.jpeg" class="crayons-avatar__image" alt="" loading="lazy" />
|
||
</span>
|
||
<span class="crayons-link crayons-subtitle-2 mt-5">
|
||
Jamie Gaskins
|
||
|
||
|
||
</span>
|
||
</a>
|
||
</div>
|
||
|
||
<div class="print-hidden">
|
||
<button name="button" type="button" data-info="{"className":"User","style":"","id":17908,"name":"Jamie Gaskins"}" class="crayons-btn follow-action-button whitespace-nowrap w-100 follow-user" aria-label="Follow user: Jamie Gaskins" aria-pressed="false">Follow</button>
|
||
</div>
|
||
<div class="color-base-70">
|
||
I put the bits into the computers and they give me different bits back. I'm still working out the details of what happens in between.
|
||
</div>
|
||
|
||
<div class="user-metadata-details">
|
||
<ul class="user-metadata-details-inner">
|
||
<li>
|
||
<div class="key">
|
||
Location
|
||
</div>
|
||
<div class="value">
|
||
Baltimore, MD
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Work
|
||
</div>
|
||
<div class="value">
|
||
Shopify
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Joined
|
||
</div>
|
||
<div class="value">
|
||
<time datetime="2017-05-02T11:56:42Z" class="date">May 2, 2017</time>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
<span class="color-base-30 px-2 m:pl-0" role="presentation">•</span>
|
||
|
||
<a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1gflh" class="comment-date crayons-link crayons-link--secondary fs-s">
|
||
<time datetime="2021-07-22T13:39:45Z" class=date-short-year>
|
||
Jul 22 '21
|
||
</time>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
<div class="comment__dropdown" data-tracking-name="comment_dropdown">
|
||
<button id="comment-dropdown-trigger-748895" aria-controls="comment-dropdown-748895" aria-expanded="false"
|
||
class="dropbtn comment__dropdown-trigger crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon "
|
||
aria-label="Toggle dropdown menu" aria-haspopup="true">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="ak1f7p590syvwo6sxkth57cbybn99mla" class="crayons-icon pointer-events-none"><title id="ak1f7p590syvwo6sxkth57cbybn99mla">Dropdown menu</title>
|
||
<path fill-rule="evenodd" clip-rule="evenodd" d="M8.25 12a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm5.25 0a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm3.75 1.5a1.5 1.5 0 100-3 1.5 1.5 0 000 3z"></path>
|
||
</svg>
|
||
|
||
</button>
|
||
<div id="comment-dropdown-748895" class="crayons-dropdown right-1 s:right-0 s:left-auto fs-base dropdown">
|
||
<ul class="m-0">
|
||
<li><a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1gflh" class="crayons-link crayons-link--block permalink-copybtn" aria-label="Copy link to Jamie Gaskins's comment" data-no-instant>Copy link</a></li>
|
||
<li class="comment-actions hidden" data-user-id="17908" data-action="settings-button" data-path="https://dev.to/jgaskins/comment/1gflh/settings" aria-label="Go to Jamie Gaskins's comment settings"></li>
|
||
<li class="comment-actions hidden" data-action="hide-button" data-commentable-user-id="119473" data-user-id="17908">
|
||
<button
|
||
class="flex justify-between crayons-link crayons-link--block w-100 bg-transparent border-0 hide-comment"
|
||
data-hide-type="hide"
|
||
data-comment-id="748895"
|
||
data-comment-url="https://dev.to/jgaskins/comment/1gflh"
|
||
aria-label="Hide Jamie Gaskins's comment">
|
||
Hide
|
||
</button>
|
||
</li>
|
||
<li class="mod-actions hidden mod-actions-comment-button" data-path="https://dev.to/jgaskins/comment/1gflh/mod" aria-label="Moderate Jamie Gaskins's comment"></li>
|
||
<li class="report-abuse-link-wrapper" data-path="/report-abuse?url=https://dev.to/jgaskins/comment/1gflh" aria-label="Report Jamie Gaskins's comment as abusive or violating our code of conduct and/or terms and conditions"></li>
|
||
<li class="current-user-actions"></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<div
|
||
class="
|
||
comment__body
|
||
text-styles
|
||
text-styles--secondary
|
||
body
|
||
|
||
|
||
">
|
||
<p>Ooh, nice catch! I'll go ahead and port that worker over to use <code>scan_each</code>. Thanks Josh!</p>
|
||
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
<script>
|
||
</script>
|
||
|
||
<footer class="comment__footer print-hidden">
|
||
<button
|
||
class="crayons-tooltip__activator relative crayons-btn crayons-btn--ghost crayons-btn--icon-left crayons-btn--s mr-1 reaction-like inline-flex reaction-button"
|
||
id="button-for-comment-748895"
|
||
data-comment-id="748895"
|
||
aria-label="like"
|
||
data-tracking-name="comment_heart_button">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="aguqkbrg42w5u7ogjlu3tv3k9t1tplu" class="crayons-icon reaction-icon not-reacted"><title id="aguqkbrg42w5u7ogjlu3tv3k9t1tplu">Like comment: </title><path d="M18.884 12.595l.01.011L12 19.5l-6.894-6.894.01-.01A4.875 4.875 0 0112 5.73a4.875 4.875 0 016.884 6.865zM6.431 7.037a3.375 3.375 0 000 4.773L12 17.38l5.569-5.569a3.375 3.375 0 10-4.773-4.773L9.613 10.22l-1.06-1.062 2.371-2.372a3.375 3.375 0 00-4.492.25v.001z"></path></svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="at8c7dqbq3drjreqjo0p926v5eztea3z" class="crayons-icon crayons-icon reaction-icon--like reaction-icon reacted"><title id="at8c7dqbq3drjreqjo0p926v5eztea3z">Like comment: </title>
|
||
<path d="M5.116 12.595a4.875 4.875 0 015.56-7.68h-.002L7.493 8.098l1.06 1.061 3.181-3.182a4.875 4.875 0 016.895 6.894L12 19.5l-6.894-6.894.01-.01z"></path>
|
||
</svg>
|
||
|
||
<span class="reactions-count">3</span><span class="reactions-label hidden m:inline-block"> likes</span>
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Like
|
||
</span>
|
||
</button>
|
||
|
||
<span class="favorite-control-root" data-favorite-control="true" data-variant="comment" data-favoritable-type="Comment" data-favoritable-id="748895" data-favoritable-user-id="17908" data-favorited="false" data-label-favorite="Pick as gem" data-label-favorited="Picked as gem" data-label-favorited-by-you="Picked as gem by you" data-modal-title="Gem Picked!" data-modal-body="You've picked this as a community gem!" data-modal-remaining-zero="You have no more gems left to give out today." data-modal-remaining-one="You have 1 more gem left to give out today." data-modal-remaining-other="You have %{count} more gems left to give out today." data-modal-remaining-unlimited="You have unlimited gems to give out." data-modal-close="Got it" data-modal-exhausted-title="Out of Gems" data-modal-exhausted-body="You have no more gems left to give out today. Your allowance will refresh soon!" data-modal-exhausted-close="Got it"></span>
|
||
|
||
<button
|
||
class="actions crayons-btn crayons-btn--ghost crayons-btn--s crayons-btn--icon-left toggle-reply-form mr-1 inline-flex"
|
||
data-comment-id="748895"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1gflh"
|
||
data-tracking-name="comment_reply_button"
|
||
data-testid="reply-button-748895"
|
||
rel="nofollow">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="a1tdml993y7zvll42p1uu3loqntlk45e" class="crayons-icon reaction-icon not-reacted"><title id="a1tdml993y7zvll42p1uu3loqntlk45e">Comment button</title><path d="M10.5 5h3a6 6 0 110 12v2.625c-3.75-1.5-9-3.75-9-8.625a6 6 0 016-6zM12 15.5h1.5a4.501 4.501 0 001.722-8.657A4.5 4.5 0 0013.5 6.5h-3A4.5 4.5 0 006 11c0 2.707 1.846 4.475 6 6.36V15.5z"></path></svg>
|
||
|
||
<span class="hidden m:inline-block">Reply</span>
|
||
</button>
|
||
|
||
</footer>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
|
||
</div>
|
||
</details>
|
||
|
||
|
||
</div>
|
||
</details>
|
||
|
||
|
||
</div>
|
||
</details>
|
||
|
||
<div class="js-billboard-container mid-comments-billboard-container pb-6" data-async-url="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/bmar11/post_comments_mid"></div>
|
||
<details class="comment-wrapper js-comment-wrapper comment-wrapper--deep-0
|
||
root
|
||
" open>
|
||
<summary aria-label="Toggle this comment (and replies)" data-tracking-name="expand_comment_toggle">
|
||
<span class="m:mx-1 inline-block align-middle">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="ap32bfegdkp0llfro45ff6j9obg7tzwr" class="crayons-icon expanded"><title id="ap32bfegdkp0llfro45ff6j9obg7tzwr">Collapse</title>
|
||
<path d="M12 10.677L8 6.935 9 6l3 2.807L15 6l1 .935-4 3.742zm0 4.517L9 18l-1-.935 4-3.742 4 3.742-1 .934-3-2.805z"></path>
|
||
</svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="a1k5apw9xm4d5bki5wftwsximsjizehy" class="crayons-icon collapsed"><title id="a1k5apw9xm4d5bki5wftwsximsjizehy">Expand</title>
|
||
<path d="M12 18l-4-3.771 1-.943 3 2.829 3-2.829 1 .943L12 18zm0-10.115l-3 2.829-1-.943L12 6l4 3.771-1 .942-3-2.828z"></path>
|
||
</svg>
|
||
|
||
</span>
|
||
<span class="js-collapse-comment-content inline-block align-middle"></span>
|
||
</summary>
|
||
<div
|
||
id="comment-node-745011"
|
||
class="
|
||
comment single-comment-node
|
||
|
||
root
|
||
comment--deep-0
|
||
|
||
"
|
||
data-comment-id="745011"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1ga27"
|
||
data-comment-author-id="74157"
|
||
data-content-user-id="74157">
|
||
<a name="comment-1ga27" style="position: absolute; top: -8px;"> </a>
|
||
|
||
<div class="comment__inner">
|
||
<a href="https://dev.to/bennypowers" class="shrink-0 crayons-avatar m:crayons-avatar--l mt-4 m:mt-3">
|
||
<img class="crayons-avatar__image" width="32" height="32" src="https://media2.dev.to/dynamic/image/width=50,height=50,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F74157%2F1d0119c2-14e0-4dd6-a3ec-084d7b3c6323.png" alt="bennypowers profile image" loading="lazy" />
|
||
</a>
|
||
|
||
|
||
<div class="inner-comment comment__details">
|
||
<div class="comment__content crayons-card">
|
||
|
||
|
||
|
||
<div class="comment__header">
|
||
<a href="https://dev.to/bennypowers" class="crayons-link crayons-link--secondary flex items-center fw-medium m:hidden">
|
||
<span class="js-comment-username">
|
||
Benny Powers 🇮🇱🦁️
|
||
</span>
|
||
</a>
|
||
<div class="profile-preview-card relative mb-4 s:mb-0 fw-medium hidden m:block">
|
||
<button id="comment-profile-preview-trigger-745011" aria-controls="comment-profile-preview-content-745011" class="profile-preview-card__trigger p-1 -my-1 -ml-1 crayons-btn crayons-btn--ghost" aria-label="Benny Powers 🇮🇱🦁️ profile details">
|
||
Benny Powers 🇮🇱🦁️
|
||
|
||
|
||
</button>
|
||
<div id="comment-profile-preview-content-745011" class="profile-preview-card__content p-4 pt-0 branded-7 crayons-dropdown" style="--card-color: #d946b2; border-top-color: var(--card-color);" data-testid="profile-preview-card" data-repositioning-dropdown="true">
|
||
<div class="gap-4 grid">
|
||
<div class="-mt-4">
|
||
<a href="/bennypowers" class="flex">
|
||
<span class="crayons-avatar crayons-avatar--xl mr-2 shrink-0">
|
||
<img src="https://media2.dev.to/dynamic/image/width=90,height=90,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F74157%2F1d0119c2-14e0-4dd6-a3ec-084d7b3c6323.png" class="crayons-avatar__image" alt="" loading="lazy" />
|
||
</span>
|
||
<span class="crayons-link crayons-subtitle-2 mt-5">
|
||
Benny Powers 🇮🇱🦁️
|
||
|
||
|
||
</span>
|
||
</a>
|
||
</div>
|
||
|
||
<div class="print-hidden">
|
||
<button name="button" type="button" data-info="{"className":"User","style":"","id":74157,"name":"Benny Powers 🇮🇱🦁️"}" class="crayons-btn follow-action-button whitespace-nowrap w-100 follow-user" aria-label="Follow user: Benny Powers 🇮🇱🦁️" aria-pressed="false">Follow</button>
|
||
</div>
|
||
<div class="color-base-70">
|
||
Coding is as much a matter of personal growth as it is of logic and control-flow. I keep patience, curiosity, & exuberance in the same toolbox as vim and git.
|
||
|
||
*Opinions posted are my own*
|
||
</div>
|
||
|
||
<div class="user-metadata-details">
|
||
<ul class="user-metadata-details-inner">
|
||
<li>
|
||
<div class="key">
|
||
Location
|
||
</div>
|
||
<div class="value">
|
||
Jerusalem
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Education
|
||
</div>
|
||
<div class="value">
|
||
Self Taught / Online
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Work
|
||
</div>
|
||
<div class="value">
|
||
Senior Principal Software Engineer @RedHat
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Joined
|
||
</div>
|
||
<div class="value">
|
||
<time datetime="2018-05-22T06:57:08Z" class="date">May 22, 2018</time>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
<span class="color-base-30 px-2 m:pl-0" role="presentation">•</span>
|
||
|
||
<a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1ga27" class="comment-date crayons-link crayons-link--secondary fs-s">
|
||
<time datetime="2021-07-14T21:40:21Z" class=date-short-year>
|
||
Jul 14 '21
|
||
</time>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
<div class="comment__dropdown" data-tracking-name="comment_dropdown">
|
||
<button id="comment-dropdown-trigger-745011" aria-controls="comment-dropdown-745011" aria-expanded="false"
|
||
class="dropbtn comment__dropdown-trigger crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon "
|
||
aria-label="Toggle dropdown menu" aria-haspopup="true">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="aqfbp6i9s38jupcf5zhs5xt06g7fmrzf" class="crayons-icon pointer-events-none"><title id="aqfbp6i9s38jupcf5zhs5xt06g7fmrzf">Dropdown menu</title>
|
||
<path fill-rule="evenodd" clip-rule="evenodd" d="M8.25 12a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm5.25 0a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm3.75 1.5a1.5 1.5 0 100-3 1.5 1.5 0 000 3z"></path>
|
||
</svg>
|
||
|
||
</button>
|
||
<div id="comment-dropdown-745011" class="crayons-dropdown right-1 s:right-0 s:left-auto fs-base dropdown">
|
||
<ul class="m-0">
|
||
<li><a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1ga27" class="crayons-link crayons-link--block permalink-copybtn" aria-label="Copy link to Benny Powers 🇮🇱🦁️'s comment" data-no-instant>Copy link</a></li>
|
||
<li class="comment-actions hidden" data-user-id="74157" data-action="settings-button" data-path="https://dev.to/bennypowers/comment/1ga27/settings" aria-label="Go to Benny Powers 🇮🇱🦁️'s comment settings"></li>
|
||
<li class="comment-actions hidden" data-action="hide-button" data-commentable-user-id="119473" data-user-id="74157">
|
||
<button
|
||
class="flex justify-between crayons-link crayons-link--block w-100 bg-transparent border-0 hide-comment"
|
||
data-hide-type="hide"
|
||
data-comment-id="745011"
|
||
data-comment-url="https://dev.to/bennypowers/comment/1ga27"
|
||
aria-label="Hide Benny Powers 🇮🇱🦁️'s comment">
|
||
Hide
|
||
</button>
|
||
</li>
|
||
<li class="mod-actions hidden mod-actions-comment-button" data-path="https://dev.to/bennypowers/comment/1ga27/mod" aria-label="Moderate Benny Powers 🇮🇱🦁️'s comment"></li>
|
||
<li class="report-abuse-link-wrapper" data-path="/report-abuse?url=https://dev.to/bennypowers/comment/1ga27" aria-label="Report Benny Powers 🇮🇱🦁️'s comment as abusive or violating our code of conduct and/or terms and conditions"></li>
|
||
<li class="current-user-actions"></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<div
|
||
class="
|
||
comment__body
|
||
text-styles
|
||
text-styles--secondary
|
||
body
|
||
|
||
|
||
">
|
||
<p>Great run down thanks for being open</p>
|
||
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
<script>
|
||
</script>
|
||
|
||
<footer class="comment__footer print-hidden">
|
||
<button
|
||
class="crayons-tooltip__activator relative crayons-btn crayons-btn--ghost crayons-btn--icon-left crayons-btn--s mr-1 reaction-like inline-flex reaction-button"
|
||
id="button-for-comment-745011"
|
||
data-comment-id="745011"
|
||
aria-label="like"
|
||
data-tracking-name="comment_heart_button">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="aq91l6kgt5jnpwskqupkaldsl8dalrm" class="crayons-icon reaction-icon not-reacted"><title id="aq91l6kgt5jnpwskqupkaldsl8dalrm">Like comment: </title><path d="M18.884 12.595l.01.011L12 19.5l-6.894-6.894.01-.01A4.875 4.875 0 0112 5.73a4.875 4.875 0 016.884 6.865zM6.431 7.037a3.375 3.375 0 000 4.773L12 17.38l5.569-5.569a3.375 3.375 0 10-4.773-4.773L9.613 10.22l-1.06-1.062 2.371-2.372a3.375 3.375 0 00-4.492.25v.001z"></path></svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="a1g6avk3qde5kz4fk8m1gmsax49jgpzp" class="crayons-icon crayons-icon reaction-icon--like reaction-icon reacted"><title id="a1g6avk3qde5kz4fk8m1gmsax49jgpzp">Like comment: </title>
|
||
<path d="M5.116 12.595a4.875 4.875 0 015.56-7.68h-.002L7.493 8.098l1.06 1.061 3.181-3.182a4.875 4.875 0 016.895 6.894L12 19.5l-6.894-6.894.01-.01z"></path>
|
||
</svg>
|
||
|
||
<span class="reactions-count">3</span><span class="reactions-label hidden m:inline-block"> likes</span>
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Like
|
||
</span>
|
||
</button>
|
||
|
||
<span class="favorite-control-root" data-favorite-control="true" data-variant="comment" data-favoritable-type="Comment" data-favoritable-id="745011" data-favoritable-user-id="74157" data-favorited="false" data-label-favorite="Pick as gem" data-label-favorited="Picked as gem" data-label-favorited-by-you="Picked as gem by you" data-modal-title="Gem Picked!" data-modal-body="You've picked this as a community gem!" data-modal-remaining-zero="You have no more gems left to give out today." data-modal-remaining-one="You have 1 more gem left to give out today." data-modal-remaining-other="You have %{count} more gems left to give out today." data-modal-remaining-unlimited="You have unlimited gems to give out." data-modal-close="Got it" data-modal-exhausted-title="Out of Gems" data-modal-exhausted-body="You have no more gems left to give out today. Your allowance will refresh soon!" data-modal-exhausted-close="Got it"></span>
|
||
|
||
<button
|
||
class="actions crayons-btn crayons-btn--ghost crayons-btn--s crayons-btn--icon-left toggle-reply-form mr-1 inline-flex"
|
||
data-comment-id="745011"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1ga27"
|
||
data-tracking-name="comment_reply_button"
|
||
data-testid="reply-button-745011"
|
||
rel="nofollow">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="aemd98kg8mfdjiwdxsatdld1w0lwxrje" class="crayons-icon reaction-icon not-reacted"><title id="aemd98kg8mfdjiwdxsatdld1w0lwxrje">Comment button</title><path d="M10.5 5h3a6 6 0 110 12v2.625c-3.75-1.5-9-3.75-9-8.625a6 6 0 016-6zM12 15.5h1.5a4.501 4.501 0 001.722-8.657A4.5 4.5 0 0013.5 6.5h-3A4.5 4.5 0 006 11c0 2.707 1.846 4.475 6 6.36V15.5z"></path></svg>
|
||
|
||
<span class="hidden m:inline-block">Reply</span>
|
||
</button>
|
||
|
||
</footer>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
</details>
|
||
|
||
<details class="comment-wrapper js-comment-wrapper comment-wrapper--deep-0
|
||
root
|
||
" open>
|
||
<summary aria-label="Toggle this comment (and replies)" data-tracking-name="expand_comment_toggle">
|
||
<span class="m:mx-1 inline-block align-middle">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="addjut09zh4eo4dvvv8ymoa5hjuf1cap" class="crayons-icon expanded"><title id="addjut09zh4eo4dvvv8ymoa5hjuf1cap">Collapse</title>
|
||
<path d="M12 10.677L8 6.935 9 6l3 2.807L15 6l1 .935-4 3.742zm0 4.517L9 18l-1-.935 4-3.742 4 3.742-1 .934-3-2.805z"></path>
|
||
</svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="a6251tcofxe7j06tvbx5yg7wmng8e6np" class="crayons-icon collapsed"><title id="a6251tcofxe7j06tvbx5yg7wmng8e6np">Expand</title>
|
||
<path d="M12 18l-4-3.771 1-.943 3 2.829 3-2.829 1 .943L12 18zm0-10.115l-3 2.829-1-.943L12 6l4 3.771-1 .942-3-2.828z"></path>
|
||
</svg>
|
||
|
||
</span>
|
||
<span class="js-collapse-comment-content inline-block align-middle"></span>
|
||
</summary>
|
||
<div
|
||
id="comment-node-745009"
|
||
class="
|
||
comment single-comment-node
|
||
|
||
root
|
||
comment--deep-0
|
||
|
||
"
|
||
data-comment-id="745009"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1ga25"
|
||
data-comment-author-id="660027"
|
||
data-content-user-id="660027">
|
||
<a name="comment-1ga25" style="position: absolute; top: -8px;"> </a>
|
||
|
||
<div class="comment__inner">
|
||
<a href="https://dev.to/0xjepsen" class="shrink-0 crayons-avatar m:crayons-avatar--l mt-4 m:mt-3">
|
||
<img class="crayons-avatar__image" width="32" height="32" src="https://media2.dev.to/dynamic/image/width=50,height=50,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F660027%2F495d1b26-1ba8-40a0-802e-19993833c61e.jpg" alt="0xjepsen profile image" loading="lazy" />
|
||
</a>
|
||
|
||
|
||
<div class="inner-comment comment__details">
|
||
<div class="comment__content crayons-card">
|
||
|
||
|
||
|
||
<div class="comment__header">
|
||
<a href="https://dev.to/0xjepsen" class="crayons-link crayons-link--secondary flex items-center fw-medium m:hidden">
|
||
<span class="js-comment-username">
|
||
Jepsen
|
||
</span>
|
||
</a>
|
||
<div class="profile-preview-card relative mb-4 s:mb-0 fw-medium hidden m:block">
|
||
<button id="comment-profile-preview-trigger-745009" aria-controls="comment-profile-preview-content-745009" class="profile-preview-card__trigger p-1 -my-1 -ml-1 crayons-btn crayons-btn--ghost" aria-label="Jepsen profile details">
|
||
Jepsen
|
||
|
||
|
||
</button>
|
||
<div id="comment-profile-preview-content-745009" class="profile-preview-card__content p-4 pt-0 branded-7 crayons-dropdown" style="--card-color: #000000; border-top-color: var(--card-color);" data-testid="profile-preview-card" data-repositioning-dropdown="true">
|
||
<div class="gap-4 grid">
|
||
<div class="-mt-4">
|
||
<a href="/0xjepsen" class="flex">
|
||
<span class="crayons-avatar crayons-avatar--xl mr-2 shrink-0">
|
||
<img src="https://media2.dev.to/dynamic/image/width=90,height=90,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F660027%2F495d1b26-1ba8-40a0-802e-19993833c61e.jpg" class="crayons-avatar__image" alt="" loading="lazy" />
|
||
</span>
|
||
<span class="crayons-link crayons-subtitle-2 mt-5">
|
||
Jepsen
|
||
|
||
|
||
</span>
|
||
</a>
|
||
</div>
|
||
|
||
<div class="print-hidden">
|
||
<button name="button" type="button" data-info="{"className":"User","style":"","id":660027,"name":"Jepsen"}" class="crayons-btn follow-action-button whitespace-nowrap w-100 follow-user" aria-label="Follow user: Jepsen" aria-pressed="false">Follow</button>
|
||
</div>
|
||
<div class="color-base-70">
|
||
I’m a computer scientist passionate about high impact technology. My research interests lay at the intersection of mathematics, computer science, and philosophy. I value work ethic and tenacity above
|
||
</div>
|
||
|
||
<div class="user-metadata-details">
|
||
<ul class="user-metadata-details-inner">
|
||
<li>
|
||
<div class="key">
|
||
Location
|
||
</div>
|
||
<div class="value">
|
||
San Francisco
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Education
|
||
</div>
|
||
<div class="value">
|
||
BS Mathematics and philosophy, MS Computer Science
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Work
|
||
</div>
|
||
<div class="value">
|
||
Research and Engineering
|
||
</div>
|
||
</li>
|
||
<li>
|
||
<div class="key">
|
||
Joined
|
||
</div>
|
||
<div class="value">
|
||
<time datetime="2021-07-01T20:49:39Z" class="date">Jul 1, 2021</time>
|
||
</div>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
|
||
<span class="color-base-30 px-2 m:pl-0" role="presentation">•</span>
|
||
|
||
<a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1ga25" class="comment-date crayons-link crayons-link--secondary fs-s">
|
||
<time datetime="2021-07-14T21:32:32Z" class=date-short-year>
|
||
Jul 14 '21
|
||
</time>
|
||
|
||
</a>
|
||
|
||
|
||
|
||
<div class="comment__dropdown" data-tracking-name="comment_dropdown">
|
||
<button id="comment-dropdown-trigger-745009" aria-controls="comment-dropdown-745009" aria-expanded="false"
|
||
class="dropbtn comment__dropdown-trigger crayons-btn crayons-btn--s crayons-btn--ghost crayons-btn--icon "
|
||
aria-label="Toggle dropdown menu" aria-haspopup="true">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="adelz3hpmqbuwu03sn5riobta7qupznr" class="crayons-icon pointer-events-none"><title id="adelz3hpmqbuwu03sn5riobta7qupznr">Dropdown menu</title>
|
||
<path fill-rule="evenodd" clip-rule="evenodd" d="M8.25 12a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm5.25 0a1.5 1.5 0 11-3 0 1.5 1.5 0 013 0zm3.75 1.5a1.5 1.5 0 100-3 1.5 1.5 0 000 3z"></path>
|
||
</svg>
|
||
|
||
</button>
|
||
<div id="comment-dropdown-745009" class="crayons-dropdown right-1 s:right-0 s:left-auto fs-base dropdown">
|
||
<ul class="m-0">
|
||
<li><a href="https://dev.to/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj#comment-1ga25" class="crayons-link crayons-link--block permalink-copybtn" aria-label="Copy link to Jepsen's comment" data-no-instant>Copy link</a></li>
|
||
<li class="comment-actions hidden" data-user-id="660027" data-action="settings-button" data-path="https://dev.to/0xjepsen/comment/1ga25/settings" aria-label="Go to Jepsen's comment settings"></li>
|
||
<li class="comment-actions hidden" data-action="hide-button" data-commentable-user-id="119473" data-user-id="660027">
|
||
<button
|
||
class="flex justify-between crayons-link crayons-link--block w-100 bg-transparent border-0 hide-comment"
|
||
data-hide-type="hide"
|
||
data-comment-id="745009"
|
||
data-comment-url="https://dev.to/0xjepsen/comment/1ga25"
|
||
aria-label="Hide Jepsen's comment">
|
||
Hide
|
||
</button>
|
||
</li>
|
||
<li class="mod-actions hidden mod-actions-comment-button" data-path="https://dev.to/0xjepsen/comment/1ga25/mod" aria-label="Moderate Jepsen's comment"></li>
|
||
<li class="report-abuse-link-wrapper" data-path="/report-abuse?url=https://dev.to/0xjepsen/comment/1ga25" aria-label="Report Jepsen's comment as abusive or violating our code of conduct and/or terms and conditions"></li>
|
||
<li class="current-user-actions"></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<div
|
||
class="
|
||
comment__body
|
||
text-styles
|
||
text-styles--secondary
|
||
body
|
||
|
||
|
||
">
|
||
<p>Wow Thanks for the update of the experience! </p>
|
||
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
<script>
|
||
</script>
|
||
|
||
<footer class="comment__footer print-hidden">
|
||
<button
|
||
class="crayons-tooltip__activator relative crayons-btn crayons-btn--ghost crayons-btn--icon-left crayons-btn--s mr-1 reaction-like inline-flex reaction-button"
|
||
id="button-for-comment-745009"
|
||
data-comment-id="745009"
|
||
aria-label="like"
|
||
data-tracking-name="comment_heart_button">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="a7dfrniakygjxyhjkl8i5w7pp00y1iay" class="crayons-icon reaction-icon not-reacted"><title id="a7dfrniakygjxyhjkl8i5w7pp00y1iay">Like comment: </title><path d="M18.884 12.595l.01.011L12 19.5l-6.894-6.894.01-.01A4.875 4.875 0 0112 5.73a4.875 4.875 0 016.884 6.865zM6.431 7.037a3.375 3.375 0 000 4.773L12 17.38l5.569-5.569a3.375 3.375 0 10-4.773-4.773L9.613 10.22l-1.06-1.062 2.371-2.372a3.375 3.375 0 00-4.492.25v.001z"></path></svg>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" role="img" aria-labelledby="a8fibwapem3e1ne9zef6gxqxw7bx1ucf" class="crayons-icon crayons-icon reaction-icon--like reaction-icon reacted"><title id="a8fibwapem3e1ne9zef6gxqxw7bx1ucf">Like comment: </title>
|
||
<path d="M5.116 12.595a4.875 4.875 0 015.56-7.68h-.002L7.493 8.098l1.06 1.061 3.181-3.182a4.875 4.875 0 016.895 6.894L12 19.5l-6.894-6.894.01-.01z"></path>
|
||
</svg>
|
||
|
||
<span class="reactions-count">3</span><span class="reactions-label hidden m:inline-block"> likes</span>
|
||
<span data-testid="tooltip" class="crayons-tooltip__content">
|
||
Like
|
||
</span>
|
||
</button>
|
||
|
||
<span class="favorite-control-root" data-favorite-control="true" data-variant="comment" data-favoritable-type="Comment" data-favoritable-id="745009" data-favoritable-user-id="660027" data-favorited="false" data-label-favorite="Pick as gem" data-label-favorited="Picked as gem" data-label-favorited-by-you="Picked as gem by you" data-modal-title="Gem Picked!" data-modal-body="You've picked this as a community gem!" data-modal-remaining-zero="You have no more gems left to give out today." data-modal-remaining-one="You have 1 more gem left to give out today." data-modal-remaining-other="You have %{count} more gems left to give out today." data-modal-remaining-unlimited="You have unlimited gems to give out." data-modal-close="Got it" data-modal-exhausted-title="Out of Gems" data-modal-exhausted-body="You have no more gems left to give out today. Your allowance will refresh soon!" data-modal-exhausted-close="Got it"></span>
|
||
|
||
<button
|
||
class="actions crayons-btn crayons-btn--ghost crayons-btn--s crayons-btn--icon-left toggle-reply-form mr-1 inline-flex"
|
||
data-comment-id="745009"
|
||
data-path="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/comments/1ga25"
|
||
data-tracking-name="comment_reply_button"
|
||
data-testid="reply-button-745009"
|
||
rel="nofollow">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" role="img" aria-labelledby="arkw93ylt2fz3oy5t2w6ze4t6xall7p4" class="crayons-icon reaction-icon not-reacted"><title id="arkw93ylt2fz3oy5t2w6ze4t6xall7p4">Comment button</title><path d="M10.5 5h3a6 6 0 110 12v2.625c-3.75-1.5-9-3.75-9-8.625a6 6 0 016-6zM12 15.5h1.5a4.501 4.501 0 001.722-8.657A4.5 4.5 0 0013.5 6.5h-3A4.5 4.5 0 006 11c0 2.707 1.846 4.475 6 6.36V15.5z"></path></svg>
|
||
|
||
<span class="hidden m:inline-block">Reply</span>
|
||
</button>
|
||
|
||
</footer>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</div>
|
||
</details>
|
||
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<div class="align-center">
|
||
|
||
|
||
<nav class="fs-s align-center block" aria-label="Conduct controls">
|
||
<a href="/code-of-conduct" class="crayons-link crayons-link--secondary">Code of Conduct</a>
|
||
<span class="opacity-25 px-2" role="presentation">•</span>
|
||
<a href="/report-abuse" class="crayons-link crayons-link--secondary">Report abuse</a>
|
||
</nav>
|
||
|
||
</div>
|
||
|
||
</section>
|
||
<div id="hide-comments-modal" class="hidden">
|
||
<form id="hide-comments-modal__form" class="hide-comments-modal__form" data-type="json" action="/comments/hide" accept-charset="UTF-8" data-remote="true" method="post"><input type="hidden" name="_method" value="patch" autocomplete="off" /><input type="hidden" name="authenticity_token" value="WrTiU0-G2cv7KpD3K-QjVWSnq_fV3KepS0swuSscVZfcPlPIufHcJ_v_H117saCHr9WjRX62YWVme1wBTbtRDQ" autocomplete="off" />
|
||
<div class="hide-comments-modal__content">
|
||
<p class="pb-2">
|
||
Are you sure you want to hide this comment? It will become hidden in your post, but will still be visible via the comment's <a id="hide-comments-modal__comment-permalink" href="#">permalink</a>.
|
||
</p>
|
||
<label class="crayons-field crayons-field--checkbox my-2">
|
||
<input name="hide_children" type="hidden" value="0" autocomplete="off" /><input class="hide_children crayons-checkbox" type="checkbox" value="1" name="hide_children" id="hide_children" />
|
||
<p class="crayons-field__label">Hide child comments as well</p>
|
||
</label>
|
||
<p class="pb-4 pt-2">
|
||
<button type="submit" class="crayons-btn">
|
||
Confirm
|
||
</button>
|
||
</p>
|
||
</div>
|
||
</form> <p class="fs-s color-base-60">For further actions, you may consider blocking this person and/or <a id="hide-comments-modal__report-link" href="/report-abuse">reporting abuse</a></p>
|
||
</div>
|
||
|
||
|
||
|
||
</article>
|
||
|
||
<div class="pb-4 crayons-layout__comments-billboard">
|
||
<div class="new-bb-container pb-4 crayons-layout__comments-billboard" data-async-url="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/bmar11/post_comments"></div>
|
||
</div>
|
||
|
||
<!-- Bottom content skipped via SKIP_BOTTOM_CONTENT config -->
|
||
</div>
|
||
</main>
|
||
|
||
<aside class="crayons-layout__sidebar-right" aria-label="Author details">
|
||
<div class="crayons-article-sticky grid gap-4 pb-4 break-word" id="article-show-primary-sticky-nav">
|
||
<div class="crayons-card crayons-card--secondary branded-7 p-4 pt-0 gap-4 grid" style="border-top-color: #080808;">
|
||
<div class="-mt-4">
|
||
<a href="/devteam" class="flex">
|
||
<span class="crayons-logo crayons-logo--xl mr-2 shrink-0">
|
||
<img src="https://media2.dev.to/dynamic/image/width=90,height=90,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Forganization%2Fprofile_image%2F1%2Fd908a186-5651-4a5a-9f76-15200bc6801f.jpg" class="crayons-logo__image" alt="" loading="lazy" />
|
||
</span>
|
||
<span class="crayons-link crayons-subtitle-2 mt-5">
|
||
The DEV Team
|
||
|
||
|
||
</span>
|
||
</a>
|
||
</div>
|
||
|
||
<div class="print-hidden">
|
||
<button name="button" type="button" data-info="{"className":"Organization","style":"","id":1,"name":"The DEV Team"}" class="crayons-btn follow-action-button whitespace-nowrap w-100 " aria-label="Follow organization: The DEV Team" aria-pressed="false">Follow</button>
|
||
</div>
|
||
<div class="color-base-70">
|
||
The hardworking team behind DEV ❤️
|
||
</div>
|
||
|
||
<div>
|
||
<p>Want to contribute to open source and help make the DEV community stronger?</p>
|
||
|
||
<p>The code that powers DEV is called <strong>Forem</strong> and is freely available on GitHub.</p>
|
||
|
||
<p>You're welcome to jump in!</p>
|
||
|
||
</div>
|
||
<div>
|
||
<a href="https://github.com/forem/forem" class="crayons-btn crayons-btn--outlined w-100">
|
||
Contribute to Forem
|
||
</a>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
<div class="crayons-card crayons-card--secondary">
|
||
<header class="crayons-card__header">
|
||
<h3 class="crayons-subtitle-2">
|
||
Trending on <a href="https://dev.to">DEV Community</a>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 44 44" width="24" height="24" role="img" aria-labelledby="aajerku6x0g50ja7kp0eq29r9ec3wy1h" class="crayons-icon crayons-icon--default"><title id="aajerku6x0g50ja7kp0eq29r9ec3wy1h">Hot</title>
|
||
<g class="nc-icon-wrapper">
|
||
<path fill="#F4900C" d="M39 23a16.96 16.96 0 00-1.04-5.868c-.46 5.389-3.333 8.157-6.335 6.868-2.812-1.208-.917-5.917-.777-8.164.236-3.809-.012-8.169-6.931-11.794 2.875 5.5.333 8.917-2.333 9.125-2.958.231-5.667-2.542-4.667-7.042-3.238 2.386-3.332 6.402-2.333 9 1.042 2.708-.042 4.958-2.583 5.208-2.84.28-4.418-3.041-2.963-8.333A16.936 16.936 0 005 23c0 9.389 7.611 17 17 17s17-7.611 17-17z"></path>
|
||
<path fill="#FFCC4D" d="M32.394 27.999c.148 3.084-2.561 4.293-4.019 3.709-2.106-.843-1.541-2.291-2.083-5.291s-2.625-5.083-5.708-6c2.25 6.333-1.247 8.667-3.08 9.084-1.872.426-3.753-.001-3.968-4.007A11.964 11.964 0 0010 34c0 .368.023.73.055 1.09C13.125 38.124 17.342 40 22 40s8.875-1.876 11.945-4.91c.032-.36.055-.722.055-1.09 0-2.187-.584-4.236-1.606-6.001z"></path>
|
||
</g>
|
||
</svg>
|
||
|
||
</h3>
|
||
</header>
|
||
<div>
|
||
<a class="crayons-link crayons-link--contentful flex" href="/devstackhub/10-ai-website-builders-i-tested-so-you-can-skip-the-trial-and-error-1ac8">
|
||
<span class="crayons-avatar mr-2 shrink-0">
|
||
<img src="https://media2.dev.to/dynamic/image/width=90,height=90,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F4081963%2F45530630-a154-4a97-bc5a-a6bd68302ffc.png" class="crayons-avatar__image" loading="lazy" alt="Harsh Raval profile image">
|
||
</span>
|
||
<div>
|
||
10 AI Website Builders I Tested So You Can Skip the Trial and Error
|
||
<div class="crayons-link__secondary -ml-1">
|
||
<span class="mr-1"><span class="opacity-50">#</span>ai</span>
|
||
<span class="mr-1"><span class="opacity-50">#</span>nocode</span>
|
||
<span class="mr-1"><span class="opacity-50">#</span>design</span>
|
||
<span class="mr-1"><span class="opacity-50">#</span>discuss</span>
|
||
</div>
|
||
</div>
|
||
</a>
|
||
<a class="crayons-link crayons-link--contentful flex" href="/georgekobaidze/the-same-day-180-times-how-does-the-unglamorous-machinery-of-locking-in-actually-work-4n0l">
|
||
<span class="crayons-avatar mr-2 shrink-0">
|
||
<img src="https://media2.dev.to/dynamic/image/width=90,height=90,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F55651%2F3ad144e9-ca91-4395-b73a-9a0a3d843af9.jpg" class="crayons-avatar__image" loading="lazy" alt="Giorgi Kobaidze profile image">
|
||
</span>
|
||
<div>
|
||
The Same Day, 180 Times: How Does the Unglamorous Machinery of Locking In Actually Work?
|
||
<div class="crayons-link__secondary -ml-1">
|
||
<span class="mr-1"><span class="opacity-50">#</span>productivity</span>
|
||
<span class="mr-1"><span class="opacity-50">#</span>career</span>
|
||
<span class="mr-1"><span class="opacity-50">#</span>community</span>
|
||
<span class="mr-1"><span class="opacity-50">#</span>journey</span>
|
||
</div>
|
||
</div>
|
||
</a>
|
||
<a class="crayons-link crayons-link--contentful flex" href="/sylwia-lask/ai-is-already-better-at-coding-than-most-software-developers-4hno">
|
||
<span class="crayons-avatar mr-2 shrink-0">
|
||
<img src="https://media2.dev.to/dynamic/image/width=90,height=90,fit=cover,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.us-east-2.amazonaws.com%2Fuploads%2Fuser%2Fprofile_image%2F3535771%2Fe22860d5-274b-43c9-819b-56b162e5bd5a.jpeg" class="crayons-avatar__image" loading="lazy" alt="Sylwia Laskowska profile image">
|
||
</span>
|
||
<div>
|
||
AI Is Already Better at Coding Than Most Software Developers
|
||
<div class="crayons-link__secondary -ml-1">
|
||
<span class="mr-1"><span class="opacity-50">#</span>discuss</span>
|
||
<span class="mr-1"><span class="opacity-50">#</span>ai</span>
|
||
<span class="mr-1"><span class="opacity-50">#</span>webdev</span>
|
||
<span class="mr-1"><span class="opacity-50">#</span>programming</span>
|
||
</div>
|
||
</div>
|
||
</a>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="crayons-article-sticky grid gap-4 break-word sidebar-bb" data-async-url="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/bmar11/post_sidebar"></div>
|
||
|
||
|
||
</aside>
|
||
</div>
|
||
|
||
<div class="mod-actions-menu print-hidden"></div>
|
||
<div data-testid="unpublish-post-modal-container" class="unpublish-post-modal-container hidden"></div>
|
||
|
||
<div class="fullscreen-code js-fullscreen-code"></div>
|
||
<script src="https://assets.dev.to/assets/billboard-1b75959755e3f87ef58c28efef9eab9853df3b84cf27a937749b2c3449fd450d.js" defer="defer"></script>
|
||
<script src="https://assets.dev.to/assets/localizeArticleDates-ddf0f7415ce8497325dd1b64bc1e022762d4d14aec7f538e6988744eeaa9a501.js" defer="defer"></script>
|
||
<script src="https://assets.dev.to/assets/articleReactions-03b03e63c7319a6caa1788ca81e7b25dff9fe90b38ca051b8b979a773b011e3b.js" defer="defer"></script>
|
||
<script src="https://assets.dev.to/assets/favoriteControls-882b1c32df621133f9c47da91e59df01e50d77cb6cd48a6611a285ce9036acc6.js" defer="defer"></script>
|
||
|
||
<script>
|
||
</script>
|
||
<div class="js-billboard-container pb-4 crayons-layout__comments-billboard" data-async-url="/devteam/incident-retro-failing-comment-creation-erroneous-push-notifications-55dj/bmar11/post_fixed_bottom"></div>
|
||
|
||
|
||
<div id="runtime-banner-container"></div>
|
||
<div id="i18n-translations" data-translations="{"en":{"core":{"add_comment":"Add comment","beta":"beta","comment":"Comment","copy_link":"Copy link","edit_profile":"Edit profile","follow":"Follow","follow_back":"Follow back","following":"Following","like":"Like","loading":"loading...","reaction":"Reaction","report_abuse":"Report abuse","search":"Search","success_settings":"Successfully updated settings.","search_placeholder":"Search...","search_find_related_posts":"Find related posts...","search_powered_by":"Powered by Algolia","search_submit_search":"Submit search for advanced filtering.","search_displaying_recommendations":"Displaying Algolia Recommendations — Start typing to search","article_form_save_changes":"Save changes","article_form_schedule":"Schedule","article_form_publish":"Publish","article_form_loading_preview":"Loading preview","article_form_preview_loaded":"Preview loaded","article_form_co_authors":"Co-authors","article_form_co_authors_description":"Add up to 4 co-authors from %{org_name}.","article_form_co_authors_placeholder":"Add up to 4...","comments_preview":"Preview","comments_continue_editing":"Continue editing","survey_enter_response":"Enter your response here...","survey_thank_you_response":"Thank you for your response.","survey_thank_you_completing":"Thank you for completing the survey!","dashboard_analytics_readers":"Readers","dashboard_analytics_avg_read_time":"avg. read time %{seconds}s","dashboard_analytics_comments":"Comments","dashboard_analytics_reactions":"Reactions","dashboard_analytics_unique_reactors":"from %{count} unique users","dashboard_analytics_bookmarks":"Bookmarks","dashboard_analytics_followers":"New Followers","top_contributors_empty":"No engagers found for this period.","top_contributors_reactions":"reactions","top_contributors_comments":"comments","top_contributors_weight_note":"Comments carry more weight","follower_engagement_ratio":"%{ratio}% of followers engaged","follower_engagement_detail":"(%{engaged} of %{total})","dashboard_analytics_no_data":"No data available for this period","dashboard_analytics_no_referrers":"No traffic sources yet","stats_by":"by","editor_new_title":"New post title here...","editor_body_placeholder":"Write your post content here...","tags_field_label":"Add up to 4 tags","tags_field_placeholder":"Add up to 4 tags...","counted_organization":{"one":"%{count} organization","other":"%{count} organizations"},"counted_user":{"one":"%{count} person","other":"%{count} people"},"not_following":"You're not following anyone","following_everyone":"You're following %{details} (everyone)","you_are_following":"You're following","and":"and"}}}"></div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
<footer id="footer" class="crayons-footer print-hidden">
|
||
<div id="footer-container" class="crayons-footer__container">
|
||
|
||
|
||
|
||
<p class="fs-s crayons-footer__description">
|
||
<a class="c-link c-link--branded fw-medium" aria-label="DEV Community Home" href="/">DEV Community</a> — A space to discuss and keep up software development and manage your software career
|
||
</p>
|
||
|
||
<ul class="footer__nav-links flex gap-2 justify-center flex-wrap fs-s p-0" style="" />
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="/">
|
||
Home
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="/challenges">
|
||
DEV Challenges
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="/++">
|
||
DEV++
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="/videos">
|
||
Videos
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="/deved">
|
||
DEV Education Tracks
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="/help">
|
||
DEV Help
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="/advertise">
|
||
Advertise on DEV
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="/organizations">
|
||
Organization Accounts
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="/showcase">
|
||
DEV Showcase
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="/about">
|
||
About
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="/contact">
|
||
Contact
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="/free-postgres-database-tier">
|
||
Free Postgres Database
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="https://shop.forem.com/">
|
||
DEV Shop
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="https://mlh.io/">
|
||
MLH
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
<ul class="footer__nav-links flex gap-2 justify-center flex-wrap fs-s p-0" style="" />
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="/code-of-conduct">
|
||
Code of Conduct
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="/privacy">
|
||
Privacy Policy
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
<li class="footer__nav-link flex items-center">
|
||
<a href="/terms">
|
||
Terms of Use
|
||
</a>
|
||
<span class="dot ml-2"></span>
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
<div class="fs-s">
|
||
<p>Built on <a class="c-link c-link--branded" target="_blank" rel="noopener" href="https://www.forem.com">Forem</a> — the <a target="_blank" rel="noopener" class="c-link c-link--branded" href="https://dev.to/t/opensource">open source</a> software that powers <a target="_blank" rel="noopener" class="c-link c-link--branded" href="https://dev.to">DEV</a> and other inclusive communities.</p>
|
||
<p>Made with love and <a target="_blank" rel="noopener" class="c-link c-link--branded" href="https://dev.to/t/rails">Ruby on Rails</a>. DEV Community <span title="copyright">©</span> 2016 - 2026.</p>
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
<div id="snack-zone"></div>
|
||
|
||
<div id="global-signup-modal" class="authentication-modal hidden">
|
||
<div class="authentication-modal__container">
|
||
<figure class="authentication-modal__image-container">
|
||
<img class="authentication-modal__image" src="https://media2.dev.to/dynamic/image/width=190,height=,fit=scale-down,gravity=auto,format=auto/https%3A%2F%2Fdev-to-uploads.s3.amazonaws.com%2Fuploads%2Farticles%2F8j7kvp660rqzt99zui8e.png"
|
||
alt="DEV Community" loading="lazy" />
|
||
</figure>
|
||
<div class="authentication-modal__content">
|
||
<p class="authentication-modal__description">
|
||
We're a place where coders share, stay up-to-date and grow their careers.
|
||
</p>
|
||
</div>
|
||
<div class="authentication-modal__actions">
|
||
<a href="https://dev.to/enter?signup_subforem=1" class="crayons-btn" aria-label="Log in" data-no-instant>
|
||
Log in
|
||
</a>
|
||
<a href="https://dev.to/enter?signup_subforem=1&state=new-user" class="crayons-btn crayons-btn--ghost-brand js-global-signup-modal__create-account" aria-label="Create new account" data-no-instant>
|
||
Create account
|
||
</a>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<script src="https://assets.dev.to/assets/signupModalShortcuts-0b25469b985100a01e94cbd7fccaf7f0a4d776e129aac65c766aa32cb28ab29a.js" defer="defer"></script>
|
||
|
||
<div id="cookie-consent"></div>
|
||
<div id="reaction-category-resources" class="hidden" aria-hidden="true">
|
||
<img data-name="Like" data-slug="like" data-position="1" src="https://assets.dev.to/assets/sparkle-heart-5f9bee3767e18deb1bb725290cb151c25234768a0e9a2bd39370c382d02920cf.svg" width="18" height="18" />
|
||
<img data-name="Unicorn" data-slug="unicorn" data-position="2" src="https://assets.dev.to/assets/multi-unicorn-b44d6f8c23cdd00964192bedc38af3e82463978aa611b4365bd33a0f1f4f3e97.svg" width="18" height="18" />
|
||
<img data-name="Exploding Head" data-slug="exploding_head" data-position="3" src="https://assets.dev.to/assets/exploding-head-daceb38d627e6ae9b730f36a1e390fca556a4289d5a41abb2c35068ad3e2c4b5.svg" width="18" height="18" />
|
||
<img data-name="Raised Hands" data-slug="raised_hands" data-position="4" src="https://assets.dev.to/assets/raised-hands-74b2099fd66a39f2d7eed9305ee0f4553df0eb7b4f11b01b6b1b499973048fe5.svg" width="18" height="18" />
|
||
<img data-name="Fire" data-slug="fire" data-position="5" src="https://assets.dev.to/assets/fire-f60e7a582391810302117f987b22a8ef04a2fe0df7e3258a5f49332df1cec71e.svg" width="18" height="18" />
|
||
</div>
|
||
|
||
<script>
|
||
var userSignedIn = false;
|
||
if (document.readyState === 'complete' || document.readyState === 'interactive') {
|
||
initAuth();
|
||
} else {
|
||
document.addEventListener('DOMContentLoaded', initAuth);
|
||
}
|
||
|
||
function initAuth() {
|
||
var paramToken = new URLSearchParams(window.location.search).get('jwt');
|
||
|
||
if (paramToken && !userSignedIn) {
|
||
authenticateUser(paramToken);
|
||
} else {
|
||
var iframe = document.createElement('iframe');
|
||
iframe.style.display = 'none';
|
||
iframe.src = 'https://forem.com/auth_pass/iframe?passed_domain=' + encodeURIComponent(window.location.origin || (window.location.protocol + '//' + window.location.host));
|
||
|
||
document.body.appendChild(iframe);
|
||
|
||
window.addEventListener('message', function(event) {
|
||
if (event.origin !== 'https://forem.com' && event.origin !== window.location.origin) {
|
||
return;
|
||
}
|
||
|
||
var data = event.data;
|
||
|
||
if (data.authenticated && !userSignedIn) {
|
||
authenticateUser(data.token);
|
||
} else if(data.authenticated && window.ReactNativeWebView && window.ReactNativeWebView.postMessage) {
|
||
window.ReactNativeWebView.postMessage(JSON.stringify({
|
||
action: 'login',
|
||
token: data.token,
|
||
}));
|
||
}
|
||
});
|
||
}
|
||
|
||
function authenticateUser(token) {
|
||
fetch('/auth_pass/token_login', {
|
||
method: 'POST',
|
||
credentials: 'include',
|
||
headers: {
|
||
'Content-Type': 'application/json',
|
||
'X-CSRF-Token': getMetaContent('csrf-token'),
|
||
},
|
||
body: JSON.stringify({ token: token }),
|
||
})
|
||
.then(function(response) {
|
||
return response.json();
|
||
})
|
||
.then(function(data) {
|
||
if (data.success) {
|
||
if (document.head.querySelector('meta[name="user-signed-in"][content="false"]')) {
|
||
// Reload the page to update the user's state
|
||
location.reload();
|
||
}
|
||
}
|
||
})
|
||
.catch(function(error) {
|
||
console.error('Error during authentication:', error);
|
||
});
|
||
}
|
||
|
||
function getMetaContent(name) {
|
||
var element = document.querySelector('meta[name="' + name + '"]');
|
||
return element ? element.getAttribute('content') : '';
|
||
}
|
||
}
|
||
</script>
|
||
|
||
|
||
|
||
|
||
</body>
|
||
</html>
|