1320 lines
91 KiB
HTML
1320 lines
91 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
|
|
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
|
<meta name="format-detection" content="telephone=no" />
|
|
|
|
|
|
<link rel="stylesheet" href="/blog/css/icomoon.css?v=HmvkT1_5XE7Qu8VBDM5zmqEK6jOiOqqN5gupI-5o_m8" />
|
|
<link rel="apple-touch-icon" sizes="180x180" href="/blog/images/apple-touch-icon.png">
|
|
<link rel="icon" type="image/png" sizes="32x32" href="/blog/images/favicon-32x32.png">
|
|
<link rel="icon" type="image/png" sizes="16x16" href="/blog/images/favicon-16x16.png">
|
|
<link rel="manifest" href="/blog/site.webmanifest">
|
|
<link rel="mask-icon" href="/blog/safari-pinned-tab.svg" color="#388ee9">
|
|
<meta name="msapplication-TileColor" content="#388ee9">
|
|
<meta name="theme-color" content="#388ee9">
|
|
|
|
<meta name="robots" content="noindex" />
|
|
<title>
|
|
Production postmortem: The heisenbug server - Ayende @ Rahien
|
|
</title>
|
|
|
|
|
|
<meta name="description" content="A production postmortem on a heisenbug server outage caused by an epoll_wait Linux kernel bug, traced with strace and ptrace.">
|
|
<meta name="keywords" content="production postmortem, heisenbug, Linux kernel bug, epoll_wait, strace fixes hung process, RavenDB networking issue, ptrace">
|
|
|
|
<meta property="og:site_name" content="Ayende @ Rahien">
|
|
<meta property="og:type" content="website">
|
|
<meta property="og:title" content="Production postmortem: The heisenbug server">
|
|
<meta property="og:description" content="A production postmortem on a heisenbug server outage caused by an epoll_wait Linux kernel bug, traced with strace and ptrace.">
|
|
<meta property="og:url" content="http://ayende.com/198849-C/production-postmortem-the-heisenbug-server?Key=1eeda567-02a8-4bbb-b90f-557523973233">
|
|
<meta property="og:image" content="http://ayende.com/blog/images/android-chrome-512x512.png">
|
|
|
|
<meta name="twitter:card" content="summary">
|
|
<meta name="twitter:creator" content="@ayende">
|
|
<meta name="twitter:site" content="@ayende">
|
|
<meta name="twitter:title" content="Production postmortem: The heisenbug server">
|
|
<meta name="twitter:description" content="A production postmortem on a heisenbug server outage caused by an epoll_wait Linux kernel bug, traced with strace and ptrace.">
|
|
<meta name="twitter:url" content="http://ayende.com/198849-C/production-postmortem-the-heisenbug-server?Key=1eeda567-02a8-4bbb-b90f-557523973233">
|
|
<meta name="twitter:image" content="http://ayende.com/blog/images/android-chrome-512x512.png">
|
|
|
|
<link rel="canonical" href="https://ravendb.net/articles/production-postmortem-the-heisenbug-server">
|
|
<script type="application/ld+json">{"@context":"https://schema.org","@type":"BlogPosting","headline":"Production postmortem: The heisenbug server","url":"https://ravendb.net/articles/production-postmortem-the-heisenbug-server","datePublished":"2023-01-16T12:00:00+00:00","dateCreated":"2023-01-13T12:43:00+00:00","description":"A production postmortem on a heisenbug server outage caused by an epoll_wait Linux kernel bug, traced with strace and ptrace.","author":{"@type":"Person","name":"Oren Eini"},"keywords":"production postmortem, heisenbug, Linux kernel bug, epoll_wait, strace fixes hung process, RavenDB networking issue, ptrace"}</script>
|
|
|
|
<link rel="stylesheet" href="/blog/css/custom/ayende.css" />
|
|
|
|
</head>
|
|
|
|
<body class="show-grid">
|
|
<div class="body-container">
|
|
<div class="leftSide">
|
|
<header>
|
|
<a class="logo" href="/blog">
|
|
<h1>Oren Eini</h1>
|
|
<span class="lead">aka Ayende Rahien</span>
|
|
</a>
|
|
<button class="menu-button" id="open-button"><i class="icon-arrow-down hide-sm" aria-hidden="true"></i></button>
|
|
|
|
</header>
|
|
|
|
<a class="logo-vertical" href="/blog">
|
|
<h1>Oren Eini</h1>
|
|
</a>
|
|
|
|
<button class="menu-enlarge" id="enlarge-button"><i class="icon-circle-arrow-left hide-sm" aria-hidden="true"></i></button>
|
|
<div class="collapseMobile flex-grow">
|
|
<div class="about flex-grow ">
|
|
|
|
<img src="/Blog/images/oren.jpg" class="oren-photo img-center" />
|
|
<div class="margin-top margin-bottom text-center">
|
|
<h2>CEO of RavenDB</h2>
|
|
<span class="lead">a NoSQL Open Source Document Database</span>
|
|
<div class="rewards margin-top">
|
|
<a class="mvp-profile-horizontal" href="https://mvp.microsoft.com/en-us/PublicProfile/4012807" title="Microsoft MVP Reconnect Oren Eini" target="_blank" rel="nofollow"><img src="/Blog/css/img/mvp-horizontal.svg" /></a>
|
|
<a class="dzone-profile-horizontal" href="https://dzone.com/users/750579/ayende.html" title="DZone 2020 Hall of Fame Award Winner" target="_blank" rel="nofollow"><img src="/Blog/css/img/dzone-horizontal.svg" /></a>
|
|
</div>
|
|
</div>
|
|
<div class="margin-top text-center">
|
|
<p>Get in touch with me:</p>
|
|
<div class="d-flex flex-wrap justify-content-center gap-3">
|
|
<small class="mail"><img src="/Blog/css/img/ico_mail.svg" class="icon-about" /><a href="mailto:oren@ravendb.net">oren@ravendb.net</a></small>
|
|
<small class="phone"><img src="/Blog/css/img/ico_phone.svg" class="icon-about" />+972 52-548-6969</small>
|
|
</div>
|
|
<p class="d-flex gap-3 justify-content-center flex-wrap icomoon margin-top">
|
|
<a href="https://twitter.com/ayende" class="icon-twitter"></a>
|
|
<a href="https://github.com/ayende" class="icon-github"></a>
|
|
<a href="mailto:ayende@ayende.com" class="icon-mail"></a>
|
|
<a href="https://www.linkedin.com/in/ravendb" class="icon-linkedin"></a>
|
|
<a href="https://www.facebook.com/pages/RavenDB/265907650186374" class="icon-facebook"></a>
|
|
<a href="http://feeds.feedburner.com/AyendeRahien" class="icon-rss"> </a>
|
|
</p>
|
|
</div>
|
|
|
|
</div>
|
|
<div class="flex-grow"></div>
|
|
<div class="rewards margin-top">
|
|
<a class="mvp-profile-vertical" href="https://mvp.microsoft.com/en-us/PublicProfile/4012807" title="Microsoft MVP Reconnect Oren Eini" target="_blank" rel="nofollow"><img src="/blog/css/img/mvp-vertical.svg" /></a>
|
|
<a class="dzone-profile-vertical" href="https://dzone.com/users/750579/ayende.html" title="DZone 2020 Hall of Fame Award Winner" target="_blank" rel="nofollow"><img src="/blog/css/img/dzone-vertical.svg" /></a>
|
|
</div>
|
|
<div class="pull-down">
|
|
|
|
<div class="hstack flex-wrap justify-content-center margin-bottom">
|
|
<div class="vstack">
|
|
Posts: <strong>7,648</strong>
|
|
</div>
|
|
<span class="separator">|</span>
|
|
<div class="vstack">
|
|
Comments: <strong>51,275</strong>
|
|
</div>
|
|
</div>
|
|
|
|
<p class="copyright">Copyright ©️ Ayende Rahien 2004 — 2026</p>
|
|
<small class="text-muted"><a href="/blog/privacy-policy" class="legal-link">Privacy Policy</a></small>
|
|
<span class="separator margin-left-sm margin-right-sm">·</span>
|
|
<small class="text-muted"><a href="/blog/terms" class="legal-link">Terms</a></small>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div id="content" class="centerCol flex-grow">
|
|
<header class="body-header">
|
|
<div class="search">
|
|
<form method="get" action="/blog/search">
|
|
<input name="q" type="search" placeholder="search" class="form-control" />
|
|
<input type="submit" />
|
|
</form>
|
|
</div>
|
|
<div style="display:inline-block">
|
|
<a href="#" id="tags-button">
|
|
<span class="icomoon-position icon-tags" aria-hidden="true"></span> filter by tags
|
|
</a>
|
|
<a href="#" id="archive-button">
|
|
<span class="icomoon-position icon-archive" aria-hidden="true"></span> archive
|
|
</a>
|
|
</div>
|
|
<nav class="pull-right views hide">
|
|
<a href="#" class="stackView" id="stackView">stack view</a>
|
|
<a href="#" class="gridView active" id="gridView">grid view</a>
|
|
</nav>
|
|
|
|
<div class="clearfix"></div>
|
|
|
|
<nav id="tags">
|
|
<ul>
|
|
<li>
|
|
<a href="/Blog/tags/architecture">architecture</a> <span>(635)</span> <a href="/Blog/rss/architecture" class="rss">rss</a>
|
|
</li>
|
|
<li>
|
|
<a href="/Blog/tags/bugs">bugs</a> <span>(451)</span> <a href="/Blog/rss/bugs" class="rss">rss</a>
|
|
</li>
|
|
<li>
|
|
<a href="/Blog/tags/challenges">challenges</a> <span>(137)</span> <a href="/Blog/rss/challenges" class="rss">rss</a>
|
|
</li>
|
|
<li>
|
|
<a href="/Blog/tags/community">community</a> <span>(391)</span> <a href="/Blog/rss/community" class="rss">rss</a>
|
|
</li>
|
|
<li>
|
|
<a href="/Blog/tags/databases">databases</a> <span>(483)</span> <a href="/Blog/rss/databases" class="rss">rss</a>
|
|
</li>
|
|
<li>
|
|
<a href="/Blog/tags/design">design</a> <span>(908)</span> <a href="/Blog/rss/design" class="rss">rss</a>
|
|
</li>
|
|
<li>
|
|
<a href="/Blog/tags/development">development</a> <span>(674)</span> <a href="/Blog/rss/development" class="rss">rss</a>
|
|
</li>
|
|
<li>
|
|
<a href="/Blog/tags/hibernating-practices">hibernating-practices</a> <span>(75)</span> <a href="/Blog/rss/hibernating-practices" class="rss">rss</a>
|
|
</li>
|
|
<li>
|
|
<a href="/Blog/tags/miscellaneous">miscellaneous</a> <span>(593)</span> <a href="/Blog/rss/miscellaneous" class="rss">rss</a>
|
|
</li>
|
|
<li>
|
|
<a href="/Blog/tags/performance">performance</a> <span>(399)</span> <a href="/Blog/rss/performance" class="rss">rss</a>
|
|
</li>
|
|
<li>
|
|
<a href="/Blog/tags/programming">programming</a> <span>(1128)</span> <a href="/Blog/rss/programming" class="rss">rss</a>
|
|
</li>
|
|
<li>
|
|
<a href="/Blog/tags/raven">raven</a> <span>(1499)</span> <a href="/Blog/rss/raven" class="rss">rss</a>
|
|
</li>
|
|
<li>
|
|
<a href="/Blog/tags/ravendb-net">ravendb.net</a> <span>(589)</span> <a href="/Blog/rss/ravendb-net" class="rss">rss</a>
|
|
</li>
|
|
</ul>
|
|
</nav>
|
|
|
|
|
|
|
|
|
|
<nav id="archive">
|
|
<ul>
|
|
<li>
|
|
<a href="#">2026</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2026/9"> September </a> (1)</li>
|
|
<li><a href="/Blog/archive/2026/8"> August </a> (2)</li>
|
|
<li><a href="/Blog/archive/2026/7"> July </a> (3)</li>
|
|
<li><a href="/Blog/archive/2026/6"> June </a> (2)</li>
|
|
<li><a href="/Blog/archive/2026/5"> May </a> (2)</li>
|
|
<li><a href="/Blog/archive/2026/4"> April </a> (5)</li>
|
|
<li><a href="/Blog/archive/2026/2"> February </a> (4)</li>
|
|
<li><a href="/Blog/archive/2026/1"> January </a> (5)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2025</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2025/12"> December </a> (8)</li>
|
|
<li><a href="/Blog/archive/2025/11"> November </a> (4)</li>
|
|
<li><a href="/Blog/archive/2025/10"> October </a> (4)</li>
|
|
<li><a href="/Blog/archive/2025/9"> September </a> (10)</li>
|
|
<li><a href="/Blog/archive/2025/8"> August </a> (6)</li>
|
|
<li><a href="/Blog/archive/2025/7"> July </a> (7)</li>
|
|
<li><a href="/Blog/archive/2025/6"> June </a> (7)</li>
|
|
<li><a href="/Blog/archive/2025/5"> May </a> (10)</li>
|
|
<li><a href="/Blog/archive/2025/4"> April </a> (10)</li>
|
|
<li><a href="/Blog/archive/2025/3"> March </a> (10)</li>
|
|
<li><a href="/Blog/archive/2025/2"> February </a> (7)</li>
|
|
<li><a href="/Blog/archive/2025/1"> January </a> (12)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2024</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2024/12"> December </a> (3)</li>
|
|
<li><a href="/Blog/archive/2024/11"> November </a> (2)</li>
|
|
<li><a href="/Blog/archive/2024/10"> October </a> (1)</li>
|
|
<li><a href="/Blog/archive/2024/9"> September </a> (3)</li>
|
|
<li><a href="/Blog/archive/2024/8"> August </a> (5)</li>
|
|
<li><a href="/Blog/archive/2024/7"> July </a> (10)</li>
|
|
<li><a href="/Blog/archive/2024/6"> June </a> (4)</li>
|
|
<li><a href="/Blog/archive/2024/5"> May </a> (6)</li>
|
|
<li><a href="/Blog/archive/2024/4"> April </a> (2)</li>
|
|
<li><a href="/Blog/archive/2024/3"> March </a> (8)</li>
|
|
<li><a href="/Blog/archive/2024/2"> February </a> (2)</li>
|
|
<li><a href="/Blog/archive/2024/1"> January </a> (14)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2023</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2023/12"> December </a> (4)</li>
|
|
<li><a href="/Blog/archive/2023/10"> October </a> (4)</li>
|
|
<li><a href="/Blog/archive/2023/9"> September </a> (6)</li>
|
|
<li><a href="/Blog/archive/2023/8"> August </a> (12)</li>
|
|
<li><a href="/Blog/archive/2023/7"> July </a> (5)</li>
|
|
<li><a href="/Blog/archive/2023/6"> June </a> (15)</li>
|
|
<li><a href="/Blog/archive/2023/5"> May </a> (3)</li>
|
|
<li><a href="/Blog/archive/2023/4"> April </a> (11)</li>
|
|
<li><a href="/Blog/archive/2023/3"> March </a> (5)</li>
|
|
<li><a href="/Blog/archive/2023/2"> February </a> (5)</li>
|
|
<li><a href="/Blog/archive/2023/1"> January </a> (8)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2022</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2022/12"> December </a> (5)</li>
|
|
<li><a href="/Blog/archive/2022/11"> November </a> (7)</li>
|
|
<li><a href="/Blog/archive/2022/10"> October </a> (7)</li>
|
|
<li><a href="/Blog/archive/2022/9"> September </a> (9)</li>
|
|
<li><a href="/Blog/archive/2022/8"> August </a> (10)</li>
|
|
<li><a href="/Blog/archive/2022/7"> July </a> (15)</li>
|
|
<li><a href="/Blog/archive/2022/6"> June </a> (12)</li>
|
|
<li><a href="/Blog/archive/2022/5"> May </a> (9)</li>
|
|
<li><a href="/Blog/archive/2022/4"> April </a> (14)</li>
|
|
<li><a href="/Blog/archive/2022/3"> March </a> (15)</li>
|
|
<li><a href="/Blog/archive/2022/2"> February </a> (13)</li>
|
|
<li><a href="/Blog/archive/2022/1"> January </a> (16)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2021</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2021/12"> December </a> (23)</li>
|
|
<li><a href="/Blog/archive/2021/11"> November </a> (20)</li>
|
|
<li><a href="/Blog/archive/2021/10"> October </a> (16)</li>
|
|
<li><a href="/Blog/archive/2021/9"> September </a> (6)</li>
|
|
<li><a href="/Blog/archive/2021/8"> August </a> (16)</li>
|
|
<li><a href="/Blog/archive/2021/7"> July </a> (11)</li>
|
|
<li><a href="/Blog/archive/2021/6"> June </a> (16)</li>
|
|
<li><a href="/Blog/archive/2021/5"> May </a> (4)</li>
|
|
<li><a href="/Blog/archive/2021/4"> April </a> (10)</li>
|
|
<li><a href="/Blog/archive/2021/3"> March </a> (11)</li>
|
|
<li><a href="/Blog/archive/2021/2"> February </a> (15)</li>
|
|
<li><a href="/Blog/archive/2021/1"> January </a> (14)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2020</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2020/12"> December </a> (10)</li>
|
|
<li><a href="/Blog/archive/2020/11"> November </a> (13)</li>
|
|
<li><a href="/Blog/archive/2020/10"> October </a> (15)</li>
|
|
<li><a href="/Blog/archive/2020/9"> September </a> (6)</li>
|
|
<li><a href="/Blog/archive/2020/8"> August </a> (9)</li>
|
|
<li><a href="/Blog/archive/2020/7"> July </a> (9)</li>
|
|
<li><a href="/Blog/archive/2020/6"> June </a> (17)</li>
|
|
<li><a href="/Blog/archive/2020/5"> May </a> (15)</li>
|
|
<li><a href="/Blog/archive/2020/4"> April </a> (14)</li>
|
|
<li><a href="/Blog/archive/2020/3"> March </a> (21)</li>
|
|
<li><a href="/Blog/archive/2020/2"> February </a> (16)</li>
|
|
<li><a href="/Blog/archive/2020/1"> January </a> (13)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2019</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2019/12"> December </a> (17)</li>
|
|
<li><a href="/Blog/archive/2019/11"> November </a> (14)</li>
|
|
<li><a href="/Blog/archive/2019/10"> October </a> (16)</li>
|
|
<li><a href="/Blog/archive/2019/9"> September </a> (10)</li>
|
|
<li><a href="/Blog/archive/2019/8"> August </a> (8)</li>
|
|
<li><a href="/Blog/archive/2019/7"> July </a> (16)</li>
|
|
<li><a href="/Blog/archive/2019/6"> June </a> (11)</li>
|
|
<li><a href="/Blog/archive/2019/5"> May </a> (13)</li>
|
|
<li><a href="/Blog/archive/2019/4"> April </a> (18)</li>
|
|
<li><a href="/Blog/archive/2019/3"> March </a> (12)</li>
|
|
<li><a href="/Blog/archive/2019/2"> February </a> (19)</li>
|
|
<li><a href="/Blog/archive/2019/1"> January </a> (23)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2018</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2018/12"> December </a> (15)</li>
|
|
<li><a href="/Blog/archive/2018/11"> November </a> (14)</li>
|
|
<li><a href="/Blog/archive/2018/10"> October </a> (19)</li>
|
|
<li><a href="/Blog/archive/2018/9"> September </a> (18)</li>
|
|
<li><a href="/Blog/archive/2018/8"> August </a> (23)</li>
|
|
<li><a href="/Blog/archive/2018/7"> July </a> (20)</li>
|
|
<li><a href="/Blog/archive/2018/6"> June </a> (20)</li>
|
|
<li><a href="/Blog/archive/2018/5"> May </a> (23)</li>
|
|
<li><a href="/Blog/archive/2018/4"> April </a> (15)</li>
|
|
<li><a href="/Blog/archive/2018/3"> March </a> (23)</li>
|
|
<li><a href="/Blog/archive/2018/2"> February </a> (19)</li>
|
|
<li><a href="/Blog/archive/2018/1"> January </a> (23)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2017</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2017/12"> December </a> (21)</li>
|
|
<li><a href="/Blog/archive/2017/11"> November </a> (24)</li>
|
|
<li><a href="/Blog/archive/2017/10"> October </a> (22)</li>
|
|
<li><a href="/Blog/archive/2017/9"> September </a> (21)</li>
|
|
<li><a href="/Blog/archive/2017/8"> August </a> (23)</li>
|
|
<li><a href="/Blog/archive/2017/7"> July </a> (21)</li>
|
|
<li><a href="/Blog/archive/2017/6"> June </a> (24)</li>
|
|
<li><a href="/Blog/archive/2017/5"> May </a> (21)</li>
|
|
<li><a href="/Blog/archive/2017/4"> April </a> (21)</li>
|
|
<li><a href="/Blog/archive/2017/3"> March </a> (23)</li>
|
|
<li><a href="/Blog/archive/2017/2"> February </a> (20)</li>
|
|
<li><a href="/Blog/archive/2017/1"> January </a> (23)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2016</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2016/12"> December </a> (17)</li>
|
|
<li><a href="/Blog/archive/2016/11"> November </a> (18)</li>
|
|
<li><a href="/Blog/archive/2016/10"> October </a> (22)</li>
|
|
<li><a href="/Blog/archive/2016/9"> September </a> (18)</li>
|
|
<li><a href="/Blog/archive/2016/8"> August </a> (23)</li>
|
|
<li><a href="/Blog/archive/2016/7"> July </a> (22)</li>
|
|
<li><a href="/Blog/archive/2016/6"> June </a> (17)</li>
|
|
<li><a href="/Blog/archive/2016/5"> May </a> (24)</li>
|
|
<li><a href="/Blog/archive/2016/4"> April </a> (16)</li>
|
|
<li><a href="/Blog/archive/2016/3"> March </a> (16)</li>
|
|
<li><a href="/Blog/archive/2016/2"> February </a> (21)</li>
|
|
<li><a href="/Blog/archive/2016/1"> January </a> (21)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2015</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2015/12"> December </a> (5)</li>
|
|
<li><a href="/Blog/archive/2015/11"> November </a> (10)</li>
|
|
<li><a href="/Blog/archive/2015/10"> October </a> (9)</li>
|
|
<li><a href="/Blog/archive/2015/9"> September </a> (17)</li>
|
|
<li><a href="/Blog/archive/2015/8"> August </a> (20)</li>
|
|
<li><a href="/Blog/archive/2015/7"> July </a> (17)</li>
|
|
<li><a href="/Blog/archive/2015/6"> June </a> (4)</li>
|
|
<li><a href="/Blog/archive/2015/5"> May </a> (12)</li>
|
|
<li><a href="/Blog/archive/2015/4"> April </a> (9)</li>
|
|
<li><a href="/Blog/archive/2015/3"> March </a> (8)</li>
|
|
<li><a href="/Blog/archive/2015/2"> February </a> (25)</li>
|
|
<li><a href="/Blog/archive/2015/1"> January </a> (17)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2014</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2014/12"> December </a> (22)</li>
|
|
<li><a href="/Blog/archive/2014/11"> November </a> (19)</li>
|
|
<li><a href="/Blog/archive/2014/10"> October </a> (21)</li>
|
|
<li><a href="/Blog/archive/2014/9"> September </a> (37)</li>
|
|
<li><a href="/Blog/archive/2014/8"> August </a> (24)</li>
|
|
<li><a href="/Blog/archive/2014/7"> July </a> (23)</li>
|
|
<li><a href="/Blog/archive/2014/6"> June </a> (13)</li>
|
|
<li><a href="/Blog/archive/2014/5"> May </a> (19)</li>
|
|
<li><a href="/Blog/archive/2014/4"> April </a> (24)</li>
|
|
<li><a href="/Blog/archive/2014/3"> March </a> (23)</li>
|
|
<li><a href="/Blog/archive/2014/2"> February </a> (21)</li>
|
|
<li><a href="/Blog/archive/2014/1"> January </a> (24)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2013</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2013/12"> December </a> (23)</li>
|
|
<li><a href="/Blog/archive/2013/11"> November </a> (29)</li>
|
|
<li><a href="/Blog/archive/2013/10"> October </a> (27)</li>
|
|
<li><a href="/Blog/archive/2013/9"> September </a> (26)</li>
|
|
<li><a href="/Blog/archive/2013/8"> August </a> (24)</li>
|
|
<li><a href="/Blog/archive/2013/7"> July </a> (24)</li>
|
|
<li><a href="/Blog/archive/2013/6"> June </a> (23)</li>
|
|
<li><a href="/Blog/archive/2013/5"> May </a> (25)</li>
|
|
<li><a href="/Blog/archive/2013/4"> April </a> (26)</li>
|
|
<li><a href="/Blog/archive/2013/3"> March </a> (24)</li>
|
|
<li><a href="/Blog/archive/2013/2"> February </a> (24)</li>
|
|
<li><a href="/Blog/archive/2013/1"> January </a> (21)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2012</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2012/12"> December </a> (19)</li>
|
|
<li><a href="/Blog/archive/2012/11"> November </a> (22)</li>
|
|
<li><a href="/Blog/archive/2012/10"> October </a> (27)</li>
|
|
<li><a href="/Blog/archive/2012/9"> September </a> (24)</li>
|
|
<li><a href="/Blog/archive/2012/8"> August </a> (30)</li>
|
|
<li><a href="/Blog/archive/2012/7"> July </a> (23)</li>
|
|
<li><a href="/Blog/archive/2012/6"> June </a> (25)</li>
|
|
<li><a href="/Blog/archive/2012/5"> May </a> (23)</li>
|
|
<li><a href="/Blog/archive/2012/4"> April </a> (25)</li>
|
|
<li><a href="/Blog/archive/2012/3"> March </a> (25)</li>
|
|
<li><a href="/Blog/archive/2012/2"> February </a> (28)</li>
|
|
<li><a href="/Blog/archive/2012/1"> January </a> (24)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2011</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2011/12"> December </a> (17)</li>
|
|
<li><a href="/Blog/archive/2011/11"> November </a> (14)</li>
|
|
<li><a href="/Blog/archive/2011/10"> October </a> (24)</li>
|
|
<li><a href="/Blog/archive/2011/9"> September </a> (28)</li>
|
|
<li><a href="/Blog/archive/2011/8"> August </a> (27)</li>
|
|
<li><a href="/Blog/archive/2011/7"> July </a> (30)</li>
|
|
<li><a href="/Blog/archive/2011/6"> June </a> (19)</li>
|
|
<li><a href="/Blog/archive/2011/5"> May </a> (16)</li>
|
|
<li><a href="/Blog/archive/2011/4"> April </a> (30)</li>
|
|
<li><a href="/Blog/archive/2011/3"> March </a> (23)</li>
|
|
<li><a href="/Blog/archive/2011/2"> February </a> (11)</li>
|
|
<li><a href="/Blog/archive/2011/1"> January </a> (26)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2010</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2010/12"> December </a> (29)</li>
|
|
<li><a href="/Blog/archive/2010/11"> November </a> (28)</li>
|
|
<li><a href="/Blog/archive/2010/10"> October </a> (35)</li>
|
|
<li><a href="/Blog/archive/2010/9"> September </a> (33)</li>
|
|
<li><a href="/Blog/archive/2010/8"> August </a> (44)</li>
|
|
<li><a href="/Blog/archive/2010/7"> July </a> (17)</li>
|
|
<li><a href="/Blog/archive/2010/6"> June </a> (20)</li>
|
|
<li><a href="/Blog/archive/2010/5"> May </a> (53)</li>
|
|
<li><a href="/Blog/archive/2010/4"> April </a> (29)</li>
|
|
<li><a href="/Blog/archive/2010/3"> March </a> (35)</li>
|
|
<li><a href="/Blog/archive/2010/2"> February </a> (33)</li>
|
|
<li><a href="/Blog/archive/2010/1"> January </a> (36)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2009</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2009/12"> December </a> (37)</li>
|
|
<li><a href="/Blog/archive/2009/11"> November </a> (35)</li>
|
|
<li><a href="/Blog/archive/2009/10"> October </a> (53)</li>
|
|
<li><a href="/Blog/archive/2009/9"> September </a> (60)</li>
|
|
<li><a href="/Blog/archive/2009/8"> August </a> (66)</li>
|
|
<li><a href="/Blog/archive/2009/7"> July </a> (29)</li>
|
|
<li><a href="/Blog/archive/2009/6"> June </a> (24)</li>
|
|
<li><a href="/Blog/archive/2009/5"> May </a> (52)</li>
|
|
<li><a href="/Blog/archive/2009/4"> April </a> (63)</li>
|
|
<li><a href="/Blog/archive/2009/3"> March </a> (35)</li>
|
|
<li><a href="/Blog/archive/2009/2"> February </a> (53)</li>
|
|
<li><a href="/Blog/archive/2009/1"> January </a> (50)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2008</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2008/12"> December </a> (58)</li>
|
|
<li><a href="/Blog/archive/2008/11"> November </a> (65)</li>
|
|
<li><a href="/Blog/archive/2008/10"> October </a> (46)</li>
|
|
<li><a href="/Blog/archive/2008/9"> September </a> (48)</li>
|
|
<li><a href="/Blog/archive/2008/8"> August </a> (96)</li>
|
|
<li><a href="/Blog/archive/2008/7"> July </a> (87)</li>
|
|
<li><a href="/Blog/archive/2008/6"> June </a> (45)</li>
|
|
<li><a href="/Blog/archive/2008/5"> May </a> (51)</li>
|
|
<li><a href="/Blog/archive/2008/4"> April </a> (52)</li>
|
|
<li><a href="/Blog/archive/2008/3"> March </a> (70)</li>
|
|
<li><a href="/Blog/archive/2008/2"> February </a> (43)</li>
|
|
<li><a href="/Blog/archive/2008/1"> January </a> (49)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2007</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2007/12"> December </a> (100)</li>
|
|
<li><a href="/Blog/archive/2007/11"> November </a> (52)</li>
|
|
<li><a href="/Blog/archive/2007/10"> October </a> (109)</li>
|
|
<li><a href="/Blog/archive/2007/9"> September </a> (68)</li>
|
|
<li><a href="/Blog/archive/2007/8"> August </a> (80)</li>
|
|
<li><a href="/Blog/archive/2007/7"> July </a> (56)</li>
|
|
<li><a href="/Blog/archive/2007/6"> June </a> (150)</li>
|
|
<li><a href="/Blog/archive/2007/5"> May </a> (115)</li>
|
|
<li><a href="/Blog/archive/2007/4"> April </a> (73)</li>
|
|
<li><a href="/Blog/archive/2007/3"> March </a> (124)</li>
|
|
<li><a href="/Blog/archive/2007/2"> February </a> (102)</li>
|
|
<li><a href="/Blog/archive/2007/1"> January </a> (68)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2006</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2006/12"> December </a> (95)</li>
|
|
<li><a href="/Blog/archive/2006/11"> November </a> (53)</li>
|
|
<li><a href="/Blog/archive/2006/10"> October </a> (120)</li>
|
|
<li><a href="/Blog/archive/2006/9"> September </a> (57)</li>
|
|
<li><a href="/Blog/archive/2006/8"> August </a> (88)</li>
|
|
<li><a href="/Blog/archive/2006/7"> July </a> (54)</li>
|
|
<li><a href="/Blog/archive/2006/6"> June </a> (103)</li>
|
|
<li><a href="/Blog/archive/2006/5"> May </a> (89)</li>
|
|
<li><a href="/Blog/archive/2006/4"> April </a> (84)</li>
|
|
<li><a href="/Blog/archive/2006/3"> March </a> (143)</li>
|
|
<li><a href="/Blog/archive/2006/2"> February </a> (78)</li>
|
|
<li><a href="/Blog/archive/2006/1"> January </a> (64)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2005</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2005/12"> December </a> (70)</li>
|
|
<li><a href="/Blog/archive/2005/11"> November </a> (97)</li>
|
|
<li><a href="/Blog/archive/2005/10"> October </a> (91)</li>
|
|
<li><a href="/Blog/archive/2005/9"> September </a> (61)</li>
|
|
<li><a href="/Blog/archive/2005/8"> August </a> (74)</li>
|
|
<li><a href="/Blog/archive/2005/7"> July </a> (92)</li>
|
|
<li><a href="/Blog/archive/2005/6"> June </a> (100)</li>
|
|
<li><a href="/Blog/archive/2005/5"> May </a> (53)</li>
|
|
<li><a href="/Blog/archive/2005/4"> April </a> (42)</li>
|
|
<li><a href="/Blog/archive/2005/3"> March </a> (41)</li>
|
|
<li><a href="/Blog/archive/2005/2"> February </a> (84)</li>
|
|
<li><a href="/Blog/archive/2005/1"> January </a> (31)</li>
|
|
</ul>
|
|
</li>
|
|
<li>
|
|
<a href="#">2004</a>
|
|
<ul>
|
|
<li><a href="/Blog/archive/2004/12"> December </a> (49)</li>
|
|
<li><a href="/Blog/archive/2004/11"> November </a> (26)</li>
|
|
<li><a href="/Blog/archive/2004/10"> October </a> (26)</li>
|
|
<li><a href="/Blog/archive/2004/9"> September </a> (6)</li>
|
|
<li><a href="/Blog/archive/2004/4"> April </a> (10)</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</nav>
|
|
|
|
|
|
</header>
|
|
<div class="bnr-carousel">
|
|
<a href="https://ravendb.net/whitepapers/couchbase-vs-ravendb-performance-at-rakuten-kobo?utm_source=ayende_blog&utm_medium=banner" target="_blank"><img src="/blog/banners/38da6f0eb15042f7aea05e6d29fc7828.png?v=5a4b8c06-662e-46aa-a948-742fc990686b" class="img-responsive img-center" title="Couchbase vs RavenDB Performance at Rakuten Kobo Whitepaper" alt="Couchbase vs RavenDB Performance at Rakuten Kobo Whitepaper" /></a>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<article class="article-body">
|
|
|
|
|
|
<nav class="nextPrev">
|
|
<a class="pull-left" href="/blog/198817-c/ravendb-sharding-webinar-recording">
|
|
<span class="icomoon-position icon-chevron-left"></span><span class="hidden-xs"> previous post</span>
|
|
</a>
|
|
|
|
<a class="pull-right" href="/blog/198881-c/production-postmortem-the-big-server-that-couldn-t-handle-the-load">
|
|
<span class="hidden-xs">next post </span>
|
|
<span class="icomoon-position icon-chevron-right"></span>
|
|
</a>
|
|
|
|
<a href="/blog/198305-b/production-postmortem-do-you-trust-this-server" class="pull-left series">
|
|
<span class="icomoon-position icon-chevron-left"></span>
|
|
<span class="hidden-xs"> previous series post</span>
|
|
</a>
|
|
|
|
<a href="/blog/posts/series/171074/production-postmortem" class="pull-left series">
|
|
<i class="icon-book"></i>
|
|
<span class="hidden-xs">all </span>series<span class="hidden-xs"> posts</span>
|
|
</a>
|
|
|
|
<a href="/blog/198881-c/production-postmortem-the-big-server-that-couldn-t-handle-the-load" class="pull-right series">
|
|
<span class="hidden-xs"> next series post</span>
|
|
<span class="icomoon-position icon-chevron-right"></span>
|
|
</a>
|
|
|
|
<div class="clearfix"></div>
|
|
</nav>
|
|
|
|
<header>
|
|
<time>
|
|
<strong>Jan 16</strong>
|
|
<span>2023</span>
|
|
</time>
|
|
<a href="/blog/198849-c/production-postmortem-the-heisenbug-server">
|
|
<h2><span class="seriesTitle">Production postmortem</span>The heisenbug server</h2>
|
|
</a>
|
|
</header>
|
|
<div class="timetoread">
|
|
<div>
|
|
<span class="icomoon-position icon-time" aria-hidden="true"></span>
|
|
time to read <strong class="force">10 min</strong> | <strong class="force">1995 words</strong>
|
|
</div>
|
|
</div>
|
|
<hr />
|
|
<div class="articleContent clearfix">
|
|
<div class="text-wrapper">
|
|
<p><a href="https://ayende.com/blog/Images/Open-Live-Writer/Production-postmortem-The-hiesien_ABC0/image_2.png"><img style="border: 0px currentcolor; float: right; display: inline; background-image: none;" title="image" src="/blog/Images/Open-Live-Writer/Production-postmortem-The-hiesien_ABC0/image_thumb.png" alt="image" width="284" height="284" align="right" border="0"></a>A user reported that they observed nodes in the cluster “going dark”. Basically, they would stop communicating with the rest of the cluster, but would otherwise appear functional. Both the internal and external metrics were all fine, the server would just stop responding to anything over the network. The solution for the problem was to restart the service (note, the service, not the whole machine), but the problem would happen every few days.</p>
|
|
<p>As you can imagine, we are taking this sort of thing <em>very</em> seriously, so we looked into the problem. And we came up short. The problem made <em>absolutely</em> no sense. The problem occurred on a (minor) version migration, but there was absolutely nothing related to this that could cause it. What was <em>really</em> weird was that the <em>service </em>itself continue to work. We could see log entries being written and it was able to execute scheduled backups, for example. It would just refuse to talk to us over the network.</p>
|
|
<p>That was <em>super</em> strange, since the <em>network </em>itself was fine. All the monitoring systems were green, after all. For that matter, the user was able to SSH into the system to restart the service. This didn’t match with any other issue we could think of. Since the user worked around the problem by restarting the server, we didn’t have a lead.</p>
|
|
<p>Then we noticed the exact same problem in one of our cloud instances, and there we have much better diagnostic capabilities. Once we had noticed a problematic server, we were able to SSH into that and try to figure out what was going on.</p>
|
|
<p>Here is what we found out:</p>
|
|
<ul>
|
|
<li>The server will not respond to HTTP(s) communication either from outside the machine or by trying to connect from <em>inside</em> the machine.</li>
|
|
<li>The server <em>will</em> respond to SNMP queries both from inside the machine and outside of it (which is how we typically monitor the system).</li>
|
|
</ul>
|
|
<p>When we designed RavenDB, we implemented a “maintenance hatch” for such scenarios, in addition to using HTTP(s) for communication, RavenDB also exposes a named pipe that allows you to connect to the server without going through the network at all. This ensures that if you have administrator privileges on the server, you are able to connect even if there are network issues, certificate problems, etc.</p>
|
|
<p>Here is the kicker. Under this particular situation, we <em>could</em> <em>not </em>activate this escape hatch. That is not supposed to be possible. Named pipes on Linux, where we run into the problem, are basically Unix Sockets. A network issue such as a firewall problem or something similar isn’t going to affect them.</p>
|
|
<p>At the same time, we were able to communicate with the process using SNMP. What is the problem?</p>
|
|
<p>Lacking any other options, we dumped the process, restarted the service, and tried to do the analysis offline. We couldn’t find any problem. All the details we looked at said that everything was fine, the server <em>was</em> properly listening to new connections and it <em>should</em> work. That was… weird.</p>
|
|
<p>And then it happened again, and we did the same analysis, and it came back the same. We were clueless. One of the things that we updated between versions was the .NET runtime that we were using, so we <a href="https://github.com/dotnet/runtime/issues/79446">opened an issue</a> to see if anyone ran into the same problem.</p>
|
|
<p>And then it happened again. This time, we knew that just looking at the dump wouldn’t help us, so we tried other avenues. Linux has a pretty rich set of knobs and dials that you can look at to see what was going on. We suspected that this may be an issue with running out of file descriptors, running out of memory, etc.</p>
|
|
<p>We tried looking into what is going on inside the process using <em>strace</em>, and everything was <strong>fine</strong>. The trace clearly showed that the server was processing requests and was able to send and receive data properly.</p>
|
|
<p>Wait, go through that statement again please!</p>
|
|
<p>It is fine? But the reason we are using <em>strace</em> is that there is a problem. It looks like the problem fixed itself. That was annoying, because we were hoping to use the trace to figure out what is going on. We added more monitoring along the way, which would let us know if the server found itself isolated. And we waited.</p>
|
|
<p>The next time we ran into the problem, the first thing we did was run <em>strace, </em>we needed to get the root cause as soon as possible, and we were afraid that it would fix itself before we had a chance to get to the root cause. The moment we used <em>strace</em>, the server got back online, continuing as if there was never any issue.</p>
|
|
<p>Over the next few instances of this issue, we were able to confirm the following observations:</p>
|
|
<ol>
|
|
<li>The service would stop responding to TCP and Unix Sockets entirely.</li>
|
|
<li>There were <em>no</em> firewall or network issues.</li>
|
|
<li>The service was up and functional, tailing the log showed activity.</li>
|
|
<li>We could query the server state using SNMP.</li>
|
|
<li>Running strace on the service process would fix the problem.</li>
|
|
</ol>
|
|
<p>There are a few more things, the actual trigger for the fix wasn’t strace itself. It was the <em>ptrace()</em> call, which it uses. That would cause the service to start responding again. The <em>ptrace() </em>call is basically the beginning and the end of debugging under Linux. <em>Everything </em>uses it.</p>
|
|
<p>If you want to dump a memory process, you start with <em>ptrace()</em>. You want to trace the calls, <em>ptrace(). </em>You want to <em>debug </em>the process? GDB will start by calling <em>ptrace()</em>, etc.</p>
|
|
<p>And doing that would alleviate the problem.</p>
|
|
<p>That was… quite annoying.</p>
|
|
<p>We still had absolutely no indication of what the root cause even was.</p>
|
|
<p>We suspected it may be something inside Kestrel that was causing a problem. But that wouldn’t affect the named pipes / Unix sockets that we also saw.</p>
|
|
<p>Networking <em>worked</em>, because SNMP did. We thought that this may be because SNMP uses UDP instead of TCP, and looked into that, but we couldn’t figure out how that would be any different.</p>
|
|
<p>Looking at this further, we found that we have this in the code dumps:</p>
|
|
<blockquote>
|
|
<pre><code> ~~~~ 5072
|
|
1 Interop+Sys.WaitForSocketEvents(IntPtr, SocketEvent*, Int32*)
|
|
1 System.Net.Sockets.SocketAsyncEngine.EventLoop()
|
|
1 System.Net.Sockets.SocketAsyncEngine+<>c.ctor>b__14_0(Object)</code></pre>
|
|
</blockquote>
|
|
<p>As you can see, we are waiting for this in the .NET Sockets thread. The SNMP, on the other hand, looked like:</p>
|
|
<blockquote>
|
|
<p><span style="font-family: 'Courier New';">Thread (0x559):<br> [Native Frames]<br> System.Net.Sockets!System.Net.Sockets.SocketPal.SysReceive()<br> System.Net.Sockets!System.Net.Sockets.SocketPal.TryCompleteReceiveFrom()<br> System.Net.Sockets!System.Net.Sockets.SocketAsyncContext.ReceiveFrom()<br> System.Net.Sockets!System.Net.Sockets.SocketPal.ReceiveFrom()<br> System.Net.Sockets!System.Net.Sockets.Socket.ReceiveFrom()<br> SharpSnmpLib.Engine!Lextm.SharpSnmpLib.Pipeline.ListenerBinding.AsyncReceive()<br> </span></p>
|
|
</blockquote>
|
|
<p>That was really interesting, since it meant that for sockets (both HTTP and Unix), we were always using async calls, but for SNMP, we were using the synchronous API. We initially suspected that this may be something related to the thread pool. Maybe we had something that blocked it, but it turns out to be a lot more interesting. Here is the code that is actually <em>handling</em> the SNMP:</p>
|
|
<blockquote>
|
|
<p>var count = _socket.ReceiveFrom(buffer, ref remote);</p>
|
|
<p>Task.Factory.StartNew(() => HandleMessage(buffer, count, (IPEndPoint)remote));</p>
|
|
</blockquote>
|
|
<p>In other words, we are actually reading from the socket in a blocking manner, but then processing the actual message using the thread pool. So being able to get results via SNMP meant the thread pool was well.</p>
|
|
<p>At this point we resulted to hair pulling, rubber ducking and in some instances, shaking our fists to heaven.</p>
|
|
<p>I reminded myself that I’m an adult with a bit of experience solving problems, and dug deeper. We started looking into how .NET is actually <a href="https://github.com/dotnet/runtime/blob/e52462326be03fb329384b7e04a33d3eb7c16736/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncEngine.Unix.cs#L176">handling sockets in async mode.</a> This end up <a href="https://github.com/dotnet/runtime/blob/e52462326be03fb329384b7e04a33d3eb7c16736/src/native/libs/System.Native/pal_networking.c#L2727">here, doing a system call</a>:</p>
|
|
<blockquote>
|
|
<p>while ((numEvents = epoll_wait(port, events, *count, -1)) < 0 && errno == EINTR);</p>
|
|
</blockquote>
|
|
<p>Reading through the man page for <em>epoll_wait()</em> I learned how <em>epoll</em>() works, that it is complex and that we need to be aware of level-triggered and edge-triggered options. Since .NET uses <a href="https://github.com/dotnet/runtime/blob/e52462326be03fb329384b7e04a33d3eb7c16736/src/native/libs/System.Native/pal_networking.c#LL2693C60-L2693C67">edge-triggered</a> events (EPOLLET, which I keep reading as electronic chicken), we focused on that.</p>
|
|
<p>There are a <em>lot</em> of edge cases and things to cover, but everything we checked was handled properly. We finally had a good smoking gun. For some reason, we weren’t getting notifications from <em>epoll</em>(), even though we should. Using <em>strace()</em> or friends somehow fixes that.</p>
|
|
<p>We actually <em>found</em> the <a href="https://stackoverflow.com/questions/20272845/strace-fixes-hung-process">exact scenario we saw in StackOverflow</a>, but without any idea what the issue was. <a href="https://xkcd.com/979/">Truly, there is an XKCD for everything</a>.</p>
|
|
<p>Our current understanding of the issue:</p>
|
|
<ul>
|
|
<li>All async sockets in .NET are going through the same socket engine, and are using <em>epoll</em>() under the covers.</li>
|
|
<li>SNMP is using <em>synchronous</em> calls, so it wasn’t using <em>epoll()</em>.</li>
|
|
</ul>
|
|
<p>That covers both of the weird things that we are seeing. So what is the issue?</p>
|
|
<p>It is not in .NET. Given the size & scope of .NET, we wouldn’t be the only ones seeing that. Below .NET, there is the kernel, so we looked into that. The machines we were running that on were using kernel 5.4.0-azure-1095, so we looked into that.</p>
|
|
<p>And it looked like it is <a href="https://lore.kernel.org/linux-fsdevel/a2f22c3c-c25a-4bda-8339-a7bdaf17849e@akamai.com/#r">a kernel bug</a>, which was fixed in the <a href="https://bugs.launchpad.net/ubuntu/+source/containerd/+bug/1996678/comments/28">next updated kernel</a>. A race condition inside the kernel would cause us to miss wakeups, and then we would basically just stall without anything to wake us up.</p>
|
|
<p>We dug deeper to understand a bit more about this situation, and <a href="https://man7.org/linux/man-pages/man2/ptrace.2.html">we got this</a>:</p>
|
|
<pre> Some system calls return with <strong>EINTR </strong>if a signal was sent to a
|
|
tracee, but delivery was suppressed by the tracer. (This is very
|
|
typical operation: it is usually done by debuggers on every
|
|
attach, in order to not introduce a bogus <strong>SIGSTOP</strong>). As of Linux
|
|
3.2.9, the following system calls are affected (this list is
|
|
likely incomplete): <a href="https://man7.org/linux/man-pages/man2/epoll_wait.2.html">epoll_wait(2)</a>, and <a href="https://man7.org/linux/man-pages/man2/read.2.html">read(2)</a> from an <a href="https://man7.org/linux/man-pages/man7/inotify.7.html">inotify(7)</a>
|
|
file descriptor. The usual symptom of this bug is that when you
|
|
attach to a quiescent process with the command
|
|
|
|
strace -p <process-ID>
|
|
|
|
then, instead of the usual and expected one-line output such as
|
|
|
|
restart_syscall(<... resuming interrupted call ...>_
|
|
|
|
or
|
|
|
|
select(6, [5], NULL, [5], NULL_
|
|
|
|
('_' denotes the cursor position), you observe more than one
|
|
line. For example:
|
|
|
|
clock_gettime(CLOCK_MONOTONIC, {15370, 690928118}) = 0
|
|
epoll_wait(4,_
|
|
|
|
What is not visible here is that the process was blocked in
|
|
<a href="https://man7.org/linux/man-pages/man2/epoll_wait.2.html">epoll_wait(2)</a> before <a href="https://man7.org/linux/man-pages/man1/strace.1.html">strace(1)</a> has attached to it. Attaching
|
|
caused <a href="https://man7.org/linux/man-pages/man2/epoll_wait.2.html">epoll_wait(2)</a> to return to user space with the error
|
|
<strong>EINTR</strong>. In this particular case, the program reacted to <strong>EINTR </strong>by
|
|
checking the current time, and then executing <a href="https://man7.org/linux/man-pages/man2/epoll_wait.2.html">epoll_wait(2)</a>
|
|
again. (Programs which do not expect such "stray" <strong>EINTR </strong>errors
|
|
may behave in an unintended way upon an <a href="https://man7.org/linux/man-pages/man1/strace.1.html">strace(1)</a> attach.)</pre>
|
|
<p>And.. that is exactly what is happening. On attaching, the <em>epoll_wait</em>() will return with <em>EINTR</em>, which will cause .NET to retry the command, and that “fixes” the issue.</p>
|
|
<p>It makes total sense now, and concludes the discovery process of a pretty nasty bug.</p>
|
|
<p>Now, if you’ll excuse me, I need to go and apologize to a rubber duck.</p>
|
|
<p><a href="https://ayende.com/blog/Images/Open-Live-Writer/Production-postmortem-The-hiesien_ABC0/image_4.png"><img style="margin: 0px; border: 0px currentcolor; display: inline; background-image: none;" title="image" src="/blog/Images/Open-Live-Writer/Production-postmortem-The-hiesien_ABC0/image_thumb_1.png" alt="image" width="228" height="229" border="0"></a></p>
|
|
</div>
|
|
</div>
|
|
<div class="postData">
|
|
<a href="#comments">
|
|
<span class="icomoon-position glyphicon icon-comment" aria-hidden="true"></span><span>0 comments</span>
|
|
</a>
|
|
<div class="tags pull-right-lg">
|
|
<span class="icomoon-position icon-tag" aria-hidden="true"></span> Tags:
|
|
<ul>
|
|
<li><a href="/blog/tags/bugs">bugs</a></li>
|
|
<li><a href="/blog/tags/challenges">challenges</a></li>
|
|
<li><a href="/blog/tags/databases">databases</a></li>
|
|
<li><a href="/blog/tags/ravendb-net">ravendb.net</a></li>
|
|
<li><a href="/blog/tags/raven">raven</a></li>
|
|
</ul>
|
|
</div>
|
|
<div class="clearfix"></div>
|
|
<hr style="margin: 0;" />
|
|
</div>
|
|
<div class="related-posts">
|
|
<h4 class="related-posts_heading">
|
|
<svg width="18" height="18" viewBox="0 0 658 876" fill="#388ee9" xmlns="http://www.w3.org/2000/svg">
|
|
<path d="M395.6 0.199219H0.400391V875.399H658V262.599L395.6 0.199219ZM352.4 65.7992L592.4 305.799H352.4V65.7992ZM509.2 720.199H150.8V632.199H510.8V720.199H509.2ZM509.2 545.799H150.8V457.799H510.8V545.799H509.2Z" />
|
|
</svg>
|
|
Related posts that you may find interesting:
|
|
</h4>
|
|
<div class="related-posts_items">
|
|
<div class="related-posts_item" tabindex="0" data-href="/blog/181953-a/production-postmortem-the-unavailable-linux-server">
|
|
<div class="related-posts_mainlink">
|
|
<span class="related-posts_date">22 Feb 2018</span>
|
|
<span class="related-posts_title">Production postmortem: The unavailable Linux server</span>
|
|
</div>
|
|
<div class="related-posts_tags">
|
|
<div class="related-posts_tags_header">
|
|
<div class="icomoon-position icon-tag" aria-hidden="true"></div>
|
|
</div>
|
|
<a class="related-posts_tag " href="/blog/tags/bugs">bugs</a>
|
|
<a class="related-posts_tag " href="/blog/tags/challenges">challenges</a>
|
|
<a class="related-posts_tag " href="/blog/tags/development">development</a>
|
|
<a class="related-posts_tag related-posts_tag--extra" href="/blog/tags/raven">raven</a>
|
|
<span class="related-posts_tag_extra">+1</span>
|
|
</div>
|
|
</div>
|
|
<div class="related-posts_item" tabindex="0" data-href="/blog/197153-b/production-postmortem-deduplicating-replication-speed">
|
|
<div class="related-posts_mainlink">
|
|
<span class="related-posts_date">29 Apr 2022</span>
|
|
<span class="related-posts_title">Production postmortem: Deduplicating replication speed</span>
|
|
</div>
|
|
<div class="related-posts_tags">
|
|
<div class="related-posts_tags_header">
|
|
<div class="icomoon-position icon-tag" aria-hidden="true"></div>
|
|
</div>
|
|
<a class="related-posts_tag " href="/blog/tags/raven">raven</a>
|
|
<a class="related-posts_tag " href="/blog/tags/ravendb-net">ravendb.net</a>
|
|
</div>
|
|
</div>
|
|
<div class="related-posts_item" tabindex="0" data-href="/blog/200289-b/production-postmortem-the-spawn-of-denial-of-service">
|
|
<div class="related-posts_mainlink">
|
|
<span class="related-posts_date">12 Dec 2023</span>
|
|
<span class="related-posts_title">Production Postmortem: The Spawn of Denial of Service</span>
|
|
</div>
|
|
<div class="related-posts_tags">
|
|
<div class="related-posts_tags_header">
|
|
<div class="icomoon-position icon-tag" aria-hidden="true"></div>
|
|
</div>
|
|
<a class="related-posts_tag " href="/blog/tags/architecture">architecture</a>
|
|
<a class="related-posts_tag " href="/blog/tags/performance">performance</a>
|
|
<a class="related-posts_tag " href="/blog/tags/raven">raven</a>
|
|
<a class="related-posts_tag related-posts_tag--extra" href="/blog/tags/ravendb-net">ravendb.net</a>
|
|
<span class="related-posts_tag_extra">+1</span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<nav class="nextPrev">
|
|
<a class="pull-left" href="/blog/198817-c/ravendb-sharding-webinar-recording">
|
|
<span class="icomoon-position icon-chevron-left"></span><span class="hidden-xs"> previous post</span>
|
|
</a>
|
|
|
|
<a class="pull-right" href="/blog/198881-c/production-postmortem-the-big-server-that-couldn-t-handle-the-load">
|
|
<span class="hidden-xs">next post </span>
|
|
<span class="icomoon-position icon-chevron-right"></span>
|
|
</a>
|
|
|
|
<a href="/blog/198305-b/production-postmortem-do-you-trust-this-server" class="pull-left series">
|
|
<span class="icomoon-position icon-chevron-left"></span>
|
|
<span class="hidden-xs"> previous series post</span>
|
|
</a>
|
|
|
|
<a href="/blog/posts/series/171074/production-postmortem" class="pull-left series">
|
|
<i class="icon-book"></i>
|
|
<span class="hidden-xs">all </span>series<span class="hidden-xs"> posts</span>
|
|
</a>
|
|
|
|
<a href="/blog/198881-c/production-postmortem-the-big-server-that-couldn-t-handle-the-load" class="pull-right series">
|
|
<span class="hidden-xs"> next series post</span>
|
|
<span class="icomoon-position icon-chevron-right"></span>
|
|
</a>
|
|
|
|
<div class="clearfix"></div>
|
|
</nav>
|
|
|
|
|
|
<section class="postsInSeries" data-state="closed">
|
|
<h4 class="text-left">
|
|
More posts in <a href="/blog/posts/series/171074/production-postmortem">
|
|
"Production postmortem"
|
|
</a> series:
|
|
</h4>
|
|
<ol>
|
|
<li><em>(07 Apr 2025)</em> <a href="/blog/202180-a/production-postmortem-the-race-condition-in-the-interlock">The race condition in the interlock</a></li>
|
|
<li><em>(12 Dec 2023)</em> <a href="/blog/200289-b/production-postmortem-the-spawn-of-denial-of-service">The Spawn of Denial of Service</a></li>
|
|
<li><em>(24 Jul 2023)</em> <a href="/blog/199745-a/production-postmortem-the-dog-ate-my-request">The dog ate my request</a></li>
|
|
<li><em>(03 Jul 2023)</em> <a href="/blog/199649-b/production-postmortem-enomem-when-trying-to-free-memory">ENOMEM when trying to free memory</a></li>
|
|
<li><em>(27 Jan 2023)</em> <a href="/blog/198913-c/production-postmortem-the-server-ate-all-my-memory">The server ate all my memory</a></li>
|
|
<li><em>(23 Jan 2023)</em> <a href="/blog/198881-c/production-postmortem-the-big-server-that-couldn-t-handle-the-load">The big server that couldn’t handle the load</a></li>
|
|
<li><strong><em>(16 Jan 2023)</em> <a href="/blog/198849-c/production-postmortem-the-heisenbug-server">The heisenbug server</a></strong></li>
|
|
<li><em>(03 Oct 2022)</em> <a href="/blog/198305-b/production-postmortem-do-you-trust-this-server">Do you trust this server?</a></li>
|
|
<li><em>(15 Sep 2022)</em> <a href="/blog/198210-c/production-postmortem-the-missed-indexing-reference">The missed indexing reference</a></li>
|
|
<li><em>(05 Aug 2022)</em> <a href="/blog/197953-a/production-postmortem-the-allocating-query">The allocating query</a></li>
|
|
<li><em>(22 Jul 2022)</em> <a href="/blog/197825-c/production-postmortem-efficiency-all-the-way-to-out-of-memory-error">Efficiency all the way to Out of Memory error</a></li>
|
|
<li><em>(18 Jul 2022)</em> <a href="/blog/197697-b/production-postmortem-broken-networks-and-compressed-streams">Broken networks and compressed streams</a></li>
|
|
<li><em>(13 Jul 2022)</em> <a href="/blog/197637-a/production-postmortem-your-math-is-wrong-recursion-doesn-t-work-this-way">Your math is wrong, recursion doesn’t work this way</a></li>
|
|
<li><em>(12 Jul 2022)</em> <a href="/blog/197636-a/production-postmortem-the-data-corruption-in-the-node-js-stack">The data corruption in the node.js stack</a></li>
|
|
<li><em>(11 Jul 2022)</em> <a href="/blog/197635-a/production-postmortem-out-of-memory-on-a-clear-sky">Out of memory on a clear sky</a></li>
|
|
<li><em>(29 Apr 2022)</em> <a href="/blog/197153-b/production-postmortem-deduplicating-replication-speed">Deduplicating replication speed</a></li>
|
|
<li><em>(25 Apr 2022)</em> <a href="/blog/197091-b/production-postmortem-the-network-latency-and-the-i-o-spikes">The network latency and the I/O spikes</a></li>
|
|
<li><em>(22 Apr 2022)</em> <a href="/blog/197090-b/production-postmortem-the-encrypted-database-that-was-too-big-to-replicate">The encrypted database that was too big to replicate</a></li>
|
|
<li><em>(20 Apr 2022)</em> <a href="/blog/197057-b/production-postmortem-misleading-security-and-other-production-snafus">Misleading security and other production snafus</a></li>
|
|
<li><em>(03 Jan 2022)</em> <a href="/blog/195937-a/production-postmortem-an-error-on-the-first-act-will-lead-to-data-corruption-on-the-second-act">An error on the first act will lead to data corruption on the second act…</a></li>
|
|
<li><em>(13 Dec 2021)</em> <a href="/blog/195681-b/production-postmortem-the-memory-leak-that-only-happened-on-linux">The memory leak that only happened on Linux</a></li>
|
|
<li><em>(17 Sep 2021)</em> <a href="/blog/194721-b/production-postmortem-the-guinness-record-for-page-faults-high-cpu">The Guinness record for page faults & high CPU</a></li>
|
|
<li><em>(07 Jan 2021)</em> <a href="/blog/192865-b/production-postmortem-the-file-system-limitation">The file system limitation</a></li>
|
|
<li><em>(23 Mar 2020)</em> <a href="/blog/190244-c/production-postmortem-high-cpu-when-there-is-little-work-to-be-done">high CPU when there is little work to be done</a></li>
|
|
<li><em>(21 Feb 2020)</em> <a href="/blog/190049-c/production-postmortem-the-self-signed-certificate-that-couldn-t">The self signed certificate that couldn’t</a></li>
|
|
<li><em>(31 Jan 2020)</em> <a href="/blog/189761-a/production-postmortem-the-slow-slowdown-of-large-systems">The slow slowdown of large systems</a></li>
|
|
<li><em>(07 Jun 2019)</em> <a href="/blog/187554-c/production-postmortem-printer-out-of-paper-and-the-ravendb-hang">Printer out of paper and the RavenDB hang</a></li>
|
|
<li><em>(18 Feb 2019)</em> <a href="/blog/186273-c/production-postmortem-this-data-corruption-bug-requires-3-simultaneous-race-conditions">This data corruption bug requires 3 simultaneous race conditions</a></li>
|
|
<li><em>(25 Dec 2018)</em> <a href="/blog/185633-c/production-postmortem-handled-errors-and-the-curse-of-recursive-error-handling">Handled errors and the curse of recursive error handling</a></li>
|
|
<li><em>(23 Nov 2018)</em> <a href="/blog/185185-c/production-postmortem-the-arm-is-killing-me">The ARM is killing me</a></li>
|
|
<li><em>(22 Feb 2018)</em> <a href="/blog/181953-a/production-postmortem-the-unavailable-linux-server">The unavailable Linux server</a></li>
|
|
<li><em>(06 Dec 2017)</em> <a href="/blog/180481/production-postmortem-data-corruption-a-view-from-inside-the-sausage">data corruption, a view from INSIDE the sausage</a></li>
|
|
<li><em>(01 Dec 2017)</em> <a href="/blog/180769/production-postmortem-the-random-high-cpu">The random high CPU</a></li>
|
|
<li><em>(07 Aug 2017)</em> <a href="/blog/179203/production-postmortem-30-boost-with-a-single-line-change">30% boost with a single line change</a></li>
|
|
<li><em>(04 Aug 2017)</em> <a href="/blog/179202/production-postmortem-the-case-of-99-99-percentile">The case of 99.99% percentile</a></li>
|
|
<li><em>(02 Aug 2017)</em> <a href="/blog/179170/production-postmortem-the-lightly-loaded-trashing-server">The lightly loaded trashing server</a></li>
|
|
<li><em>(23 Aug 2016)</em> <a href="/blog/175047/production-postmortem-the-insidious-cost-of-managed-memory">The insidious cost of managed memory</a></li>
|
|
<li><em>(05 Feb 2016)</em> <a href="/blog/173025/production-postmortem-a-null-reference-in-our-abstraction">A null reference in our abstraction</a></li>
|
|
<li><em>(27 Jan 2016)</em> <a href="/blog/172929/production-postmortem-the-razor-suicide">The Razor Suicide</a></li>
|
|
<li><em>(13 Nov 2015)</em> <a href="/blog/172449/production-postmortem-the-case-of-the-it-is-slow-on-that-machine-only">The case of the “it is slow on that machine (only)”</a></li>
|
|
<li><em>(21 Oct 2015)</em> <a href="/blog/172161/production-postmortem-the-case-of-the-slow-index-rebuild">The case of the slow index rebuild</a></li>
|
|
<li><em>(22 Sep 2015)</em> <a href="/blog/172005/production-postmortem-the-case-of-the-unicode-poo">The case of the Unicode Poo</a></li>
|
|
<li><em>(03 Sep 2015)</em> <a href="/blog/171877/production-postmortem-the-industry-at-large">The industry at large</a></li>
|
|
<li><em>(01 Sep 2015)</em> <a href="/blog/171876/production-postmortem-the-case-of-the-lying-configuration-file">The case of the lying configuration file</a></li>
|
|
<li><em>(31 Aug 2015)</em> <a href="/blog/171875/production-postmortem-the-case-of-the-memory-eater-and-high-load">The case of the memory eater and high load</a></li>
|
|
<li><em>(14 Aug 2015)</em> <a href="/blog/171683/production-postmortem-the-case-of-the-man-in-the-middle">The case of the man in the middle</a></li>
|
|
<li><em>(05 Aug 2015)</em> <a href="/blog/171585/production-postmortem-reading-the-errors">Reading the errors</a></li>
|
|
<li><em>(29 Jul 2015)</em> <a href="/blog/171457/production-postmortem-the-evil-licensing-code">The evil licensing code</a></li>
|
|
<li><em>(23 Jul 2015)</em> <a href="/blog/171329/production-postmortem-the-case-of-the-native-memory-leak">The case of the native memory leak</a></li>
|
|
<li><em>(16 Jul 2015)</em> <a href="/blog/171169/production-postmortem-the-case-of-the-intransigent-new-database">The case of the intransigent new database</a></li>
|
|
<li><em>(13 Jul 2015)</em> <a href="/blog/171137/production-postmortem-the-case-of-the-hung-over-server">The case of the hung over server</a></li>
|
|
<li><em>(09 Jul 2015)</em> <a href="/blog/171074/production-postmortem-the-case-of-the-infected-cluster">The case of the infected cluster</a></li>
|
|
</ol>
|
|
<button class="morePosts btn btn-sm btn-default">show all</button> </section>
|
|
<hr />
|
|
</article>
|
|
|
|
<section class="comments">
|
|
<a name="comments"></a>
|
|
<h2>Comments</h2>
|
|
|
|
|
|
<article class="comment preview">
|
|
<h4 class="preview-title">Comment preview</h4>
|
|
<header>
|
|
<a name="comment-preview"></a>
|
|
<div class="avatar">
|
|
<img src='https://www.gravatar.com/avatar.php?gravatar_id={0}&size=50&default=identicon' alt="" />
|
|
</div>
|
|
<time>
|
|
</time>
|
|
<div class="postedBy">
|
|
<a href=""></a>
|
|
</div>
|
|
</header>
|
|
<div class="comment-body">
|
|
</div>
|
|
</article>
|
|
|
|
<article id="postComment">
|
|
<div class="commentsClosedMessage">Comments have been closed on this topic.</div>
|
|
</article>
|
|
</section>
|
|
|
|
<div id="markdownHelp" class="modal fade" tabindex="-1">
|
|
<div class="modal-dialog modal-lg" role="document">
|
|
<div class="modal-content">
|
|
<div class="modal-header">
|
|
<div class="row">
|
|
<div class="col-sm-10">
|
|
<h3 class="modal-title">Markdown formatting</h3>
|
|
</div>
|
|
<div class="col-sm-2">
|
|
<span class="esc-info">
|
|
<kbd>ESC</kbd> to close
|
|
</span>
|
|
<button type="button" class="close" data-dismiss="modal" aria-label="Close">
|
|
<span aria-hidden="true">×</span>
|
|
</button>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="modal-body">
|
|
<p><a href="http://daringfireball.net/projects/markdown/" target="_blank">Markdown</a> turns plain text formatting into fancy HTML formatting.</p>
|
|
|
|
<div class="comment-preview">
|
|
<div class="row">
|
|
<div class="col-sm-6">
|
|
<div class="bg-gray">
|
|
<h3>Phrase Emphasis</h3>
|
|
<pre><code>*italic* **bold**
|
|
_italic_ __bold__
|
|
</code></pre>
|
|
</div>
|
|
<div class="bg-gray">
|
|
<h3>Links</h3>
|
|
<p>Inline:</p>
|
|
<pre><code>An [example](http://url.com/ "Title")
|
|
</code></pre>
|
|
|
|
<p>Reference-style labels (titles are optional):</p>
|
|
<pre><code>An [example][id]. Then, anywhere
|
|
else in the doc, define the link:
|
|
[id]: http://example.com/ "Title"
|
|
</code></pre>
|
|
</div>
|
|
<div class="bg-gray">
|
|
<h3>Images</h3>
|
|
<p>Inline (titles are optional):</p>
|
|
<pre><code>
|
|
</code></pre>
|
|
|
|
<p>Reference-style:</p>
|
|
<pre><code>![alt text][id]
|
|
[id]: /url/to/img.jpg "Title"
|
|
</code></pre>
|
|
</div>
|
|
<div class="bg-gray">
|
|
<h3>Headers</h3>
|
|
<p>Setext-style:</p>
|
|
<pre><code>Header 1
|
|
========
|
|
Header 2
|
|
--------
|
|
</code></pre>
|
|
|
|
<p>atx-style (closing #'s are optional):</p>
|
|
<pre><code># Header 1 #
|
|
## Header 2 ##
|
|
###### Header 6
|
|
</code></pre>
|
|
</div>
|
|
<div class="bg-gray">
|
|
<h3>Lists</h3>
|
|
<p>Ordered, without paragraphs:</p>
|
|
<pre><code>1. Foo
|
|
2. Bar
|
|
</code></pre>
|
|
|
|
<p>Unordered, with paragraphs:</p>
|
|
<pre><code>* A list item.
|
|
With multiple paragraphs.
|
|
* Bar
|
|
</code></pre>
|
|
|
|
<p>You can nest them:</p>
|
|
<pre><code>* Abacus
|
|
* answer
|
|
* Bubbles
|
|
1. bunk
|
|
2. bupkis
|
|
* BELITTLER
|
|
3. burper
|
|
* Cunning
|
|
</code></pre>
|
|
</div>
|
|
</div>
|
|
<div class="col-sm-6">
|
|
<div class="bg-gray">
|
|
<h3>Blockquotes</h3>
|
|
<pre><code>> Email-style angle brackets
|
|
> are used for blockquotes.
|
|
> > And, they can be nested.
|
|
> #### Headers in blockquotes
|
|
>
|
|
> * You can quote a list.
|
|
> * Etc.
|
|
</code></pre>
|
|
</div>
|
|
<div class="bg-gray">
|
|
<h3>Horizontal Rules</h3>
|
|
<p>Three or more dashes or asterisks:</p>
|
|
<pre><code>---
|
|
* * *
|
|
- - - -
|
|
</code></pre>
|
|
</div>
|
|
<div class="bg-gray">
|
|
<h3>Manual Line Breaks</h3>
|
|
<p>End a line with two or more spaces:</p>
|
|
<pre><code>Roses are red,
|
|
Violets are blue.
|
|
</code></pre>
|
|
</div>
|
|
<div class="bg-gray">
|
|
|
|
<h3>Fenced Code Blocks</h3>
|
|
<p>Code blocks delimited by 3 or more backticks or tildas:</p>
|
|
<pre><code>```
|
|
This is a preformatted
|
|
code block
|
|
```
|
|
</code></pre>
|
|
</div>
|
|
<div class="bg-gray">
|
|
<h3>Header IDs</h3>
|
|
<p>Set the id of headings with <code>{#<id>}</code> at end of heading line:</p>
|
|
<pre><code>## My Heading {#myheading}
|
|
</code></pre>
|
|
</div>
|
|
<div class="bg-gray">
|
|
<h3>Tables</h3>
|
|
|
|
<pre><code>Fruit |Color
|
|
---------|----------
|
|
Apples |Red
|
|
Pears |Green
|
|
Bananas |Yellow</code></pre>
|
|
</div>
|
|
<div class="bg-gray">
|
|
<h3>Definition Lists</h3>
|
|
<pre><code>Term 1
|
|
: Definition 1
|
|
Term 2
|
|
: Definition 2</code></pre>
|
|
</div>
|
|
<div class="bg-gray">
|
|
<h3>Footnotes</h3>
|
|
<pre><code>Body text with a footnote [^1]
|
|
[^1]: Footnote text here
|
|
</code></pre>
|
|
</div>
|
|
<div class="bg-gray">
|
|
<h3>Abbreviations</h3>
|
|
<pre><code>MDD <- will have title
|
|
*[MDD]: MarkdownDeep
|
|
</code></pre>
|
|
<p> </p>
|
|
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
</div>
|
|
<div class="modal-footer">
|
|
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
</div>
|
|
<section class="rightSide">
|
|
|
|
|
|
|
|
|
|
<div id="futurePosts">
|
|
<h4>FUTURE POSTS</h4>
|
|
<p>No future posts left, oh my!</p>
|
|
</div>
|
|
|
|
<div id="postsSeries">
|
|
<h4>RECENT SERIES</h4>
|
|
|
|
<ol>
|
|
<li>
|
|
<a href="/blog/posts/series/155809/api-design">
|
|
<strong>
|
|
API Design
|
|
</strong>
|
|
</a> <em>(10)</em>:<br/>
|
|
<em>29 Jan 2026</em> - Don't try to guess
|
|
</li>
|
|
<li>
|
|
<a href="/blog/posts/series/196577-b/recording">
|
|
<strong>
|
|
Recording
|
|
</strong>
|
|
</a> <em>(20)</em>:<br/>
|
|
<em>05 Dec 2025</em> - Build AI that understands your business
|
|
</li>
|
|
<li>
|
|
<a href="/blog/posts/series/189601-c/webinar">
|
|
<strong>
|
|
Webinar
|
|
</strong>
|
|
</a> <em>(8)</em>:<br/>
|
|
<em>16 Sep 2025</em> - Building AI Agents in RavenDB
|
|
</li>
|
|
<li>
|
|
<a href="/blog/posts/series/173057/production-postmorterm">
|
|
<strong>
|
|
Production postmorterm
|
|
</strong>
|
|
</a> <em>(2)</em>:<br/>
|
|
<em>11 Jun 2025</em> - The rookie server's untimely promotion
|
|
</li>
|
|
<li>
|
|
<a href="/blog/posts/series/201250-a/ravendb-news">
|
|
<strong>
|
|
RavenDB News
|
|
</strong>
|
|
</a> <em>(2)</em>:<br/>
|
|
<em>02 May 2025</em> - May 2025
|
|
</li>
|
|
</ol>
|
|
<a href="/blog/posts/series">View all series</a>
|
|
</div>
|
|
|
|
<div id="recentComments">
|
|
<h4>RECENT COMMENTS</h4>
|
|
<ul>
|
|
<li>
|
|
<div>
|
|
<a href="/blog/204035-a/optimizing-concurrent-regexes#comment21">
|
|
<strong class="comment-body">Christian,
Thank you, very much appreciated</strong>
|
|
</a>
|
|
</div>
|
|
<div>
|
|
<strong>By</strong> Oren Eini on <em>"Optimizing" concurrent regexes</em>
|
|
</div>
|
|
</li>
|
|
<li>
|
|
<div>
|
|
<a href="/blog/204035-a/optimizing-concurrent-regexes#comment1">
|
|
<strong class="comment-body">I follow your blog for decades now. Posts like this are the reason I'm coming back.

For the issue: wow, this is ugly and t...</strong>
|
|
</a>
|
|
</div>
|
|
<div>
|
|
<strong>By</strong> Christian Sauer on <em>"Optimizing" concurrent regexes</em>
|
|
</div>
|
|
</li>
|
|
<li>
|
|
<div>
|
|
<a href="/blog/203975-a/learning-to-code-1990s-vs-2026#comment8">
|
|
<strong class="comment-body">Peter,
That looks like giving the model / agent to run actions that it shouldn't.You should assume that the model is hostile...</strong>
|
|
</a>
|
|
</div>
|
|
<div>
|
|
<strong>By</strong> Oren Eini on <em>Learning to code, 1990s vs 2026</em>
|
|
</div>
|
|
</li>
|
|
<li>
|
|
<div>
|
|
<a href="/blog/203975-a/learning-to-code-1990s-vs-2026#comment7">
|
|
<strong class="comment-body">https://x.com/DarkWebInformer/status/2061253599758315527</strong>
|
|
</a>
|
|
</div>
|
|
<div>
|
|
<strong>By</strong> peter on <em>Learning to code, 1990s vs 2026</em>
|
|
</div>
|
|
</li>
|
|
<li>
|
|
<div>
|
|
<a href="/blog/203975-a/learning-to-code-1990s-vs-2026#comment6">
|
|
<strong class="comment-body">META's Instagram (IG) implemetned an AI bot and it is - AS WE SPEAK - allowing users to takeover other peoples' accounts </strong>
|
|
</a>
|
|
</div>
|
|
<div>
|
|
<strong>By</strong> peter on <em>Learning to code, 1990s vs 2026</em>
|
|
</div>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
<div id="syndication">
|
|
<h4>Syndication</h4>
|
|
<table>
|
|
<tbody>
|
|
<tr>
|
|
<td>Main feed</td>
|
|
<td>
|
|
<a href="https://feeds.feedburner.com/AyendeRahien">
|
|
<svg width="78" height="25" viewBox="0 0 78 25" fill="none" xmlns="http://www.w3.org/2000/svg">
|
|
<rect x="0.5" y="0.5" width="76.3685" height="23.6" rx="3.8" fill="white"/>
|
|
<rect x="0.5" y="0.5" width="76.3685" height="23.6" rx="3.8" stroke="#A6A6A6"/>
|
|
<path d="M17.3 4.3H7.29999C5.64313 4.3 4.29999 5.64314 4.29999 7.3V17.3C4.29999 18.9569 5.64313 20.3 7.29999 20.3H17.3C18.9568 20.3 20.3 18.9569 20.3 17.3V7.3C20.3 5.64314 18.9568 4.3 17.3 4.3Z" fill="#FFA71A"/>
|
|
<path d="M8.29999 18.3C9.40456 18.3 10.3 17.4046 10.3 16.3C10.3 15.1954 9.40456 14.3 8.29999 14.3C7.19542 14.3 6.29999 15.1954 6.29999 16.3C6.29999 17.4046 7.19542 18.3 8.29999 18.3Z" fill="white"/>
|
|
<path d="M6.29999 12.3C7.89129 12.3 9.41741 12.9321 10.5426 14.0574C11.6678 15.1826 12.3 16.7087 12.3 18.3H14.3C14.3 16.1783 13.4571 14.1434 11.9568 12.6431C10.4566 11.1429 8.42172 10.3 6.29999 10.3V12.3Z" fill="white"/>
|
|
<path d="M6.29999 8.3C7.61321 8.3 8.91357 8.55866 10.1268 9.0612C11.3401 9.56375 12.4425 10.3003 13.3711 11.2289C14.2996 12.1575 15.0362 13.2599 15.5388 14.4732C16.0413 15.6864 16.3 16.9868 16.3 18.3H18.3C18.3 15.1174 17.0357 12.0652 14.7853 9.81472C12.5348 7.56428 9.48259 6.3 6.29999 6.3V8.3Z" fill="white"/>
|
|
<path d="M36.7296 8.318C37.2356 8.318 37.6725 8.40226 38.0392 8.57093C38.4057 8.73224 38.7026 8.94131 38.9298 9.19788C39.1645 9.44722 39.3296 9.719 39.4249 10.0123L38.545 10.32C38.4057 9.95329 38.1709 9.66689 37.8409 9.46156C37.511 9.24891 37.111 9.1432 36.6417 9.1432C36.1504 9.1432 35.7583 9.26448 35.465 9.50648C35.179 9.74114 35.0353 10.0636 35.0353 10.4743C35.0353 10.8261 35.1498 11.0971 35.3771 11.2877C35.6044 11.471 35.9011 11.607 36.2677 11.695L37.1144 11.9039C37.8624 12.0799 38.4352 12.3732 38.8312 12.7838C39.2344 13.1871 39.4357 13.6241 39.4357 14.0934C39.4356 14.4964 39.3332 14.8666 39.1281 15.2037C38.9228 15.5337 38.6181 15.7978 38.215 15.9957C37.819 16.1864 37.3236 16.2819 36.7296 16.2819C36.3044 16.2819 35.9268 16.238 35.5968 16.15C35.2668 16.0547 34.9804 15.9266 34.7384 15.7653C34.4965 15.5967 34.2986 15.4133 34.1447 15.2155C33.9907 15.0102 33.8772 14.7934 33.8038 14.566L34.6945 14.2584C34.8045 14.581 35.0247 14.8632 35.3546 15.1051C35.6919 15.3398 36.103 15.4567 36.587 15.4567C37.1808 15.4566 37.6463 15.3397 37.9835 15.1051C38.3209 14.8704 38.4894 14.5582 38.4894 14.1696C38.4893 13.8106 38.3394 13.5213 38.0392 13.3014C37.7385 13.0741 37.342 12.8977 36.8507 12.7731L36.004 12.5534C35.6594 12.4654 35.3403 12.3363 35.047 12.1676C34.7611 11.9917 34.5296 11.7723 34.3536 11.5084C34.185 11.2371 34.1007 10.9137 34.1007 10.5397C34.1008 9.84317 34.3243 9.30036 34.7716 8.91175C35.2263 8.5158 35.879 8.31802 36.7296 8.318Z" fill="black"/>... <path d="M43.4865 8.318C43.9925 8.318 44.4294 8.40226 44.796 8.57093C45.1625 8.73224 45.4594 8.94131 45.6866 9.19788C45.9213 9.44722 46.0864 9.719 46.1818 10.0123L45.3019 10.32C45.1625 9.95329 44.9278 9.66689 44.5978 9.46156C44.2678 9.24891 43.8679 9.1432 43.3986 9.1432C42.9072 9.1432 42.5151 9.26448 42.2218 9.50648C41.9359 9.74114 41.7921 10.0636 41.7921 10.4743C41.7922 10.8261 41.9066 11.0971 42.1339 11.2877C42.3612 11.471 42.6579 11.607 43.0245 11.695L43.8712 11.9039C44.6192 12.0799 45.192 12.3732 45.588 12.7838C45.9913 13.1871 46.1925 13.6241 46.1925 14.0934C46.1924 14.4964 46.09 14.8666 45.8849 15.2037C45.6796 15.5337 45.375 15.7978 44.9718 15.9957C44.5758 16.1864 44.0805 16.2819 43.4865 16.2819C43.0612 16.2819 42.6836 16.238 42.3536 16.15C42.0236 16.0547 41.7372 15.9266 41.4952 15.7653C41.2534 15.5967 41.0554 15.4133 40.9015 15.2155C40.7475 15.0102 40.634 14.7934 40.5607 14.566L41.4513 14.2584C41.5613 14.581 41.7816 14.8632 42.1115 15.1051C42.4488 15.3398 42.8599 15.4567 43.3439 15.4567C43.9376 15.4566 44.4031 15.3397 44.7404 15.1051C45.0777 14.8704 45.2462 14.5582 45.2462 14.1696C45.2461 13.8106 45.0962 13.5213 44.796 13.3014C44.4954 13.0741 44.0989 12.8977 43.6075 12.7731L42.7609 12.5534C42.4162 12.4654 42.0972 12.3363 41.8038 12.1676C41.5179 11.9917 41.2865 11.7723 41.1105 11.5084C40.9418 11.2371 40.8575 10.9137 40.8575 10.5397C40.8576 9.84317 41.0812 9.30036 41.5284 8.91175C41.9831 8.5158 42.6359 8.31802 43.4865 8.318Z" fill="black"/>
|
|
<path fill-rule="evenodd" clip-rule="evenodd" d="M58.5392 10.5182C59.0525 10.5182 59.4963 10.6463 59.8702 10.903C60.2442 11.1522 60.5341 11.5041 60.7394 11.9586C60.9447 12.4133 61.047 12.9491 61.047 13.5651H56.7325C56.7516 13.9259 56.8235 14.2417 56.9552 14.5114C57.1165 14.8265 57.3403 15.0683 57.6261 15.2369C57.9121 15.3983 58.2389 15.4791 58.6056 15.4791C58.9941 15.4791 59.3202 15.387 59.5841 15.2037C59.8554 15.0205 60.0644 14.7787 60.2111 14.4782L60.9708 14.8629C60.8315 15.1489 60.644 15.3983 60.4093 15.611C60.1821 15.8235 59.911 15.9888 59.5958 16.1061C59.2879 16.2234 58.9429 16.2818 58.5616 16.2819C58.0263 16.2819 57.5534 16.1606 57.1427 15.9186C56.732 15.6693 56.4086 15.3288 56.174 14.8961C55.9466 14.4635 55.8331 13.9647 55.8331 13.4C55.8331 12.8354 55.9467 12.3366 56.174 11.9039C56.4085 11.4714 56.7278 11.1342 57.131 10.8922C57.5416 10.6429 58.0113 10.5182 58.5392 10.5182ZM58.5284 11.2994C58.2278 11.2994 57.9414 11.3691 57.67 11.5084C57.4062 11.6478 57.1902 11.8457 57.0216 12.1022C56.8879 12.3088 56.8101 12.5506 56.7872 12.8278H60.1027C60.0819 12.5541 60.0174 12.3121 59.9034 12.1022C59.7641 11.8455 59.5766 11.6478 59.3419 11.5084C59.1073 11.3692 58.8364 11.2995 58.5284 11.2994Z" fill="black"/>... <path fill-rule="evenodd" clip-rule="evenodd" d="M64.5441 10.5182C65.0574 10.5182 65.5011 10.6463 65.8751 10.903C66.249 11.1522 66.5389 11.5041 66.7443 11.9586C66.9496 12.4133 67.0519 12.9491 67.0519 13.5651H62.7374C62.7565 13.9259 62.8284 14.2417 62.9601 14.5114C63.1214 14.8265 63.3452 15.0683 63.631 15.2369C63.917 15.3983 64.2438 15.4791 64.6105 15.4791C64.999 15.4791 65.325 15.387 65.589 15.2037C65.8602 15.0205 66.0693 14.7787 66.2159 14.4782L66.9757 14.8629C66.8364 15.1489 66.6489 15.3983 66.4142 15.611C66.1869 15.8235 65.9159 15.9888 65.6007 16.1061C65.2928 16.2234 64.9478 16.2818 64.5665 16.2819C64.0312 16.2819 63.5582 16.1606 63.1476 15.9186C62.7369 15.6693 62.4135 15.3288 62.1788 14.8961C61.9515 14.4635 61.838 13.9647 61.838 13.4C61.838 12.8354 61.9515 12.3366 62.1788 11.9039C62.4134 11.4714 62.7327 11.1342 63.1359 10.8922C63.5465 10.6429 64.0161 10.5182 64.5441 10.5182ZM64.5333 11.2994C64.2327 11.2994 63.9463 11.3691 63.6749 11.5084C63.4111 11.6478 63.1951 11.8457 63.0265 12.1022C62.8928 12.3088 62.815 12.5506 62.7921 12.8278H66.1075C66.0868 12.5541 66.0222 12.3121 65.9083 12.1022C65.769 11.8455 65.5815 11.6478 65.3468 11.5084C65.1122 11.3692 64.8412 11.2995 64.5333 11.2994Z" fill="black"/>
|
|
<path fill-rule="evenodd" clip-rule="evenodd" d="M73.0685 16.15H72.2325L72.1925 15.2174C72.0197 15.5043 71.8019 15.7379 71.5392 15.9186C71.1945 16.1606 70.7912 16.2819 70.3292 16.2819C69.8379 16.2819 69.4054 16.1606 69.0314 15.9186C68.6574 15.6766 68.3641 15.3395 68.1515 14.9069C67.9462 14.4669 67.8429 13.9647 67.8429 13.4C67.8429 12.828 67.9462 12.3259 68.1515 11.8932C68.3641 11.4606 68.6574 11.1235 69.0314 10.8815C69.4054 10.6395 69.8379 10.5182 70.3292 10.5182C70.7912 10.5182 71.1945 10.6395 71.5392 10.8815C71.7998 11.0607 72.0164 11.292 72.1886 11.5758V8.44984H73.0685V16.15ZM70.5157 11.3209C70.1713 11.321 69.8631 11.4087 69.5919 11.5846C69.3279 11.7606 69.119 12.0066 68.965 12.3219C68.811 12.6372 68.7345 12.9967 68.7345 13.4C68.7345 13.8034 68.811 14.1628 68.965 14.4782C69.1189 14.7934 69.3314 15.0395 69.6027 15.2155C69.874 15.3915 70.1828 15.4791 70.5275 15.4791C70.8498 15.4791 71.1357 15.3912 71.3849 15.2155C71.6342 15.0395 71.8286 14.7934 71.9679 14.4782C72.1146 14.1628 72.1886 13.8034 72.1886 13.4C72.1886 12.9967 72.1146 12.6372 71.9679 12.3219C71.8286 12.0067 71.6342 11.7606 71.3849 11.5846C71.1356 11.4088 70.8456 11.3209 70.5157 11.3209Z" fill="black"/>
|
|
<path fill-rule="evenodd" clip-rule="evenodd" d="M29.8878 8.44984C30.3865 8.44984 30.8268 8.54872 31.2081 8.74671C31.5968 8.94471 31.8978 9.21991 32.1105 9.57191C32.3304 9.92384 32.4396 10.3273 32.4396 10.7819C32.4396 11.2365 32.3226 11.6399 32.088 11.9918C31.8534 12.3438 31.5343 12.6224 31.131 12.8278C31.0453 12.8698 30.9567 12.906 30.8663 12.9391L32.7482 16.15H31.7247L29.9513 13.1207C29.8906 13.1237 29.8289 13.1246 29.7667 13.1246H28.3146V16.15H27.38V8.44984H29.8878ZM28.3146 12.2672H30.0197C30.2983 12.2672 30.5477 12.2043 30.7677 12.0797C30.9876 11.9478 31.164 11.7723 31.296 11.5524C31.4354 11.325 31.505 11.0679 31.505 10.7819C31.505 10.3421 31.3509 9.98674 31.0431 9.71546C30.7425 9.43686 30.3502 9.29658 29.8663 9.29652H28.3146V12.2672Z" fill="black"/>
|
|
<path d="M55.0734 9.29652H51.4318V11.9264H54.6447V12.7731H51.4318V16.15H50.4972V8.44984H55.0734V9.29652Z" fill="black"/>
|
|
</svg>
|
|
</a>
|
|
</td>
|
|
</tr>
|
|
<tr>
|
|
<td>Comments feed </td>
|
|
<td>
|
|
<a href="https://feeds.feedburner.com/ayende/coments">
|
|
<svg width="78" height="25" viewBox="0 0 78 25" fill="none" xmlns="http://www.w3.org/2000/svg">
|
|
<rect x="0.5" y="0.5" width="76.3685" height="23.6" rx="3.8" fill="white"/>
|
|
<rect x="0.5" y="0.5" width="76.3685" height="23.6" rx="3.8" stroke="#A6A6A6"/>
|
|
<path d="M17.3 4.3H7.29999C5.64313 4.3 4.29999 5.64314 4.29999 7.3V17.3C4.29999 18.9569 5.64313 20.3 7.29999 20.3H17.3C18.9568 20.3 20.3 18.9569 20.3 17.3V7.3C20.3 5.64314 18.9568 4.3 17.3 4.3Z" fill="#FFA71A"/>
|
|
<path d="M8.29999 18.3C9.40456 18.3 10.3 17.4046 10.3 16.3C10.3 15.1954 9.40456 14.3 8.29999 14.3C7.19542 14.3 6.29999 15.1954 6.29999 16.3C6.29999 17.4046 7.19542 18.3 8.29999 18.3Z" fill="white"/>
|
|
<path d="M6.29999 12.3C7.89129 12.3 9.41741 12.9321 10.5426 14.0574C11.6678 15.1826 12.3 16.7087 12.3 18.3H14.3C14.3 16.1783 13.4571 14.1434 11.9568 12.6431C10.4566 11.1429 8.42172 10.3 6.29999 10.3V12.3Z" fill="white"/>
|
|
<path d="M6.29999 8.3C7.61321 8.3 8.91357 8.55866 10.1268 9.0612C11.3401 9.56375 12.4425 10.3003 13.3711 11.2289C14.2996 12.1575 15.0362 13.2599 15.5388 14.4732C16.0413 15.6864 16.3 16.9868 16.3 18.3H18.3C18.3 15.1174 17.0357 12.0652 14.7853 9.81472C12.5348 7.56428 9.48259 6.3 6.29999 6.3V8.3Z" fill="white"/>
|
|
<path d="M36.7296 8.318C37.2356 8.318 37.6725 8.40226 38.0392 8.57093C38.4057 8.73224 38.7026 8.94131 38.9298 9.19788C39.1645 9.44722 39.3296 9.719 39.4249 10.0123L38.545 10.32C38.4057 9.95329 38.1709 9.66689 37.8409 9.46156C37.511 9.24891 37.111 9.1432 36.6417 9.1432C36.1504 9.1432 35.7583 9.26448 35.465 9.50648C35.179 9.74114 35.0353 10.0636 35.0353 10.4743C35.0353 10.8261 35.1498 11.0971 35.3771 11.2877C35.6044 11.471 35.9011 11.607 36.2677 11.695L37.1144 11.9039C37.8624 12.0799 38.4352 12.3732 38.8312 12.7838C39.2344 13.1871 39.4357 13.6241 39.4357 14.0934C39.4356 14.4964 39.3332 14.8666 39.1281 15.2037C38.9228 15.5337 38.6181 15.7978 38.215 15.9957C37.819 16.1864 37.3236 16.2819 36.7296 16.2819C36.3044 16.2819 35.9268 16.238 35.5968 16.15C35.2668 16.0547 34.9804 15.9266 34.7384 15.7653C34.4965 15.5967 34.2986 15.4133 34.1447 15.2155C33.9907 15.0102 33.8772 14.7934 33.8038 14.566L34.6945 14.2584C34.8045 14.581 35.0247 14.8632 35.3546 15.1051C35.6919 15.3398 36.103 15.4567 36.587 15.4567C37.1808 15.4566 37.6463 15.3397 37.9835 15.1051C38.3209 14.8704 38.4894 14.5582 38.4894 14.1696C38.4893 13.8106 38.3394 13.5213 38.0392 13.3014C37.7385 13.0741 37.342 12.8977 36.8507 12.7731L36.004 12.5534C35.6594 12.4654 35.3403 12.3363 35.047 12.1676C34.7611 11.9917 34.5296 11.7723 34.3536 11.5084C34.185 11.2371 34.1007 10.9137 34.1007 10.5397C34.1008 9.84317 34.3243 9.30036 34.7716 8.91175C35.2263 8.5158 35.879 8.31802 36.7296 8.318Z" fill="black"/>... <path d="M43.4865 8.318C43.9925 8.318 44.4294 8.40226 44.796 8.57093C45.1625 8.73224 45.4594 8.94131 45.6866 9.19788C45.9213 9.44722 46.0864 9.719 46.1818 10.0123L45.3019 10.32C45.1625 9.95329 44.9278 9.66689 44.5978 9.46156C44.2678 9.24891 43.8679 9.1432 43.3986 9.1432C42.9072 9.1432 42.5151 9.26448 42.2218 9.50648C41.9359 9.74114 41.7921 10.0636 41.7921 10.4743C41.7922 10.8261 41.9066 11.0971 42.1339 11.2877C42.3612 11.471 42.6579 11.607 43.0245 11.695L43.8712 11.9039C44.6192 12.0799 45.192 12.3732 45.588 12.7838C45.9913 13.1871 46.1925 13.6241 46.1925 14.0934C46.1924 14.4964 46.09 14.8666 45.8849 15.2037C45.6796 15.5337 45.375 15.7978 44.9718 15.9957C44.5758 16.1864 44.0805 16.2819 43.4865 16.2819C43.0612 16.2819 42.6836 16.238 42.3536 16.15C42.0236 16.0547 41.7372 15.9266 41.4952 15.7653C41.2534 15.5967 41.0554 15.4133 40.9015 15.2155C40.7475 15.0102 40.634 14.7934 40.5607 14.566L41.4513 14.2584C41.5613 14.581 41.7816 14.8632 42.1115 15.1051C42.4488 15.3398 42.8599 15.4567 43.3439 15.4567C43.9376 15.4566 44.4031 15.3397 44.7404 15.1051C45.0777 14.8704 45.2462 14.5582 45.2462 14.1696C45.2461 13.8106 45.0962 13.5213 44.796 13.3014C44.4954 13.0741 44.0989 12.8977 43.6075 12.7731L42.7609 12.5534C42.4162 12.4654 42.0972 12.3363 41.8038 12.1676C41.5179 11.9917 41.2865 11.7723 41.1105 11.5084C40.9418 11.2371 40.8575 10.9137 40.8575 10.5397C40.8576 9.84317 41.0812 9.30036 41.5284 8.91175C41.9831 8.5158 42.6359 8.31802 43.4865 8.318Z" fill="black"/>
|
|
<path fill-rule="evenodd" clip-rule="evenodd" d="M58.5392 10.5182C59.0525 10.5182 59.4963 10.6463 59.8702 10.903C60.2442 11.1522 60.5341 11.5041 60.7394 11.9586C60.9447 12.4133 61.047 12.9491 61.047 13.5651H56.7325C56.7516 13.9259 56.8235 14.2417 56.9552 14.5114C57.1165 14.8265 57.3403 15.0683 57.6261 15.2369C57.9121 15.3983 58.2389 15.4791 58.6056 15.4791C58.9941 15.4791 59.3202 15.387 59.5841 15.2037C59.8554 15.0205 60.0644 14.7787 60.2111 14.4782L60.9708 14.8629C60.8315 15.1489 60.644 15.3983 60.4093 15.611C60.1821 15.8235 59.911 15.9888 59.5958 16.1061C59.2879 16.2234 58.9429 16.2818 58.5616 16.2819C58.0263 16.2819 57.5534 16.1606 57.1427 15.9186C56.732 15.6693 56.4086 15.3288 56.174 14.8961C55.9466 14.4635 55.8331 13.9647 55.8331 13.4C55.8331 12.8354 55.9467 12.3366 56.174 11.9039C56.4085 11.4714 56.7278 11.1342 57.131 10.8922C57.5416 10.6429 58.0113 10.5182 58.5392 10.5182ZM58.5284 11.2994C58.2278 11.2994 57.9414 11.3691 57.67 11.5084C57.4062 11.6478 57.1902 11.8457 57.0216 12.1022C56.8879 12.3088 56.8101 12.5506 56.7872 12.8278H60.1027C60.0819 12.5541 60.0174 12.3121 59.9034 12.1022C59.7641 11.8455 59.5766 11.6478 59.3419 11.5084C59.1073 11.3692 58.8364 11.2995 58.5284 11.2994Z" fill="black"/>... <path fill-rule="evenodd" clip-rule="evenodd" d="M64.5441 10.5182C65.0574 10.5182 65.5011 10.6463 65.8751 10.903C66.249 11.1522 66.5389 11.5041 66.7443 11.9586C66.9496 12.4133 67.0519 12.9491 67.0519 13.5651H62.7374C62.7565 13.9259 62.8284 14.2417 62.9601 14.5114C63.1214 14.8265 63.3452 15.0683 63.631 15.2369C63.917 15.3983 64.2438 15.4791 64.6105 15.4791C64.999 15.4791 65.325 15.387 65.589 15.2037C65.8602 15.0205 66.0693 14.7787 66.2159 14.4782L66.9757 14.8629C66.8364 15.1489 66.6489 15.3983 66.4142 15.611C66.1869 15.8235 65.9159 15.9888 65.6007 16.1061C65.2928 16.2234 64.9478 16.2818 64.5665 16.2819C64.0312 16.2819 63.5582 16.1606 63.1476 15.9186C62.7369 15.6693 62.4135 15.3288 62.1788 14.8961C61.9515 14.4635 61.838 13.9647 61.838 13.4C61.838 12.8354 61.9515 12.3366 62.1788 11.9039C62.4134 11.4714 62.7327 11.1342 63.1359 10.8922C63.5465 10.6429 64.0161 10.5182 64.5441 10.5182ZM64.5333 11.2994C64.2327 11.2994 63.9463 11.3691 63.6749 11.5084C63.4111 11.6478 63.1951 11.8457 63.0265 12.1022C62.8928 12.3088 62.815 12.5506 62.7921 12.8278H66.1075C66.0868 12.5541 66.0222 12.3121 65.9083 12.1022C65.769 11.8455 65.5815 11.6478 65.3468 11.5084C65.1122 11.3692 64.8412 11.2995 64.5333 11.2994Z" fill="black"/>
|
|
<path fill-rule="evenodd" clip-rule="evenodd" d="M73.0685 16.15H72.2325L72.1925 15.2174C72.0197 15.5043 71.8019 15.7379 71.5392 15.9186C71.1945 16.1606 70.7912 16.2819 70.3292 16.2819C69.8379 16.2819 69.4054 16.1606 69.0314 15.9186C68.6574 15.6766 68.3641 15.3395 68.1515 14.9069C67.9462 14.4669 67.8429 13.9647 67.8429 13.4C67.8429 12.828 67.9462 12.3259 68.1515 11.8932C68.3641 11.4606 68.6574 11.1235 69.0314 10.8815C69.4054 10.6395 69.8379 10.5182 70.3292 10.5182C70.7912 10.5182 71.1945 10.6395 71.5392 10.8815C71.7998 11.0607 72.0164 11.292 72.1886 11.5758V8.44984H73.0685V16.15ZM70.5157 11.3209C70.1713 11.321 69.8631 11.4087 69.5919 11.5846C69.3279 11.7606 69.119 12.0066 68.965 12.3219C68.811 12.6372 68.7345 12.9967 68.7345 13.4C68.7345 13.8034 68.811 14.1628 68.965 14.4782C69.1189 14.7934 69.3314 15.0395 69.6027 15.2155C69.874 15.3915 70.1828 15.4791 70.5275 15.4791C70.8498 15.4791 71.1357 15.3912 71.3849 15.2155C71.6342 15.0395 71.8286 14.7934 71.9679 14.4782C72.1146 14.1628 72.1886 13.8034 72.1886 13.4C72.1886 12.9967 72.1146 12.6372 71.9679 12.3219C71.8286 12.0067 71.6342 11.7606 71.3849 11.5846C71.1356 11.4088 70.8456 11.3209 70.5157 11.3209Z" fill="black"/>
|
|
<path fill-rule="evenodd" clip-rule="evenodd" d="M29.8878 8.44984C30.3865 8.44984 30.8268 8.54872 31.2081 8.74671C31.5968 8.94471 31.8978 9.21991 32.1105 9.57191C32.3304 9.92384 32.4396 10.3273 32.4396 10.7819C32.4396 11.2365 32.3226 11.6399 32.088 11.9918C31.8534 12.3438 31.5343 12.6224 31.131 12.8278C31.0453 12.8698 30.9567 12.906 30.8663 12.9391L32.7482 16.15H31.7247L29.9513 13.1207C29.8906 13.1237 29.8289 13.1246 29.7667 13.1246H28.3146V16.15H27.38V8.44984H29.8878ZM28.3146 12.2672H30.0197C30.2983 12.2672 30.5477 12.2043 30.7677 12.0797C30.9876 11.9478 31.164 11.7723 31.296 11.5524C31.4354 11.325 31.505 11.0679 31.505 10.7819C31.505 10.3421 31.3509 9.98674 31.0431 9.71546C30.7425 9.43686 30.3502 9.29658 29.8663 9.29652H28.3146V12.2672Z" fill="black"/>
|
|
<path d="M55.0734 9.29652H51.4318V11.9264H54.6447V12.7731H51.4318V16.15H50.4972V8.44984H55.0734V9.29652Z" fill="black"/>
|
|
</svg>
|
|
</a>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
|
|
|
|
</section>
|
|
</div>
|
|
|
|
<script src="/blog/js/jquery-1.11.2.js?v=WMJwNbei5YnfOX5dfgVCS5C4waqvc-_0fV7W2uy3DyU"></script>
|
|
<script src="/blog/js/jquery-migrate-1.2.1.js?v=LqNz0-55MRORhtKlmCREWPLg0MeKQXpt9KT37XqXJYs"></script>
|
|
<script src="/blog/js/bootstrap.js?v=R79rLguyGEnyBaTy2QyOQLJ3Pz_fTHZEcc0FDvCoc3g"></script>
|
|
|
|
<script src="/blog/js/main-bundle.min.js"></script>
|
|
|
|
|
|
<script type="text/javascript">
|
|
window.twttr = (function (d, s, id) {
|
|
var js, fjs = d.getElementsByTagName(s)[0], t = window.twttr || {};
|
|
if (d.getElementById(id)) return;
|
|
js = d.createElement(s);
|
|
js.id = id;
|
|
js.src = "https://platform.twitter.com/widgets.js";
|
|
fjs.parentNode.insertBefore(js, fjs);
|
|
t._e = [];
|
|
t.ready = function (f) { t._e.push(f); };
|
|
return t;
|
|
}(document, "script", "twitter-wjs"));
|
|
|
|
document.addEventListener('DOMContentLoaded', function () {
|
|
document.querySelectorAll('.related-posts_item').forEach(function(item) {
|
|
item.addEventListener('click', function(e) {
|
|
if (e.target.closest('.related-posts_tag')) return;
|
|
var href = item.getAttribute('data-href');
|
|
if (href) {
|
|
window.location = href;
|
|
}
|
|
});
|
|
item.addEventListener('keydown', function(e) {
|
|
if ((e.key === 'Enter' || e.key === ' ') && !e.target.closest('.related-posts_tag')) {
|
|
var href = item.getAttribute('data-href');
|
|
if (href) {
|
|
window.location = href;
|
|
}
|
|
}
|
|
});
|
|
});
|
|
});
|
|
</script>
|
|
<script src='https://www.google.com/recaptcha/api.js'></script>
|
|
</body>
|
|
</html> |