Files
nexus/sreweekly/articles/102/01-sysadvent-day-13-half-dead-tcp-connections-and-why-heartbeats-matter.html
2026-09-12 17:23:01 +08:00

1412 lines
57 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html dir='ltr' xmlns='http://www.w3.org/1999/xhtml' xmlns:b='http://www.google.com/2005/gml/b' xmlns:data='http://www.google.com/2005/gml/data' xmlns:expr='http://www.google.com/2005/gml/expr'>
<head>
<link href='https://www.blogger.com/static/v1/widgets/4165133002-widget_css_bundle.css' rel='stylesheet' type='text/css'/>
<meta content='text/html; charset=UTF-8' http-equiv='Content-Type'/>
<meta content='blogger' name='generator'/>
<link href='https://sysadvent.blogspot.com/favicon.ico' rel='icon' type='image/x-icon'/>
<link href='https://sysadvent.blogspot.com/2017/12/day-13-half-dead-tcp-connections-and.html' rel='canonical'/>
<link rel="alternate" type="application/atom+xml" title="sysadvent - Atom" href="https://sysadvent.blogspot.com/feeds/posts/default" />
<link rel="alternate" type="application/rss+xml" title="sysadvent - RSS" href="https://sysadvent.blogspot.com/feeds/posts/default?alt=rss" />
<link rel="service.post" type="application/atom+xml" title="sysadvent - Atom" href="https://www.blogger.com/feeds/3615332969083650973/posts/default" />
<link rel="alternate" type="application/atom+xml" title="sysadvent - Atom" href="https://sysadvent.blogspot.com/feeds/699285441357427180/comments/default" />
<!--Can't find substitution for tag [blog.ieCssRetrofitLinks]-->
<link href='https://github.com/aebm/sysadvent_2017/blob/master/images/pub_sub.png?raw=true' rel='image_src'/>
<meta content='https://sysadvent.blogspot.com/2017/12/day-13-half-dead-tcp-connections-and.html' property='og:url'/>
<meta content='Day 13 - Half-Dead TCP Connections and Why Heartbeats Matter' property='og:title'/>
<meta content='By: Alejandro Brito Monedero ( @ae_bm ) 
Edited By: J. Paul Reed ( @jpaulreed ) We are living interesting times in the tech world, full o...' property='og:description'/>
<meta content='https://lh3.googleusercontent.com/blogger_img_proxy/AEn0k_t5kc9q__Co9QNo_k3ZHJ4ijB-KHFpR3oA6WXQY5LqxyITEqxGvy3I8svKnJDbYzFWD2y8aVJirErB4-UKGHhL3sByqvF2ztvDEac5GKuHl112jYgsr1mdNF1g65V9W-NbWEW3l0sPIBF4qbY7gd4PtqQ=w1200-h630-p-k-no-nu' property='og:image'/>
<title>
sysadvent: Day 13 - Half-Dead TCP Connections and Why Heartbeats Matter
</title>
<style type='text/css'>@font-face{font-family:'Allerta';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/allerta/v19/TwMO-IAHRlkbx940YnYXSA.woff2)format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcDRrBZQI.woff2)format('woff2');unicode-range:U+0460-052F,U+1C80-1C8A,U+20B4,U+2DE0-2DFF,U+A640-A69F,U+FE2E-FE2F;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcBBrBZQI.woff2)format('woff2');unicode-range:U+0301,U+0400-045F,U+0490-0491,U+04B0-04B1,U+2116;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcDBrBZQI.woff2)format('woff2');unicode-range:U+1F00-1FFF;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcAxrBZQI.woff2)format('woff2');unicode-range:U+0370-0377,U+037A-037F,U+0384-038A,U+038C,U+038E-03A1,U+03A3-03FF;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcAhrBZQI.woff2)format('woff2');unicode-range:U+0307-0308,U+0590-05FF,U+200C-2010,U+20AA,U+25CC,U+FB1D-FB4F;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcDxrBZQI.woff2)format('woff2');unicode-range:U+0102-0103,U+0110-0111,U+0128-0129,U+0168-0169,U+01A0-01A1,U+01AF-01B0,U+0300-0301,U+0303-0304,U+0308-0309,U+0323,U+0329,U+1EA0-1EF9,U+20AB;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcDhrBZQI.woff2)format('woff2');unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF;}@font-face{font-family:'Arimo';font-style:normal;font-weight:400;font-display:swap;src:url(//fonts.gstatic.com/s/arimo/v36/P5sfzZCDf9_T_3cV7NCUECyoxNk37cxcABrB.woff2)format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD;}</style>
<style id='page-skin-1' type='text/css'><!--
/*
-----------------------------------------------
Blogger Template Style
Name: Minima
Designer: Douglas Bowman
URL: www.stopdesign.com
Date: 26 Feb 2004
Updated by: Blogger Team
----------------------------------------------- */
/* Variable definitions
====================
<Variable name="bgcolor" description="Page Background Color"
type="color" default="#fff">
<Variable name="textcolor" description="Text Color"
type="color" default="#333">
<Variable name="linkcolor" description="Link Color"
type="color" default="#58a">
<Variable name="pagetitlecolor" description="Blog Title Color"
type="color" default="#666">
<Variable name="descriptioncolor" description="Blog Description Color"
type="color" default="#999">
<Variable name="titlecolor" description="Post Title Color"
type="color" default="#c60">
<Variable name="bordercolor" description="Border Color"
type="color" default="#ccc">
<Variable name="sidebarcolor" description="Sidebar Title Color"
type="color" default="#999">
<Variable name="sidebartextcolor" description="Sidebar Text Color"
type="color" default="#666">
<Variable name="visitedlinkcolor" description="Visited Link Color"
type="color" default="#999">
<Variable name="bodyfont" description="Text Font"
type="font" default="normal normal 100% Georgia, Serif">
<Variable name="headerfont" description="Sidebar Title Font"
type="font"
default="normal normal 78% 'Trebuchet MS',Trebuchet,Arial,Verdana,Sans-serif">
<Variable name="pagetitlefont" description="Blog Title Font"
type="font"
default="normal normal 200% Georgia, Serif">
<Variable name="descriptionfont" description="Blog Description Font"
type="font"
default="normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif">
<Variable name="postfooterfont" description="Post Footer Font"
type="font"
default="normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif">
<Variable name="startSide" description="Side where text starts in blog language"
type="automatic" default="left">
<Variable name="endSide" description="Side where text ends in blog language"
type="automatic" default="right">
*/
/* Use this with templates/template-twocol.html */
body {
background:#ffffff;
margin:0;
color:#333333;
font:x-small Georgia Serif;
font-size/* */:/**/small;
font-size: /**/small;
text-align: center;
}
a:link {
color:#5588aa;
text-decoration:none;
}
a:visited {
color:#999999;
text-decoration:none;
}
a:hover {
color:#cc6600;
text-decoration:underline;
}
a img {
border-width:0;
}
/* Header
-----------------------------------------------
*/
#header-wrapper {
width:760px;
margin:0 auto 10px;
/*border:1px solid #cccccc;*/
}
#header-inner {
background-position: center;
margin-left: auto;
margin-right: auto;
}
#header {
margin: 5px;
/*border: 1px solid #cccccc;*/
text-align: center;
color:#666666;
}
#header h1 {
margin:5px 5px 0;
padding:15px 20px .25em;
line-height:1.2em;
text-transform:uppercase;
letter-spacing:.2em;
font: normal normal 200% Georgia, Serif;
}
#header a {
color:#666666;
text-decoration:none;
}
#header a:hover {
color:#666666;
}
#header .description {
margin:0 5px 5px;
padding:0 20px 15px;
max-width:800px;
text-transform:uppercase;
letter-spacing:.2em;
line-height: 1.4em;
font: normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif;
color: #999999;
}
#header img {
margin-left: auto;
margin-right: auto;
}
/* Outer-Wrapper
----------------------------------------------- */
#outer-wrapper {
width: 760px;
margin:0 auto;
padding:10px;
text-align:left;
font: normal normal 100% Arimo;
}
#main-wrapper {
width: 510px;
float: left;
word-wrap: break-word; /* fix for long text breaking sidebar float in IE */
overflow: hidden; /* fix for long non-text content breaking IE sidebar float */
}
#sidebar-wrapper {
width: 220px;
float: right;
word-wrap: break-word; /* fix for long text breaking sidebar float in IE */
overflow: hidden; /* fix for long non-text content breaking IE sidebar float */
}
/* Headings
----------------------------------------------- */
h2 {
margin:1.5em 0 .75em;
font:normal normal 130% Allerta;
line-height: 1.4em;
text-transform:uppercase;
letter-spacing:.2em;
color:#999999;
}
/* Posts
-----------------------------------------------
*/
h2.date-header {
margin:1.5em 0 .5em;
}
.post {
margin:.5em 0 1.5em;
border-bottom:1px dotted #cccccc;
padding-bottom:1.5em;
}
.post h3 {
margin:.25em 0 0;
padding:0 0 4px;
font-size:140%;
font-weight:normal;
line-height:1.4em;
color:#cc6600;
}
.post h3 a, .post h3 a:visited, .post h3 strong {
display:block;
text-decoration:none;
color:#cc6600;
font-weight:normal;
}
.post h3 strong, .post h3 a:hover {
color:#333333;
}
.post-body {
margin:0 0 .75em;
line-height:1.6em;
}
.post-body blockquote {
line-height:1.3em;
}
.post-footer {
margin: .75em 0;
color:#999999;
text-transform:uppercase;
letter-spacing:.1em;
font: normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif;
line-height: 1.4em;
}
.comment-link {
margin-left:.6em;
}
.post img {
padding:4px;
border:1px solid #cccccc;
}
.post blockquote {
margin:1em 20px;
}
.post blockquote p {
margin:.75em 0;
}
.post pre {
overflow: auto;
padding-left: 5px;
padding-right: 5px;
border: 1px solid #CCCCFF !important;
}
.post pre a {
text-decoration: underline;
}
/* Comments
----------------------------------------------- */
#comments h4 {
margin:1em 0;
font-weight: bold;
line-height: 1.4em;
text-transform:uppercase;
letter-spacing:.2em;
color: #999999;
}
#comments-block {
margin:1em 0 1.5em;
line-height:1.6em;
}
#comments-block .comment-author {
margin:.5em 0;
}
#comments-block .comment-body {
margin:.25em 0 0;
}
#comments-block .comment-footer {
margin:-.25em 0 2em;
line-height: 1.4em;
text-transform:uppercase;
letter-spacing:.1em;
}
#comments-block .comment-body p {
margin:0 0 .75em;
}
.deleted-comment {
font-style:italic;
color:gray;
}
#blog-pager-newer-link {
float: left;
}
#blog-pager-older-link {
float: right;
}
#blog-pager {
text-align: center;
}
.feed-links {
clear: both;
line-height: 2.5em;
}
/* Sidebar Content
----------------------------------------------- */
.sidebar {
color: #666666;
line-height: 1.5em;
}
.sidebar ul {
list-style:none;
margin:0 0 0;
padding:0 0 0;
}
.sidebar li {
margin:0;
padding-top:0;
padding-right:0;
padding-bottom:.25em;
padding-left:15px;
text-indent:-15px;
line-height:1.5em;
}
.sidebar .widget, .main .widget {
border-bottom:1px dotted #cccccc;
margin:0 0 1.5em;
padding:0 0 1.5em;
}
.main .Blog {
border-bottom-width: 0;
}
/* Profile
----------------------------------------------- */
.profile-img {
float: left;
margin-top: 0;
margin-right: 5px;
margin-bottom: 5px;
margin-left: 0;
padding: 4px;
border: 1px solid #cccccc;
}
.profile-data {
margin:0;
text-transform:uppercase;
letter-spacing:.1em;
font: normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif;
color: #999999;
font-weight: bold;
line-height: 1.6em;
}
.profile-datablock {
margin:.5em 0 .5em;
}
.profile-textblock {
margin: 0.5em 0;
line-height: 1.6em;
}
.profile-link {
font: normal normal 78% 'Trebuchet MS', Trebuchet, Arial, Verdana, Sans-serif;
text-transform: uppercase;
letter-spacing: .1em;
}
/* Footer
----------------------------------------------- */
#footer {
width:760px;
clear:both;
margin:0 auto;
padding-top:15px;
line-height: 1.6em;
text-transform:uppercase;
letter-spacing:.1em;
text-align: center;
}
@media only screen and (max-width:480px) {
a:link {
color: #d12a39;
}
.mobile-link-button {
background-color: #d12a39;
}
.mobile-link-button a:link, .mobile-link-button a:visited {
color: #ffffff;
}
}
--></style>
<link href='https://www.blogger.com/dyncss/3615332969083650973/authorization.css?zx=24cd04b2-38ff-45f4-a3f0-f0a07f75ec8f' media='none' onload='if(media!=&#39;all&#39;)media=&#39;all&#39;' rel='stylesheet'/><noscript><link href='https://www.blogger.com/dyncss/3615332969083650973/authorization.css?zx=24cd04b2-38ff-45f4-a3f0-f0a07f75ec8f' rel='stylesheet'/></noscript>
<meta name='google-adsense-platform-account' content='ca-host-pub-1556223355139109'/>
<meta name='google-adsense-platform-domain' content='blogspot.com'/>
<!-- data-ad-client=ca-pub-6194074709963145 -->
</head>
<body>
<div class='navbar section' id='navbar'><div class='widget Navbar' data-version='1' id='Navbar1'><script type="text/javascript">
function setAttributeOnload(object, attribute, val) {
if(window.addEventListener) {
window.addEventListener('load',
function(){ object[attribute] = val; }, false);
} else {
window.attachEvent('onload', function(){ object[attribute] = val; });
}
}
</script>
<div id="navbar-iframe-container"></div>
<script type="text/javascript" src="https://apis.google.com/js/platform.js"></script>
<script type="text/javascript">
gapi.load("gapi.iframes:gapi.iframes.style.bubble", function() {
if (gapi.iframes && gapi.iframes.getContext) {
gapi.iframes.getContext().openChild({
url: 'https://www.blogger.com/navbar/3615332969083650973?po\x3d699285441357427180\x26origin\x3dhttps://sysadvent.blogspot.com',
where: document.getElementById("navbar-iframe-container"),
id: "navbar-iframe"
});
}
});
</script><script type="text/javascript">
(function() {
var script = document.createElement('script');
script.type = 'text/javascript';
script.src = '//pagead2.googlesyndication.com/pagead/js/google_top_exp.js';
var head = document.getElementsByTagName('head')[0];
if (head) {
head.appendChild(script);
}})();
</script>
</div></div>
<div id='outer-wrapper'>
<div id='wrap2'>
<!-- skip links for text browsers -->
<span id='skiplinks' style='display:none;'>
<a href='#main'>
skip to main
</a>
|
<a href='#sidebar'>
skip to sidebar
</a>
</span>
<div id='header-wrapper'>
<div class='header section' id='header'><div class='widget Header' data-version='1' id='Header1'>
<div id='header-inner'>
<a href='https://sysadvent.blogspot.com/' style='display: block'>
<img alt='sysadvent' height='240px; ' id='Header1_headerimg' src='https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiYVlIMEU0sXvTBtcY5iFj5QtiT3Klf0S3sgSamkTPsjU1QGvOWHA-6yq173iHys3-ImrIFY26KIPuKEoSNSbitvtnXsZU6b8YPGQ48ir-8Jo2NFrTsaiO_Nl9W9XLt_wyWhTM_16_HeN0/s750/sys-advent-long.png' style='display: block' width='750px; '/>
</a>
</div>
</div></div>
</div>
<div id='content-wrapper'>
<div id='crosscol-wrapper' style='text-align:center'>
<div class='crosscol no-items section' id='crosscol'></div>
</div>
<div id='main-wrapper'>
<div class='main section' id='main'><div class='widget Blog' data-version='1' id='Blog1'>
<div class='blog-posts hfeed'>
<!--Can't find substitution for tag [defaultAdStart]-->
<div class="date-outer">
<h2 class='date-header'>
<span>
December 13, 2017
</span>
</h2>
<div class="date-posts">
<div class='post-outer'>
<div class='post hentry'>
<a name='699285441357427180'></a>
<h3 class='post-title entry-title'>
<a href='https://sysadvent.blogspot.com/2017/12/day-13-half-dead-tcp-connections-and.html'>
Day 13 - Half-Dead TCP Connections and Why Heartbeats Matter
</a>
</h3>
<div class='post-header-line-1'></div>
<div class='post-body entry-content'>
<p>By: Alejandro Brito Monedero (<a href="https://twitter.com/ae_bm">@ae_bm</a>)<br>
&#8232;Edited By: J. Paul Reed (<a href="https://twitter.com/jpaulreed">@jpaulreed</a>)</p>
<p>We are living interesting times in the tech world, full of trendy technologies, like cloud computing, containers, schedulers, and serverless.</p>
<p>They&rsquo;re all rainbows and unicorns when they work. But we can start to forget about the systems that support our abstractions until they break, and we have to give our best to fix them.</p>
<p>Some time ago in one of our multiple pub-sub systems, there were some processes publishing messages to a message broker. Those messages are consumed by a process running in a container. So far this isn&rsquo;t too exotic; it&rsquo;s easy to diagram out:</p>
<p><img alt="Pub Sub" src="https://github.com/aebm/sysadvent_2017/blob/master/images/pub_sub.png?raw=true"></p>
<p>For the most part, this system worked as expected. However, at one point, we received some alerts from the monitoring system. Those alerts reported that the broker had a lot of queued messages and no consumers connected. Our first reaction was to restart the consumer container and call it a day. But the error kept happening, always at the worst possible time.</p>
<p>While taking a closer look at the problem, we confirmed that the broker has no consumers to deliver the messages. The surprise came when we inspect the consumer container. It was still running and seemingly all was well, except we did notice it was blocked on the socket used to communicate with the broker. When we inspect the socket statistics inside the container&rsquo;s network namespace, it showed a connection to the broker:</p>
<div><pre><code class="language-none"># ss -tpno
ESTAB 0 0 &lt;container ip&gt;:&lt;some port&gt; &lt;broker ip&gt;:&lt;broker port&gt;</code></pre></div>
<p>Upon seeing this, our reaction could pretty much be summed up as:</p>
<p><img alt="" src="https://media.giphy.com/media/l3q2K5jinAlChoCLS/giphy.gif"></p>
<p>The problem seemed to be that the connection state between the broker and consumer was not synchronized. In the host network namespace (where the broker is running), the status showed there weren&rsquo;t any TCP connection from the container. Instead in the container network namespace there is an established connection to the broker. Our dear <a href="https://tools.ietf.org/html/rfc793#page-33">RFC 793</a> mentions this situation <strong>Half-Open Connections and Other Anomalies</strong> (emphases mine):</p>
<blockquote>
<p>An established connection is said to be &ldquo;half-open&rdquo; if one of the TCPs has closed or aborted the connection at its end
without the knowledge of the other, or if the two ends of the connection have become desynchronized owing to a crash that
resulted in loss of memory. <strong>Such connections will automatically become reset if an attempt is made to <em>send data</em> in
either direction</strong>. However, half-open connections are expected to be unusual, and the recovery procedure is mildly
involved.</p>
<p>If at site A the connection no longer exists, then an attempt by the user at site B to <strong>send</strong> any data on it will
result in the site B TCP receiving a <strong>reset</strong> control message. Such a message indicates to the site B TCP that something
is wrong, and it is expected to abort the connection.</p>
</blockquote>
<p>After that nice enlightenment, we started to think of possible causes for that &ldquo;desynchronization.&rdquo; Options that came to mind included:</p>
<ul>
<li>the broker restarting or crashing</li>
<li><a href="https://en.wikipedia.org/wiki/Man-in-the-middle_attack">man-in-the-middle</a> attack (MITM)</li>
<li>A grumpy kernel (iptables, ebtables, bridges, etc)</li>
<li>A grumpy container engine</li>
<li>Some Lovecraftian horror show</li>
</ul>
<p>To determine which it was, we first checked if the broker has crashed or has been restarted. Upon inspection, we found it&rsquo;d been running for a long time and other systems using it were working normally. So it wasn&rsquo;t a problem with the broker.</p>
<p>The kernel iptables and bridge didn&rsquo;t show anything weird. A MITM attack seemed a bit exotic. The other options were hard to prove, and we thought it wouldn&rsquo;t be very professional of us to blame the container system without any evidence. ;-)</p>
<p>While trying to think of other causes it could be, we kept <code>tcpdump</code> running on one of consumer containers. <code>tcpdump</code> captured an <code>RST</code> message sent from the container IP to the broker in response to the broker sending a data message to the consumer container after a long period of inactivity. The weird thing is that network traffic never reached the container, neither the <code>RST</code> originated from the container. Maybe the MITM attack wasn&rsquo;t such an exotic possibility after all?!</p>
<p>Meanwhile, while trying to re-create the problem end state and work toward making our our systems resilient to this situation: we used <code>iptables</code> to drop or reset traffic from the broker to the container after the container connected to the broker. Both methods allowed us to observe the same end-state we were getting in production, confirming the container never learns that the broker connection is lost. Figuring out how to find how to learn that your peer is down even if the TCP connection state is established proved difficult. But after some Internet searching, we found <a href="https://tools.ietf.org/html/rfc1122#page-101">RFC 1122</a>&rsquo;s section on <strong>TCP Keep-Alives</strong> (again emphases mine):</p>
<blockquote>
<p>Implementors <strong>MAY</strong> include &ldquo;keep-alives&rdquo; in their TCP implementations, although <strong>this practice is not universally
accepted</strong>. If keep-alives are included, the application MUST be able to turn them on or off for each TCP connection, and
they MUST default to off.</p>
<p>Keep-alive packets MUST only be sent when <strong>no data or acknowledgement packets have been received for the connection within
an interval</strong>*. This interval MUST be configurable and MUST default to no less than two hours.</p>
<p>&hellip;</p>
<p>DISCUSSION:</p>
<p>A &ldquo;keep-alive&rdquo; mechanism periodically probes the other end of a connection when the connection is otherwise idle, even
when there is no data to be sent. The TCP specification does not include a keep-alive mechanism because it could:<br>
(1) cause perfectly good connections to <strong>break during transient Internet failures;</strong> (2) consume unnecessary bandwidth
(&ldquo;if no one is using the connection, who cares if it is still good?&rdquo;); and (3) cost money for an Internet path that
charges for packets.</p>
<p>&hellip;</p>
<p>A TCP keep-alive mechanism should only be invoked in server applications that might otherwise <strong>hang indefinitely</strong> and
consume resources unnecessarily if a client crashes or aborts a connection during a network failure.</p>
</blockquote>
<p>Translation: distributed systems are fun&hellip; and determining whether a connection is still valid is often the cherry on top.</p>
<p>But before trying to poke at the TCP stack, we kept investigating. We found out that <a href="https://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol">AMQP</a> supports heartbeats, and they are used to check if a connection is <a href="https://www.rabbitmq.com/heartbeats.html">still valid</a>. The library we were using had this option disabled by default, which explains why the container was blocked and waiting instead of trying to reconnect to the broker. To make things worse because the container is a consumer, it never sends data to the broker. If the container has sent data using the same socket it could detect on its own whether the connection was still valid.</p>
<p>To fix this, we evaluated two solutions:</p>
<ul>
<li>The TCP keep-alive fix was the fastest to implement, but we didn&#8217;t like it because it deletgated detection of the broken connection to the kernel TCP implementation. Also we didn&#8217;t really want to mess with kernel socket options.</li>
<li>For an alternative, we ran some tests with other applications and they handled it at the application level (thank you Bandwagon effect). Through this testing, we found we could change the library to activate the AMQP heartbeats. It took more time, but it felt like a better solution to use the mechanisms provided by the AMQP protocol.</li>
</ul>
<p>But what about the MITM attack we thought we were seeing?</p>
<p>First some context: we run periodic, short-lived helper containers. We have observed with <code>tcpdump</code> that when a container starts, it announces some ICMPv6 memberships. Also by default, the container network namespace is attached to a network bridge. The network bridge uses a cache to associate addresses with its respective port, like a network switch. It populates this table when it sees network traffic; as you might imagine, if there isn&#8217;t any traffic for some time, the data in the table becomes stale.</p>
<p>The MITM &ldquo;attack&rdquo; happens when in a period without traffic between the broker and the container, the bridge cache is stale and a short live container is launched, there is a chance for it to get the same IP address the consumer container has. This new container changes the bridge cache, then if the broker sends a message to the consumer, the bridge delivers it to the new container, who then answers with a TCP RST because it doesn&#8217;t have a TCP connection with the broker. Finally the broker receives the TCP RST and aborts its connection with the consumer. The magic of giving the same IP address to different containers.</p>
<p>A picture is worth a thousand words.</p>
<h3>Without the MITM</h3>
<img alt="" src="https://github.com/aebm/sysadvent_2017/raw/master/images/working.png">
<h3>With the MITM</h3>
<img alt="" src="https://github.com/aebm/sysadvent_2017/raw/master/images/mitm.png">
<p>Ultimately, the problem turned out to be one of the most exotic possibilities we had come up with, making us feel pretty:</p>
<p><img alt="" src="https://media.giphy.com/media/l378rrt5tAawaCQ9i/giphy.gif"></p>
<h2 id="toc_1">Conclusion</h2>
<p>Our programs must be prepared to handle network disruptions even when the network traffic doesn&#8217;t leave a single host and you are using containers. Remember: <strong>the network is not reliable</strong>! If we forget this, we will have a lot of &ldquo;fun&rdquo; with distributed systems bugs.</p>
<p>Perhaps more importantly: always remember that if your program never sends traffic on its TCP socket, you can&#8217;t be sure whether the connection is valid or if you will end up waiting for a message that will never arrive.</p>
<p>There are two solutions to avoid this situation: the first is to use TCP keepalives and delegate detection of stale connections to the OS; The other is to implement or use a heartbeat mechanism at a higher layer in the protocol.</p>
<p>Both alternatives have their pros and cons, so you&rsquo;ll need to find which one is best for your team and hte distributed system you run. But now that you&rsquo;ve seen a story where TCP half-open connections, &ldquo;anomelies,&rdquo; and keep-alives all worked together, you&rsquo;ll know that MITM &ldquo;attack&rdquo; might not be such an exotic cause of the problem, even if it&rsquo;s not an attacker trying to get in, but rather your own the kernel.</p>
<h3 id="toc_2">Extras</h3>
<p>While preparing this post, I found <a href="https://blog.stephencleary.com/2009/05/detection-of-half-open-dropped.html">this article</a>, which discusses half open connections; it would have been handy when explaining this problem to my coworkers.</p>
<div style='clear: both;'></div>
</div>
<div class='post-footer'>
<div class='post-footer-line post-footer-line-1'>
<span class='post-author vcard'>
Posted by
<span class='fn'>
cwebber
</span>
</span>
<span class='post-timestamp'>
</span>
<span class='post-comment-link'>
</span>
<span class='post-icons'>
<span class='item-action'>
<a href='https://www.blogger.com/email-post/3615332969083650973/699285441357427180' title='Email Post'>
<img alt="" class="icon-action" height="13" src="//www.blogger.com/img/icon18_email.gif" width="18">
</a>
</span>
<span class='item-control blog-admin pid-1452848009'>
<a href='https://www.blogger.com/post-edit.g?blogID=3615332969083650973&postID=699285441357427180&from=pencil' title='Edit Post'>
<img alt="" class="icon-action" height="18" src="//img2.blogblog.com/img/icon18_edit_allbkg.gif" width="18">
</a>
</span>
</span>
</div>
<div class='post-footer-line post-footer-line-2'>
<span class='post-labels'>
Labels:
<a href='https://sysadvent.blogspot.com/search/label/network' rel='tag'>
network
</a>
,
<a href='https://sysadvent.blogspot.com/search/label/tcp' rel='tag'>
tcp
</a>
</span>
</div>
<div class='post-footer-line post-footer-line-3'></div>
</div>
</div>
<div class='comments' id='comments'>
<a name='comments'></a>
<h4>
No comments
:
</h4>
<div id='Blog1_comments-block-wrapper'>
<dl class='' id='comments-block'>
</dl>
</div>
<p class='comment-footer'>
<a href='https://www.blogger.com/comment/fullpage/post/3615332969083650973/699285441357427180' onclick=''>
Post a Comment
</a>
</p>
<div id='backlinks-container'>
<div id='Blog1_backlinks-container'>
</div>
</div>
</div>
</div>
</div></div>
<!--Can't find substitution for tag [adEnd]-->
</div>
<div class='blog-pager' id='blog-pager'>
<span id='blog-pager-newer-link'>
<a class='blog-pager-newer-link' href='https://sysadvent.blogspot.com/2017/12/day-14-pets-vs-cattle-prods-silence-of.html' id='Blog1_blog-pager-newer-link' title='Newer Post'>
Newer Post
</a>
</span>
<span id='blog-pager-older-link'>
<a class='blog-pager-older-link' href='https://sysadvent.blogspot.com/2017/12/day-12-monitoring-postgres-replication.html' id='Blog1_blog-pager-older-link' title='Older Post'>
Older Post
</a>
</span>
<a class='home-link' href='https://sysadvent.blogspot.com/'>
Home
</a>
</div>
<div class='clear'></div>
<div class='post-feeds'>
<div class='feed-links'>
Subscribe to:
<a class='feed-link' href='https://sysadvent.blogspot.com/feeds/699285441357427180/comments/default' target='_blank' type='application/atom+xml'>
Post Comments
(
Atom
)
</a>
</div>
</div>
</div></div>
</div>
<div id='sidebar-wrapper'>
<div class='sidebar section' id='sidebar'><div class='widget HTML' data-version='1' id='HTML2'>
<h2 class='title'>
What is sysadvent?
</h2>
<div class='widget-content'>
<p>
One article for each day of December, ending on the 25th article.
</p><p>
With the goals of sharing, openness, and mentoring, we aim to provide great articles about systems administration topics written by fellow sysadmins.
</p><p>
Want to get involved? Join the <a href="http://groups.google.com/group/sysadvent">mailing list</a>!
<p></p></p>
</div>
<div class='clear'></div>
</div><div class='widget Subscribe' data-version='1' id='Subscribe1'>
<div style='white-space:nowrap'>
<h2 class='title'>
Subscribe
</h2>
<div class='widget-content'>
<div class='subscribe-wrapper subscribe-type-POST'>
<div class='subscribe expanded subscribe-type-POST' id='SW_READER_LIST_Subscribe1POST' style='display:none;'>
<div class='top'>
<span class='inner' onclick='return(_SW_toggleReaderList(event, "Subscribe1POST"));'>
<img class='subscribe-dropdown-arrow' src='https://resources.blogblog.com/img/widgets/arrow_dropdown.gif'/>
<img align='absmiddle' alt='' border='0' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Posts
</span>
<div class='feed-reader-links'>
<a class='feed-reader-link' href='http://www.netvibes.com/subscribe.php?url=https%3A%2F%2Fsysadvent.blogspot.com%2Ffeeds%2Fposts%2Fdefault' target='_blank'>
<img src='https://resources.blogblog.com/img/widgets/subscribe-netvibes.png'/>
</a>
<a class='feed-reader-link' href='http://add.my.yahoo.com/content?url=https%3A%2F%2Fsysadvent.blogspot.com%2Ffeeds%2Fposts%2Fdefault' target='_blank'>
<img src='https://resources.blogblog.com/img/widgets/subscribe-yahoo.png'/>
</a>
<a class='feed-reader-link' href='https://sysadvent.blogspot.com/feeds/posts/default' target='_blank'>
<img align='absmiddle' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Atom
</a>
</div>
</div>
<div class='bottom'></div>
</div>
<div class='subscribe' id='SW_READER_LIST_CLOSED_Subscribe1POST' onclick='return(_SW_toggleReaderList(event, "Subscribe1POST"));'>
<div class='top'>
<span class='inner'>
<img class='subscribe-dropdown-arrow' src='https://resources.blogblog.com/img/widgets/arrow_dropdown.gif'/>
<span onclick='return(_SW_toggleReaderList(event, "Subscribe1POST"));'>
<img align='absmiddle' alt='' border='0' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Posts
</span>
</span>
</div>
<div class='bottom'></div>
</div>
</div>
<div class='subscribe-wrapper subscribe-type-PER_POST'>
<div class='subscribe expanded subscribe-type-PER_POST' id='SW_READER_LIST_Subscribe1PER_POST' style='display:none;'>
<div class='top'>
<span class='inner' onclick='return(_SW_toggleReaderList(event, "Subscribe1PER_POST"));'>
<img class='subscribe-dropdown-arrow' src='https://resources.blogblog.com/img/widgets/arrow_dropdown.gif'/>
<img align='absmiddle' alt='' border='0' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Comments
</span>
<div class='feed-reader-links'>
<a class='feed-reader-link' href='http://www.netvibes.com/subscribe.php?url=https%3A%2F%2Fsysadvent.blogspot.com%2Ffeeds%2F699285441357427180%2Fcomments%2Fdefault' target='_blank'>
<img src='https://resources.blogblog.com/img/widgets/subscribe-netvibes.png'/>
</a>
<a class='feed-reader-link' href='http://add.my.yahoo.com/content?url=https%3A%2F%2Fsysadvent.blogspot.com%2Ffeeds%2F699285441357427180%2Fcomments%2Fdefault' target='_blank'>
<img src='https://resources.blogblog.com/img/widgets/subscribe-yahoo.png'/>
</a>
<a class='feed-reader-link' href='https://sysadvent.blogspot.com/feeds/699285441357427180/comments/default' target='_blank'>
<img align='absmiddle' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Atom
</a>
</div>
</div>
<div class='bottom'></div>
</div>
<div class='subscribe' id='SW_READER_LIST_CLOSED_Subscribe1PER_POST' onclick='return(_SW_toggleReaderList(event, "Subscribe1PER_POST"));'>
<div class='top'>
<span class='inner'>
<img class='subscribe-dropdown-arrow' src='https://resources.blogblog.com/img/widgets/arrow_dropdown.gif'/>
<span onclick='return(_SW_toggleReaderList(event, "Subscribe1PER_POST"));'>
<img align='absmiddle' alt='' border='0' class='feed-icon' src='https://resources.blogblog.com/img/icon_feed12.png'/>
Comments
</span>
</span>
</div>
<div class='bottom'></div>
</div>
</div>
<div style='clear:both'></div>
</div>
</div>
<div class='clear'></div>
</div><div class='widget BlogArchive' data-version='1' id='BlogArchive2'>
<h2>
Blog Archive
</h2>
<div class='widget-content'>
<div id='ArchiveList'>
<div id='BlogArchive2_ArchiveList'>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2021/'>
2021
</a>
<span class='post-count' dir='ltr'>
(
22
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2021/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
22
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2019/'>
2019
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2019/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate expanded'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy toggle-open'>
&#9660;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2017/'>
2017
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate expanded'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy toggle-open'>
&#9660;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2017/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='posts'>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-25-how-to-choose-data-store-for-new.html'>
Day 25 - How to choose a data store for the new sh...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-24-on-premise-kubernetes-with.html'>
Day 24 - On-premise Kubernetes with dynamic load b...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-23-open-source-licensing-in-real.html'>
Day 23 - Open Source Licensing in the Real World
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-22-building-secure-bastion-host-or.html'>
Day 22 - Building a secure bastion host, or, 50 wa...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-21-lighting-up-your-haunted.html'>
Day 21 - Lighting Up Your Haunted Graveyards
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-20-regarding-responsibility-of.html'>
Day 20 - Regarding the Responsibility of Systems A...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-19-infrastructure-testing-sanity.html'>
Day 19 - Infrastructure Testing: Sanity Checking V...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-18-awesome-command-line-fuzzy.html'>
Day 18 - Awesome command-line fuzzy finding with fzf
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-17-don-fall-for-hybrid-cloud-trap.html'>
Day 17 - Don&#8217;t Fall for the Hybrid Cloud Trap
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-16-inspec-gives-insight.html'>
Day 16 - Inspec gives insight
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-15-devops-christmas-carol.html'>
Day 15 - A DevOps Christmas Carol
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-14-pets-vs-cattle-prods-silence-of.html'>
Day 14 - Pets vs. Cattle Prods: The Silence of the...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-13-half-dead-tcp-connections-and.html'>
Day 13 - Half-Dead TCP Connections and Why Heartbe...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-12-monitoring-postgres-replication.html'>
Day 12 - Monitoring Postgres Replication Lag
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-11-scaling-your-on-duty-team.html'>
Day 11 - Scaling your on-duty team
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-10-from-product-eng-to-systems-eng.html'>
Day 10 - From Product Eng to Systems Eng
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-9-using-kubernetes-for-multi.html'>
Day 9 - Using Kubernetes for multi-provider, multi...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-8-breaking-in-new-company-as-sre.html'>
Day 8 - Breaking in a New Company as an SRE
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-7-running-inspec-as-push-job.html'>
Day 7 - Running InSpec as a Push Job&#8230;or&#8230;The Nightm...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-6-sysadmins-evolution-of-role.html'>
Day 6 - sysadmins - the evolution of a role amidst...
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-5-do-you-want-to-build-helm-chart.html'>
Day 5 - Do you want to build a helm chart?
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-4-this-christmas-build-your-own-pki.html'>
Day 4 - This Christmas, build your own PKI
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-3-root-cause-is-plural.html'>
Day 3 - Root Cause is Plural
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-2-shifting-left-securely-with-inspec.html'>
Day 2 - Shifting Left Securely With inSpec
</a>
</li>
<li>
<a href='https://sysadvent.blogspot.com/2017/12/day-1-comfortably-numb-or-corporate.html'>
Day 1 - Comfortably Numb - or - &#8220;How Corporate Ana...
</a>
</li>
</ul>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2016/'>
2016
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2016/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2015/'>
2015
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2015/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2014/'>
2014
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2014/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2013/'>
2013
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2013/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2012/'>
2012
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2012/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2011/'>
2011
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2011/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2010/'>
2010
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2010/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2009/'>
2009
</a>
<span class='post-count' dir='ltr'>
(
26
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2009/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
26
)
</span>
</li>
</ul>
</li>
</ul>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2008/'>
2008
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
<ul class='hierarchy'>
<li class='archivedate collapsed'>
<a class='toggle' href='javascript:void(0)'>
<span class='zippy'>
&#9658;&#160;
</span>
</a>
<a class='post-count-link' href='https://sysadvent.blogspot.com/2008/12/'>
December
</a>
<span class='post-count' dir='ltr'>
(
25
)
</span>
</li>
</ul>
</li>
</ul>
</div>
</div>
<div class='clear'></div>
</div>
</div></div>
</div>
<!-- spacer for skins that want sidebar and main to be the same height-->
<div class='clear'>
&#160;
</div>
</div>
<!-- end content-wrapper -->
<div id='footer-wrapper'>
<div class='footer no-items section' id='footer'></div>
</div>
</div>
</div>
<!-- end outer-wrapper -->
<script type='text/javascript'>
var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
</script>
<script type='text/javascript'>
try {
var pageTracker = _gat._getTracker("UA-6522917-1");
pageTracker._trackPageview();
} catch(err) {}</script>
<script type="text/javascript" src="https://www.blogger.com/static/v1/widgets/1363620594-widgets.js"></script>
<script type='text/javascript'>
window['__wavt'] = 'AAPvtVkSOrjwzWruuDgYK4fHQwxP:1789070106567';_WidgetManager._Init('//www.blogger.com/rearrange?blogID\x3d3615332969083650973','//sysadvent.blogspot.com/2017/12/day-13-half-dead-tcp-connections-and.html','3615332969083650973');
_WidgetManager._SetDataContext([{'name': 'blog', 'data': {'blogId': '3615332969083650973', 'title': 'sysadvent', 'url': 'https://sysadvent.blogspot.com/2017/12/day-13-half-dead-tcp-connections-and.html', 'canonicalUrl': 'https://sysadvent.blogspot.com/2017/12/day-13-half-dead-tcp-connections-and.html', 'homepageUrl': 'https://sysadvent.blogspot.com/', 'searchUrl': 'https://sysadvent.blogspot.com/search', 'canonicalHomepageUrl': 'https://sysadvent.blogspot.com/', 'blogspotFaviconUrl': 'https://sysadvent.blogspot.com/favicon.ico', 'bloggerUrl': 'https://www.blogger.com', 'hasCustomDomain': false, 'httpsEnabled': true, 'enabledCommentProfileImages': false, 'gPlusViewType': 'FILTERED_POSTMOD', 'adultContent': false, 'analyticsAccountNumber': '', 'encoding': 'UTF-8', 'locale': 'en', 'localeUnderscoreDelimited': 'en', 'languageDirection': 'ltr', 'isPrivate': false, 'isMobile': false, 'isMobileRequest': false, 'mobileClass': '', 'isPrivateBlog': false, 'isDynamicViewsAvailable': true, 'feedLinks': '\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22sysadvent - Atom\x22 href\x3d\x22https://sysadvent.blogspot.com/feeds/posts/default\x22 /\x3e\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/rss+xml\x22 title\x3d\x22sysadvent - RSS\x22 href\x3d\x22https://sysadvent.blogspot.com/feeds/posts/default?alt\x3drss\x22 /\x3e\n\x3clink rel\x3d\x22service.post\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22sysadvent - Atom\x22 href\x3d\x22https://www.blogger.com/feeds/3615332969083650973/posts/default\x22 /\x3e\n\n\x3clink rel\x3d\x22alternate\x22 type\x3d\x22application/atom+xml\x22 title\x3d\x22sysadvent - Atom\x22 href\x3d\x22https://sysadvent.blogspot.com/feeds/699285441357427180/comments/default\x22 /\x3e\n', 'meTag': '', 'adsenseClientId': 'ca-pub-6194074709963145', 'adsenseHostId': 'ca-host-pub-1556223355139109', 'adsenseHasAds': false, 'adsenseAutoAds': false, 'boqCommentIframeForm': true, 'loginRedirectParam': '', 'isGoogleEverywhereLinkTooltipEnabled': true, 'view': '', 'dynamicViewsCommentsSrc': '//www.blogblog.com/dynamicviews/4224c15c4e7c9321/js/comments.js', 'dynamicViewsScriptSrc': '//www.blogblog.com/dynamicviews/1890a9484e710b1c', 'plusOneApiSrc': 'https://apis.google.com/js/platform.js', 'disableGComments': true, 'interstitialAccepted': false, 'sharing': {'platforms': [{'name': 'Get link', 'key': 'link', 'shareMessage': 'Get link', 'target': ''}, {'name': 'Facebook', 'key': 'facebook', 'shareMessage': 'Share to Facebook', 'target': 'facebook'}, {'name': 'BlogThis!', 'key': 'blogThis', 'shareMessage': 'BlogThis!', 'target': 'blog'}, {'name': 'X', 'key': 'twitter', 'shareMessage': 'Share to X', 'target': 'twitter'}, {'name': 'Pinterest', 'key': 'pinterest', 'shareMessage': 'Share to Pinterest', 'target': 'pinterest'}, {'name': 'Email', 'key': 'email', 'shareMessage': 'Email', 'target': 'email'}], 'disableGooglePlus': true, 'googlePlusShareButtonWidth': 0, 'googlePlusBootstrap': '\x3cscript type\x3d\x22text/javascript\x22\x3ewindow.___gcfg \x3d {\x27lang\x27: \x27en\x27};\x3c/script\x3e'}, 'hasCustomJumpLinkMessage': false, 'jumpLinkMessage': 'Read more', 'pageType': 'item', 'postId': '699285441357427180', 'postImageUrl': 'https://github.com/aebm/sysadvent_2017/blob/master/images/pub_sub.png?raw\x3dtrue', 'pageName': 'Day 13 - Half-Dead TCP Connections and Why Heartbeats Matter', 'pageTitle': 'sysadvent: Day 13 - Half-Dead TCP Connections and Why Heartbeats Matter'}}, {'name': 'features', 'data': {}}, {'name': 'messages', 'data': {'edit': 'Edit', 'linkCopiedToClipboard': 'Link copied to clipboard!', 'ok': 'Ok', 'postLink': 'Post Link'}}, {'name': 'template', 'data': {'name': 'custom', 'localizedName': 'Custom', 'isResponsive': false, 'isAlternateRendering': false, 'isCustom': true}}, {'name': 'view', 'data': {'classic': {'name': 'classic', 'url': '?view\x3dclassic'}, 'flipcard': {'name': 'flipcard', 'url': '?view\x3dflipcard'}, 'magazine': {'name': 'magazine', 'url': '?view\x3dmagazine'}, 'mosaic': {'name': 'mosaic', 'url': '?view\x3dmosaic'}, 'sidebar': {'name': 'sidebar', 'url': '?view\x3dsidebar'}, 'snapshot': {'name': 'snapshot', 'url': '?view\x3dsnapshot'}, 'timeslide': {'name': 'timeslide', 'url': '?view\x3dtimeslide'}, 'isMobile': false, 'title': 'Day 13 - Half-Dead TCP Connections and Why Heartbeats Matter', 'description': 'By: Alejandro Brito Monedero ( @ae_bm ) \u2028Edited By: J. Paul Reed ( @jpaulreed ) We are living interesting times in the tech world, full o...', 'featuredImage': 'https://lh3.googleusercontent.com/blogger_img_proxy/AEn0k_t5kc9q__Co9QNo_k3ZHJ4ijB-KHFpR3oA6WXQY5LqxyITEqxGvy3I8svKnJDbYzFWD2y8aVJirErB4-UKGHhL3sByqvF2ztvDEac5GKuHl112jYgsr1mdNF1g65V9W-NbWEW3l0sPIBF4qbY7gd4PtqQ', 'url': 'https://sysadvent.blogspot.com/2017/12/day-13-half-dead-tcp-connections-and.html', 'type': 'item', 'isSingleItem': true, 'isMultipleItems': false, 'isError': false, 'isPage': false, 'isPost': true, 'isHomepage': false, 'isArchive': false, 'isLabelSearch': false, 'postId': 699285441357427180}}]);
_WidgetManager._RegisterWidget('_NavbarView', new _WidgetInfo('Navbar1', 'navbar', document.getElementById('Navbar1'), {}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_HeaderView', new _WidgetInfo('Header1', 'header', document.getElementById('Header1'), {}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_BlogView', new _WidgetInfo('Blog1', 'main', document.getElementById('Blog1'), {'cmtInteractionsEnabled': false, 'lightboxEnabled': true, 'lightboxModuleUrl': 'https://www.blogger.com/static/v1/jsbin/700075392-lbx.js', 'lightboxCssUrl': 'https://www.blogger.com/static/v1/v-css/828616780-lightbox_bundle.css'}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_HTMLView', new _WidgetInfo('HTML2', 'sidebar', document.getElementById('HTML2'), {}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_SubscribeView', new _WidgetInfo('Subscribe1', 'sidebar', document.getElementById('Subscribe1'), {}, 'displayModeFull'));
_WidgetManager._RegisterWidget('_BlogArchiveView', new _WidgetInfo('BlogArchive2', 'sidebar', document.getElementById('BlogArchive2'), {'languageDirection': 'ltr', 'loadingMessage': 'Loading\x26hellip;'}, 'displayModeFull'));
</script>
</body>
</html>