Files
nexus/sreweekly/articles/102/03-reminders-about-using-traceroute-in-multi-path-networks.html
2026-09-12 17:23:01 +08:00

3 lines
24 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!doctype html><html lang=en-us><head><link rel=icon href=../../favicon_main.svg><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta http-equiv=x-ua-compatible content="ie=edge"><meta property="og:url" content="https://michael-kehoe.io/post/reminders-about-using-traceroute-in-multi-path-networks/"><meta property="og:type" content="article"><meta property="og:title" content="Reminders about using traceroute in multi-path networks"><meta property="og:description" content><meta property="og:image:width" content="375"><meta property="og:image:height" content="250"><meta property="og:image" content="https://michael-kehoe.io/images/default_summary_hu2be732aee21a2469ead3fe1f2df8caca_2088687_600x0_resize_q75_box.jpg"><title>Michael Kehoe - SRE</title><link href="https://fonts.googleapis.com/css?family=Open+Sans" rel=stylesheet><link href="https://fonts.googleapis.com/css?family=Open+Sans" rel=stylesheet><link href="https://fonts.googleapis.com/css?family=Open+Sans" rel=stylesheet><link rel=stylesheet type=text/css href=../../css/style.min.9550457997b8fb88858cc0ba56a42685802c65e54d799dba46adc1649cdd7c03.css integrity="sha256-lVBFeZe4+4iFjMC6VqQmhYAsZeVNeZ26Rq3BZJzdfAM="><link rel=stylesheet type=text/css href=../../css/monokai-sublime.9.15.8.min.min.3a5d282f03108101d715e80fd4c07b55502ec4673fc063f2b6e415d4def5b354.css integrity="sha256-Ol0oLwMQgQHXFegP1MB7VVAuxGc/wGPytuQV1N71s1Q="><link rel=stylesheet type=text/css href=../../css/icons.min.ee23494515027c9d455d659dd0760f472e9c02e16079f933928bc4820d5b9e58.css integrity="sha256-7iNJRRUCfJ1FXWWd0HYPRy6cAuFgefkzkovEgg1bnlg="><link rel=stylesheet type=text/css href=../../css/refresh.min.64bc524382017d235682ac49a1f565e2e83d4a52ec42d323f0ee9dc8a92cf278.css integrity="sha256-ZLxSQ4IBfSNWgqxJofVl4ug9SlLsQtMj8O6dyKks8ng="><link rel=stylesheet type=text/css href=../../css/devicon.min.min.13fddf418611477a10293a87542de7703396184696bd8cf50642c4c72994c78a.css integrity="sha256-E/3fQYYRR3oQKTqHVC3ncDOWGEaWvYz1BkLExymUx4o="></head><body><script async src="https://www.googletagmanager.com/gtag/js?id=UA-164510550-1"></script><script>window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments);}
gtag('js',new Date());gtag('config','UA-164510550-1');</script><div id=preloader><div id=status></div></div><nav class="navbar is-fresh is-transparent no-shadow" role=navigation aria-label="main navigation"><div class=container><div class=navbar-brand><a class=navbar-item><div class="menu-icon-wrapper left-menu-icon-wrapper" style=visibility:visible><svg width="1e3" height="1e3"><path class="path1" d="M3e2 4e2H7e2c2e2.0 2e2 350-1e2 450A4e2 4e2.0 012e2 2e2L8e2 8e2"/><path class="path2" d="M3e2 5e2H7e2"/><path class="path3" d="M7e2 6e2H3e2c-2e2.0-2e2-4e2 1e2-450A4e2 380 0 112e2 8e2L8e2 2e2"/></svg><button id=menu-icon-trigger class=menu-icon-trigger></button></div><div class="navbar-item left-menu-icon-wrapper">Tags</div></a><div class="navbar-item is-expanded"></div><a class="navbar-item is-hidden-desktop"><div data-target=navbar-menu class="navbar-item right-menu-icon-wrapper is-hidden-desktop">Menu</div><div data-target=navbar-menu class="menu-icon-wrapper right-menu-icon-wrapper" style=visibility:visible><svg width="1e3" height="1e3"><path class="path1" d="M3e2 4e2H7e2c2e2.0 2e2 350-1e2 450A4e2 4e2.0 012e2 2e2L8e2 8e2"/><path class="path2" d="M3e2 5e2H7e2"/><path class="path3" d="M7e2 6e2H3e2c-2e2.0-2e2-4e2 1e2-450A4e2 380 0 112e2 8e2L8e2 2e2"/></svg><button id=menu-icon-trigger class=menu-icon-trigger></button></div></a></div><div id=navbar-menu class="navbar-menu is-static"><div class=navbar-end><a href=../../ class="navbar-item is-secondary">Home</a><a href=../../speaking/ class="navbar-item is-secondary">Speaking</a><a href=../../writing/ class="navbar-item is-secondary">Writing</a><a href=../../media/ class="navbar-item is-secondary">Media</a><a href=../../about/ class="navbar-item is-secondary">About Me</a><a href=../../post/ class="navbar-item is-secondary">Posts</a></div></div></div></nav><nav id=navbar-clone class="navbar is-fresh is-transparent" role=navigation aria-label="main navigation"><div class=container><div class=navbar-brand><a class=navbar-item><div class="menu-icon-wrapper left-menu-icon-wrapper" style=visibility:visible><svg width="1e3" height="1e3"><path class="path1" d="M3e2 4e2H7e2c2e2.0 2e2 350-1e2 450A4e2 4e2.0 012e2 2e2L8e2 8e2"/><path class="path2" d="M3e2 5e2H7e2"/><path class="path3" d="M7e2 6e2H3e2c-2e2.0-2e2-4e2 1e2-450A4e2 380 0 112e2 8e2L8e2 2e2"/></svg><button id=menu-icon-trigger class=menu-icon-trigger></button></div><div class="navbar-item left-menu-icon-wrapper">Tags</div></a><div class="navbar-item is-expanded"></div><a class="navbar-item is-hidden-desktop"><div data-target=cloned-navbar-menu class="navbar-item right-menu-icon-wrapper is-hidden-desktop">Menu</div><div data-target=cloned-navbar-menu class="menu-icon-wrapper right-menu-icon-wrapper" style=visibility:visible><svg width="1e3" height="1e3"><path class="path1" d="M3e2 4e2H7e2c2e2.0 2e2 350-1e2 450A4e2 4e2.0 012e2 2e2L8e2 8e2"/><path class="path2" d="M3e2 5e2H7e2"/><path class="path3" d="M7e2 6e2H3e2c-2e2.0-2e2-4e2 1e2-450A4e2 380 0 112e2 8e2L8e2 2e2"/></svg><button id=menu-icon-trigger class=menu-icon-trigger></button></div></a></div><div id=cloned-navbar-menu class="navbar-menu is-static"><div class=navbar-end><a href=../../ class="navbar-item is-secondary">Home</a><a href=../../speaking/ class="navbar-item is-secondary">Speaking</a><a href=../../writing/ class="navbar-item is-secondary">Writing</a><a href=../../media/ class="navbar-item is-secondary">Media</a><a href=../../about/ class="navbar-item is-secondary">About Me</a><a href=../../post/ class="navbar-item is-secondary">Posts</a></div></div></div></nav><section class="section is-medium"><div class=container><div class=columns><div class="column is-centered-tablet-portrait"><h1 class="title is-2 section-title">Reminders about using traceroute in multi-path networks</h1><h5 class="subtitle is-5 is-muted"></h5><div class=divider></div><section class="section content has-text-justified"><h3 id=introduction>Introduction</h3><p>First off, I would like to give a plug for <a href=https://www.nanog.org/meetings/nanog45/presentations/Sunday/RAS_traceroute_N45.pdf>this</a> presentation about how to use/ interpret traceroute.</p><p>Traceroute was written before the times of MPLS, ECMP and Clos network designs. Since the rise of those technologies/ topologies, the way that engineers troubleshoot latency/ packet-loss hasn’t really evolved.</p><p>If you look at companies like Google/ Facebook/ LinkedIn, they all run Clos topologies within their production datacenters. So what does mean to engineers troubleshooting production issues?</p><p>Firstly, the number of paths from <em>Host A</em> to <em>Host B</em> is usually going to be > 1. This means <a href=https://en.wikipedia.org/wiki/MTR_(software)>MTR</a> (and some default uses of <a href=https://en.wikipedia.org/wiki/Traceroute>traceroute</a> are largely obselete.</p><h3 id=example>Example</h3><p>Systems engineer A complains about higher latency from his application (spread across the datacenter) to the Oracle database that his application uses. Systems engineer B also complains that his (different) application is seeing higher latency to a different Oracle database (which are located in the same segment of the network).</p><p>The go-to troubleshooting step here is to ask the DBA to verify if they are seeing higher execution times on the database which correlates in the percieved rise in latency. In this case, the DBA says there is no issue.</p><p>The systems engineers then go to the network engineers and ask for help diagnosing the issue, confident that it isn’t an application-level issue. The network engineers ask for a traceroute and an <a href=https://en.wikipedia.org/wiki/MTR_(software)>mtr</a> print-out. This would be perfect in an older-style network topology, but is not good for Clos style networks. In this particular case, there is ~20 distinct paths between <em>Host A</em> (application) and *Host B *(database). In a best case scenario, traceroute might show you 2-3 paths (by default), mtr will only show you one (the first one it discovers). The chances of you finding the bad path is unlikely.</p><h3 id=what-to-do-next>What to do next?</h3><p>Linux traceroute (with a few arguments) can actually help you here. traceroute allows you to send multiple queries to help discover all the interfaces you’d be routed through and potentially see interfaces with packet-loss or higher latency.</p><p>e.g.</p><p><code>[root@hostA ~]$ traceroute -q 10 hostB</code></p><p>This may not discover all paths, but it is a start. Ofcourse you can run it multiple times to build a bigger picture of the network topology.</p><h3 id=tools>Tools</h3><ul><li>Facebook wrote <a href=https://code.facebook.com/posts/1534350660228025/netnorad-troubleshooting-networks-via-end-to-end-probing/>this</a> great article on troubleshooting networks and released <a href=https://github.com/facebook/fbtracert>fbtracert</a> which is designed to fix this exact problem.</li><li><a href=https://paris-traceroute.net/>Paris Traceroute</a> is another open-source tool aiming to address this space. I haven’t used it, so I cannot vouch for its usefulness.</li></ul></section></div></div></div><div class=container><div class=columns><div class="column has-text-right is-family-monospace is-centered-tablet-portrait">Last modified:&nbsp;25 April 2020</div></div></div></section><footer class="footer footer-dark"><div class=container><div class=columns><div class=column><img src=../../footer.svg alt></div><div class=column><div class=footer-column><div class=footer-header><h3>Website</h3></div><ul class=link-list><li><a href=../../><span class=icon><i class="fa fa-home"></i></span>Home</a></li><li><a href=../../credits/><span class=icon><i class="fa fa-cube"></i></span>Credits</a></li><li><a href=../../tags/><span class=icon><i class="fa fa-tag"></i></span>All Tags</a></li></ul></div></div><div class=column><div class=footer-column><div class=footer-header><h3>Contacts</h3></div><ul class=link-list><li><a href=https://www.linkedin.com/in/michaelkkehoe target=_blank><span class=icon><i class="fa fa-linkedin"></i></span>LinkedIn</a></li><li><a href=https://github.com/michaelkkehoe target=_blank><span class=icon><i class="fa fa-github"></i></span>Github</a></li><li><a href=https://twitter.com/michaelkkehoe target=_blank><span class=icon><i class="fa fa-twitter"></i></span>Twitter</a></li><li><a href=https://slideshare.net/MichaelKehoe3 target=_blank><span class=icon><i class="fa fa-slideshare"></i></span>slideshare</a></li></ul></div></div><div class=column><div class=footer-column><div class=footer-header><h3>Copyright</h3></div><ul class=link-list><li><a><span class=icon><i class="fa fa-copyright"></i></span>Michael Kehoe - 2020</a></li></ul></div></div></div></div></footer><div id=backtotop><a href=#></a></div><div class="sidebar scroll"><div class=sidebar-header><img src=../../sidebar.svg alt>
<a class=sidebar-close href=javascript:void(0);><i data-feather=x></i></a></div><div class=inner><ul class=sidebar-menu><li class=no-children><a href=../../tags/><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cubes"></i></span>All Tags</td><td class=has-text-right></td></tr></table></div></a><li class=no-children><a href=../../tags/conferences><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>conferences</td><td class=has-text-right><div class=tag-number>14</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/weekly-wrap><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>weekly-wrap</td><td class=has-text-right><div class=tag-number>12</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/publications><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>publications</td><td class=has-text-right><div class=tag-number>7</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/linux><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>linux</td><td class=has-text-right><div class=tag-number>5</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/updates><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>updates</td><td class=has-text-right><div class=tag-number>5</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/linkedin><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>LinkedIn</td><td class=has-text-right><div class=tag-number>4</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/monitorama><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>monitorama</td><td class=has-text-right><div class=tag-number>4</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/srecon><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>srecon</td><td class=has-text-right><div class=tag-number>4</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/command-of-the-day><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>command-of-the-day</td><td class=has-text-right><div class=tag-number>3</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/create-blog><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>create-blog</td><td class=has-text-right><div class=tag-number>3</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/hugo><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>hugo</td><td class=has-text-right><div class=tag-number>3</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/recruiting><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>recruiting</td><td class=has-text-right><div class=tag-number>3</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/security><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>security</td><td class=has-text-right><div class=tag-number>3</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/sre><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>SRE</td><td class=has-text-right><div class=tag-number>3</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/static-site><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>static-site</td><td class=has-text-right><div class=tag-number>3</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/future-of-sre><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>future-of-sre</td><td class=has-text-right><div class=tag-number>2</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/home-assistant><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>home assistant</td><td class=has-text-right><div class=tag-number>2</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/incident-management><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>incident-management</td><td class=has-text-right><div class=tag-number>2</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/inovelli><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>inovelli</td><td class=has-text-right><div class=tag-number>2</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/interop-itx><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>interop-itx</td><td class=has-text-right><div class=tag-number>2</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/networking><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>networking</td><td class=has-text-right><div class=tag-number>2</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/next-play><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>next-play</td><td class=has-text-right><div class=tag-number>2</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/postmortem><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>postmortem</td><td class=has-text-right><div class=tag-number>2</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/smart-home><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>smart home</td><td class=has-text-right><div class=tag-number>2</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/ssh><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>ssh</td><td class=has-text-right><div class=tag-number>2</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/2fa><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>2FA</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/blog><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>Blog</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/chaos-engineering><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>chaos-engineering</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/cloud-native-infrastructure-azure><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>cloud-native-infrastructure-azure</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/confluent><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>confluent</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/ebpf><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>ebpf</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/full-stack-journey><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>full-stack-journey</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/interviewing><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>interviewing</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/iptables><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>iptables</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/job><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>job</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/lldp><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>lldp</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/podcast><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>podcast</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/questions><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>questions</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/speaking><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>speaking</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/ubiquiti><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>ubiquiti</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/yubico><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>Yubico</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/yuibkey><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>Yuibkey</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/z2m><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>z2m</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/zigbee><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>zigbee</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/zigbee2mqtt><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>zigbee2mqtt</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li><li class=no-children><a href=../../tags/zwavejs><div class=columns><table width=100%><tr><td><span class=icon><i class="fa fa-cube"></i></span>zwavejs</td><td class=has-text-right><div class=tag-number>1</div></td></tr></table></div></a></li></ul></div></div><script src=../../js/jquery-2.2.4.min.dead7ea11aa9f68488bfe2f9a1be00b7773bea3874f0a221cb898f5c4b041b6a.js integrity="sha256-3q1+oRqp9oSIv+L5ob4At3c76jh08KIhy4mPXEsEG2o="></script><script src=../../js/feather.4.22.0.min.df46a360906c5f1e61c5867bb42b1f7a8b0166f8351e175a1a4f2958f331e42e.js integrity="sha256-30ajYJBsXx5hxYZ7tCsfeosBZvg1HhdaGk8pWPMx5C4="></script><script src=../../js/modernizr-3.6.0.min.2f19d51e4b89c87bdabcdb05684b34f3c1ac6e5e3919ae1c067377c4f64f23cc.js integrity="sha256-LxnVHkuJyHvavNsFaEs088Gsbl45Ga4cBnN3xPZPI8w="></script><script src=../../js/refresh.min.58d0ebd018c8eacfa18b730fb90e94bc3e3d1b009ae12f99fb0f8f5f4bf6a581.js integrity="sha256-WNDr0BjI6s+hi3MPuQ6UvD49GwCa4S+Z+w+PX0v2pYE="></script><script>window.MathJax={loader:{load:['core','input/tex-base','output/chtml'],source:{'core':'\/js\/mathjax\/core.min.d48fedf25c74c54fa6bf79646de92b02155872bdc4f5f7d0bbfc662523d8b4f5.js','input/tex-base':'\/js\/mathjax\/tex-base.min.1b68b8741dfc54e8f7222f88bea8ffcfc57ac54b2a2d8f4edf6800aa44d49441.js','output/chtml':'\/js\/mathjax\/chtml.min.926cd166e0f8c1f8a566a718d60c9c58a2b7142b156d30d5f02cec7c3b0ad60a.js','output/chtml/fonts/tex':'\/js\/mathjax\/tex_out.min.b8b2bb939c0dae84bf1390bfe6d32af13e83f647cdac01d544d2a7a517477e9d.js'},},chtml:{fontURL:'/fonts'},};</script><script src=../../js/mathjax/startup.min.234a2513e6bdbc1eee06ca19abceca30fe4034e82afb373c08274c1ba2feb1a6.js integrity="sha256-I0olE+a9vB7uBsoZq87KMP5ANOgq+zc8CCdMG6L+saY="></script><script src=../../js/highlight.9.18.1.min.9b10829380588e6015e08971bb20e0351dc94741846b6747d60778c696d00f94.js integrity="sha256-mxCCk4BYjmAV4IlxuyDgNR3JR0GEa2dH1gd4xpbQD5Q="></script><script src=../../js/highlightjs-line-numbers.2.7.0.min.min.ddfe282e07b7ec1ed069c23f92c7c8216ddb3f1879c4e962d37fd52adbd15a05.js integrity="sha256-3f4oLge37B7QacI/ksfIIW3bPxh5xOli03/VKtvRWgU="></script><script>hljs.initHighlightingOnLoad();hljs.initLineNumbersOnLoad();document.addEventListener('DOMContentLoaded',(event)=>{document.querySelectorAll('.codeinline').forEach((block)=>{hljs.highlightBlock(block);});});</script></body></html>