847 lines
45 KiB
HTML
847 lines
45 KiB
HTML
|
||
<!doctype html>
|
||
<html lang="en-US" class="no-js">
|
||
<head>
|
||
<!--
|
||
88
|
||
88
|
||
88
|
||
,adPPYba, 88,dPPYba, ,adPPYYba, 8b d8
|
||
a8P_____88 88P' "8a "" `Y8 `8b d8'
|
||
8PP""""""" 88 d8 ,adPPPPP88 `8b d8'
|
||
"8b, ,aa 88b, ,a8" 88, ,88 `8b,d8'
|
||
`"Ybbd8"' 8Y"Ybbd8"' `"8bbdP"Y8 Y88'
|
||
d8'
|
||
d8'
|
||
|
||
Kickstart your career @ https://careers.ebayinc.com
|
||
-->
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
<meta name="description" content="eBay’s notification platform team built a fault-tolerant, resilient system by injecting faults in the application level.">
|
||
|
||
<title>How eBay’s Notification Platform Used Fault Injection in New Ways</title>
|
||
|
||
<base href="https://innovation.ebayinc.com/" />
|
||
|
||
<link rel="prefetch" href="https://static.ebayinc.com/static/assets/Uploads/Blog/Posts/221101-Fault-Injection-tech-blog-v1-inc-16x9-hero.jpg" />
|
||
<meta name="DC.date" content="2022-12-15" scheme="DCTERMS.W3CDTF" />
|
||
<meta name="robots" content="max-image-preview:large">
|
||
<meta name="thumbnail" content="https://static.ebayinc.com/static/assets/Uploads/Blog/Posts/_resampled/ScaleWidthWzIwMF0/221101-Fault-Injection-tech-blog-v1-inc-16x9-hero.jpg?fs=a7a4d1345591e6c4" />
|
||
<meta name="twitter:card" content="summary_large_image" />
|
||
<meta name="twitter:description" content="eBay’s notification platform team built a fault-tolerant, resilient system by injecting faults in the application level." />
|
||
<meta name="twitter:image" content="https://static.ebayinc.com/static/assets/Uploads/Blog/Posts/_resampled/FillWzEwMjQsNTEyXQ/221101-Fault-Injection-tech-blog-v1-inc-16x9-hero.jpg?fs=b6f3420ba6505733" />
|
||
<meta name="twitter:site" content="@eBayNewsroom" />
|
||
<meta name="twitter:title" content="How eBay’s Notification Platform Used Fault Injection in New Ways" />
|
||
<meta property="article:modified_time" content="2025-03-18T14:49:39-07:00" />
|
||
<meta property="article:published_time" content="2022-12-15" />
|
||
<meta property="dateModified" content="2025-03-18T14:49:39-07:00" />
|
||
<meta property="datePublished" content="2022-12-15" />
|
||
<meta property="og:description" content="eBay’s notification platform team built a fault-tolerant, resilient system by injecting faults in the application level." />
|
||
<meta property="og:image" content="https://static.ebayinc.com/static/assets/Uploads/Blog/Posts/_resampled/ScaleWidthWzEwMjRd/221101-Fault-Injection-tech-blog-v1-inc-16x9-hero.jpg?fs=0dc949f19134e344" />
|
||
<meta property="og:locale" content="en_US" />
|
||
<meta property="og:site_name" content="eBay Inc." />
|
||
<meta property="og:title" content="How eBay’s Notification Platform Used Fault Injection in New Ways" />
|
||
<meta property="og:type" content="article" />
|
||
<meta property="og:updated_time" content="2025-03-18T14:49:39-07:00" />
|
||
<meta property="og:url" content="https://innovation.ebayinc.com/stories/how-ebays-notification-platform-used-fault-injection-in-new-ways/" />
|
||
<link rel="canonical" href="https://innovation.ebayinc.com/stories/how-ebays-notification-platform-used-fault-injection-in-new-ways/">
|
||
|
||
|
||
|
||
<link rel="preconnect" href="//static.ebayinc.com" crossorigin>
|
||
<link rel="dns-prefetch" href="//static.ebayinc.com">
|
||
|
||
<link rel="preconnect" href="//ajax.googleapis.com" crossorigin>
|
||
<link rel="dns-prefetch" href="//ajax.googleapis.com">
|
||
|
||
<link rel="preconnect" href="//www.googletagmanager.com" crossorigin>
|
||
<link rel="dns-prefetch" href="//www.googletagmanager.com">
|
||
|
||
<link rel="preconnect" href="//www.google-analytics.com" crossorigin>
|
||
<link rel="dns-prefetch" href="//www.google-analytics.com">
|
||
|
||
|
||
|
||
<link rel="preload" href="https://static.ebayinc.com/static/theme/css/ebi-header.css?v=752435c" as="style">
|
||
|
||
|
||
|
||
|
||
<link rel="preload" href="https://static.ebayinc.com/static/theme/fonts/build/MarketSans-Regular-WebS-688dde54.woff2" as="font" type="font/woff2" crossorigin>
|
||
|
||
<link rel="preload" href="https://static.ebayinc.com/static/theme/fonts/build/MarketSans-Bold-WebS-49f0420f.woff2" as="font" type="font/woff2" crossorigin>
|
||
|
||
<link rel="preload" href="https://static.ebayinc.com/static/theme/fonts/build/ebi-icons-8ce07361.woff2" as="font" type="font/woff2" crossorigin>
|
||
|
||
|
||
|
||
|
||
<link rel="alternate" type="application/rss+xml" title="eBay Newsroom" href="https://www.ebayinc.com/stories/news/rss/" />
|
||
|
||
<link rel="alternate" type="application/rss+xml" title="Innovation Stories" href="https://innovation.ebayinc.com/stories/rss/" />
|
||
|
||
|
||
|
||
<link rel="stylesheet" href="https://static.ebayinc.com/static/theme/css/global.css?v=752435c" type="text/css">
|
||
<link rel="stylesheet" href="https://static.ebayinc.com/static/theme/css/main.css?v=752435c" type="text/css">
|
||
<link rel="stylesheet" href="https://static.ebayinc.com/static/theme/css/themes.css?v=752435c" type="text/css">
|
||
<script type="text/javascript" src="https://static.ebayinc.com/static/theme/javascript/build/head.js?v=752435c"></script>
|
||
|
||
|
||
|
||
|
||
|
||
<script src="https://static.ebayinc.com/static/theme/javascript/build/components.js?v=752435c"></script>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
|
||
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
|
||
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
|
||
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
|
||
})(window,document,'script','dataLayer', "GTM-N57M22R");</script>
|
||
|
||
|
||
<link rel="stylesheet" type="text/css" href="https://static.ebayinc.com/static/theme/css/special/page-nav.css?v=752435c" />
|
||
</head>
|
||
|
||
<body class="plv1 ">
|
||
|
||
|
||
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id="GTM-N57M22R"" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||
|
||
|
||
|
||
|
||
<a id="skip-to-main" class="aria-only" href="/stories/how-ebays-notification-platform-used-fault-injection-in-new-ways/#main-content">Skip to main content</a>
|
||
|
||
|
||
|
||
<ebi-header no-hl>
|
||
<template shadowrootmode="open">
|
||
<link rel="stylesheet" href="https://static.ebayinc.com/static/theme/css/ebi-header.css?v=752435c">
|
||
<header class="main-header js-header">
|
||
|
||
|
||
<sticky-scroll when="scroll-up">
|
||
<div class="main-header-top js-header-top clear ">
|
||
|
||
<a href="https://www.ebayinc.com/" class="header-logo mobile js-header-home-url">
|
||
<img class="js-header-logo" src="https://static.ebayinc.com/static/theme/images/logo.svg?v=752435c" alt="Home" title="eBay Inc. Home Page">
|
||
</a>
|
||
<button class="hamburger js-hamburger hamburger--3dx mobile" type="button" aria-label="Menu" aria-controls="main-nav-menu">
|
||
<span class="hamburger-box">
|
||
<span class="hamburger-inner"></span>
|
||
</span>
|
||
</button>
|
||
<div class="main-header-page-subtitle mobile">Company Information</div>
|
||
|
||
<div class="inner">
|
||
|
||
<div class="main-nav-overlay js-nav-overlay"></div>
|
||
|
||
|
||
<div class="main-nav-menu js-nav-menu instant" id="main-nav-menu">
|
||
|
||
|
||
<nav class="header-nav js-header-nav" aria-label="Site" role="navigation">
|
||
<ul class="header-nav-top">
|
||
|
||
<li class="desktop">
|
||
<a href="https://www.ebayinc.com/" class="header-logo">
|
||
<img src="https://static.ebayinc.com/static/theme/images/logo.svg?v=752435c" alt="Home" title="eBay Inc. Home Page">
|
||
</a>
|
||
</li>
|
||
|
||
|
||
<li class="l1 js-l1 link first has-children js-has-children" data-id="6">
|
||
<a class="js-l1-link" href="https://www.ebayinc.com/company/" aria-controls="subnav-company" >Company</a>
|
||
|
||
|
||
|
||
<ul class="js-mobile-subnav" id="subnav-company">
|
||
|
||
<li class="l2 js-l2 link " data-id="152">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/company/">Overview</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="149">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/company/our-leaders/">Our Leaders</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="151">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/company/our-history/">Our History</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="8817">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/company/our-purpose-in-action/">Our Purpose in Action</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="5467">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/impact/sustainable-commerce/">Sustainable Commerce</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="5523">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/company/be-for-everyone/">Be for Everyone</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="526">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/company/government-relations/">Government Relations</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="5608">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/company/privacy-center/">Privacy Center</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="8766">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/impact/policies-downloads/">Reports & Policies</a>
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
|
||
</li>
|
||
|
||
<li class="l1 js-l1 link has-children js-has-children" data-id="523">
|
||
<a class="js-l1-link" href="https://www.ebayinc.com/stories/news/" aria-controls="subnav-stories" >Stories</a>
|
||
|
||
|
||
|
||
<ul class="js-mobile-subnav" id="subnav-stories">
|
||
|
||
<li class="l2 js-l2 link " data-id="6021">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/stories/news/">Our News</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="3208">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/stories/press-room/">Press Room</a>
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
|
||
</li>
|
||
|
||
<li class="l1 js-l1 link has-children js-has-children" data-id="17">
|
||
<a class="js-l1-link" href="https://www.ebayinc.com/impact/" aria-controls="subnav-impact" >Impact</a>
|
||
|
||
|
||
|
||
<ul class="js-mobile-subnav" id="subnav-impact">
|
||
|
||
<li class="l2 js-l2 link " data-id="160">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/impact/">Overview</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="8027">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/impact/economic-opportunity/">Economic Opportunity</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="7450">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/impact/sustainable-commerce/">Sustainable Commerce</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="8555">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/impact/culture-and-workforce/">Culture & Workforce</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="7452">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/impact/trusted-marketplace/">Trusted Marketplace</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="7453">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/impact/goals-progress/">Goals & Progress</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="156">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/impact/ebay-for-charity/">eBay for Charity</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="155">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/impact/ebay-foundation/">eBay Foundation</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="6689">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/impact/policies-downloads/">Policies & Downloads</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="4468">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/impact/seller-showcase/">Seller Showcase</a>
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
|
||
</li>
|
||
|
||
<li class="l1 js-l1 current " data-id="9062">
|
||
<a class="js-l1-link" href="https://innovation.ebayinc.com/stories/" aria-current="page">Innovation</a>
|
||
|
||
|
||
|
||
|
||
</li>
|
||
|
||
<li class="l1 js-l1 link has-children js-has-children" data-id="16">
|
||
<a class="js-l1-link" href="https://investors.ebayinc.com/" aria-controls="subnav-investors" >Investors</a>
|
||
|
||
|
||
|
||
<ul class="js-mobile-subnav" id="subnav-investors">
|
||
|
||
<li class="l2 js-l2 link " data-id="582">
|
||
<a class="js-l2-link" href="https://investors.ebayinc.com/overview/default.aspx">Overview</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="583">
|
||
<a class="js-l2-link" href="https://investors.ebayinc.com/fast-facts/default.aspx">Fast Facts</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="584">
|
||
<a class="js-l2-link" href="https://investors.ebayinc.com/stock-information/Historical-Stock-Information/default.aspx">Stock Information</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="585">
|
||
<a class="js-l2-link" href="https://investors.ebayinc.com/investor-news/default.aspx">Investor News</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="586">
|
||
<a class="js-l2-link" href="https://investors.ebayinc.com/events/default.aspx">Events & Presentations</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="587">
|
||
<a class="js-l2-link" href="https://investors.ebayinc.com/financial-information/financial-summary/default.aspx">Financial Information</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="588">
|
||
<a class="js-l2-link" href="https://investors.ebayinc.com/corporate-governance/governance-documents/default.aspx">Corporate Governance</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="589">
|
||
<a class="js-l2-link" href="https://investors.ebayinc.com/esg-investors/default.aspx">ESG Investors</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="591">
|
||
<a class="js-l2-link" href="https://investors.ebayinc.com/contact-us-and-resources/faq/default.aspx">Resources & Contact</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="8564">
|
||
<a class="js-l2-link" href="https://www.ebayinc.com/investors/investor-day/">Investor Day</a>
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
|
||
</li>
|
||
|
||
<li class="l1 js-l1 link has-children js-has-children" data-id="9997">
|
||
<a class="js-l1-link" href="https://careers.ebayinc.com/" aria-controls="subnav-careers" >Careers</a>
|
||
|
||
|
||
|
||
<ul class="js-mobile-subnav" id="subnav-careers">
|
||
|
||
<li class="l2 js-l2 link " data-id="9998">
|
||
<a class="js-l2-link" href="https://careers.ebayinc.com/">Overview</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="9999">
|
||
<a class="js-l2-link" href="https://careers.ebayinc.com/how-we-hire/">How We Hire</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="10000">
|
||
<a class="js-l2-link" href="https://careers.ebayinc.com/jobs-by-location/">Jobs by Location</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="10001">
|
||
<a class="js-l2-link" href="https://careers.ebayinc.com/how-we-work/">How We Work</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="10002">
|
||
<a class="js-l2-link" href="https://careers.ebayinc.com/jobs-by-category/">Jobs by Category</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="10003">
|
||
<a class="js-l2-link" href="https://careers.ebayinc.com/our-culture/">Our Culture</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="10004">
|
||
<a class="js-l2-link" href="https://careers.ebayinc.com/our-benefits/">Our Benefits</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="10005">
|
||
<a class="js-l2-link" href="https://careers.ebayinc.com/student-opportunities/">Student Opportunities</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="10006">
|
||
<a class="js-l2-link" href="https://jobs.ebayinc.com/us/en/recommended-jobs">Recommended Jobs</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="10009">
|
||
<a class="js-l2-link" href="https://careers.ebayinc.com/talent-privacy-notice/">Talent Privacy Notice</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="10007">
|
||
<a class="js-l2-link" href="https://jobs.ebayinc.com/us/en/jobcart">Saved Jobs</a>
|
||
</li>
|
||
|
||
<li class="l2 js-l2 link " data-id="10008">
|
||
<a class="js-l2-link" href="https://jobs.ebayinc.com/us/en/search-results">Search Jobs</a>
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
|
||
|
||
<div class="secondary-navigation js-secondary-navigation mobile" role="list" aria-label="Quick Links">
|
||
|
||
<span role="listitem" class="js-secondary footer-only"><a class="js-secondary-link" href="https://www.ebay.com" data-id="2454">eBay.com</a></span>
|
||
|
||
<span role="listitem" class="js-secondary "><a class="js-secondary-link" href="https://www.ebayinc.com/stories/press-room/" data-id="142">Press Room</a></span>
|
||
|
||
<span role="listitem" class="js-secondary "><a class="js-secondary-link" href="https://www.ebayinc.com/company/follow-us/" data-id="144">Follow Us</a></span>
|
||
|
||
<span role="listitem" class="js-secondary "><a class="js-secondary-link" href="https://www.ebayinc.com/company/contact-us/" data-id="143">Contact Us</a></span>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<form action="https://www.ebayinc.com/search/" method="get" class="main-search-mobile mobile js-site-search-form round-corners" role="search" aria-label="Search Site">
|
||
<input type="text" name="query" class="js-site-search-input" placeholder="Search eBayInc.com" maxlength="128" />
|
||
<button type="submit" class="js-site-search-submit">Search Website</button>
|
||
</form>
|
||
|
||
|
||
|
||
<div class="secondary-navigation js-secondary-navigation desktop" role="list" aria-label="Quick Links">
|
||
|
||
<span role="listitem" class="js-secondary footer-only"><a class="js-secondary-link" href="https://www.ebay.com" data-id="2454">eBay.com</a></span>
|
||
|
||
<span role="listitem" class="js-secondary "><a class="js-secondary-link" href="https://www.ebayinc.com/stories/press-room/" data-id="142">Press Room</a></span>
|
||
|
||
<span role="listitem" class="js-secondary "><a class="js-secondary-link" href="https://www.ebayinc.com/company/follow-us/" data-id="144">Follow Us</a></span>
|
||
|
||
<span role="listitem" class="js-secondary "><a class="js-secondary-link" href="https://www.ebayinc.com/company/contact-us/" data-id="143">Contact Us</a></span>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<form action="https://www.ebayinc.com/search/" method="get" class="main-search-desktop desktop js-site-search-form js-site-search-desktop" role="search" aria-label="Search Site">
|
||
<input type="text" name="query" class="js-site-search-input" placeholder="Search eBayInc.com" maxlength="128" aria-label="Type your search now and press enter or press escape to close." />
|
||
<button type="submit" class="js-site-search-submit">Search Website</button>
|
||
</form>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
</sticky-scroll>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div style="display: none;">
|
||
|
||
<a class="js-tertiary-link" href="https://www.ebayinc.com/terms-of-use/" data-id="146">Terms of Use</a>
|
||
|
||
<a class="js-tertiary-link" href="https://www.ebayinc.com/company/privacy-center/" data-id="147">Privacy</a>
|
||
|
||
<a class="js-tertiary-link" href="https://www.ebayinc.com/accessibility/" data-id="2796">Accessibility</a>
|
||
|
||
<a class="js-tertiary-link" href="https://www.ebayinc.com/stories/press-room/" data-id="9192">Settlements</a>
|
||
|
||
</div>
|
||
|
||
</header>
|
||
</template>
|
||
</ebi-header>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<main id="main-content" class="main-content">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="main-content__inner main-content__inner--wide main-content__inner--no-margin">
|
||
|
||
|
||
<div class="stories-article blog-post content-width content-width--desktop">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<article class="main-col ">
|
||
|
||
<div class="content">
|
||
<div class="intro">
|
||
<div class="intro-wrapper">
|
||
|
||
|
||
|
||
<div class="blog-primary-attrib">
|
||
<a href="https://innovation.ebayinc.com/stories/archive/?tag=testing" title="More articles in Testing">Testing</a>
|
||
</div>
|
||
|
||
|
||
<h1>How eBay’s Notification Platform Used Fault Injection in New Ways</h1>
|
||
|
||
|
||
|
||
|
||
|
||
<div class="top-bar">
|
||
<div class="date-authors">
|
||
<time datetime="2022-12-15" class="article-date" aria-label="December 15, 2022" role="contentinfo"><span aria-hidden="true">Dec 15, 2022</span></time>
|
||
<span class="divider"></span>
|
||
<div class="authors">By: Wei Chen</div>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
<share-icons title="How eBay’s Notification Platform Used Fault Injection in New Ways" link="https://ebayinc.to/4kyLmNL" twitter="https://ebayinc.to/4hKBv4G" linkedin="https://ebayinc.to/3FNUNJ3" facebook="https://ebayinc.to/3DSCQsl"></share-icons>
|
||
|
||
</div>
|
||
|
||
|
||
<h4 class="blurb">eBay’s notification platform team built a fault-tolerant, resilient system by injecting faults in the application level.</h4>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<figure class="intro-media">
|
||
|
||
|
||
|
||
<picture class="re__pic "><source srcset="https://static.ebayinc.com/static/assets/Uploads/Blog/Posts/_resampled/FillWzg2Miw0ODVd/221101-Fault-Injection-tech-blog-v1-inc-16x9-hero.jpg?fs=bf717c649a449c79" media="(max-width: 480px) and (min-resolution: 144dpi), (max-width: 480px) and (-webkit-min-device-pixel-ratio: 1.5)"><source srcset="https://static.ebayinc.com/static/assets/Uploads/Blog/Posts/_resampled/FillWzE2MDAsOTAwXQ/221101-Fault-Injection-tech-blog-v1-inc-16x9-hero.jpg?fs=f2b2e061da01391a" media="(min-width: 1080px) and (min-resolution: 144dpi), (min-width: 1080px) and (-webkit-min-device-pixel-ratio: 1.5)"><source srcset="https://static.ebayinc.com/static/assets/Uploads/Blog/Posts/_resampled/FillWzE1MDAsODQ0XQ/221101-Fault-Injection-tech-blog-v1-inc-16x9-hero.jpg?fs=746dd0b3a4663298" media="(min-width: 768px) and (min-resolution: 144dpi), (min-width: 768px) and (-webkit-min-device-pixel-ratio: 1.5)"><source srcset="https://static.ebayinc.com/static/assets/Uploads/Blog/Posts/_resampled/FillWzE0MzYsODA4XQ/221101-Fault-Injection-tech-blog-v1-inc-16x9-hero.jpg?fs=f009459dac15467d" media="(min-width: 481px) and (min-resolution: 144dpi), (min-width: 481px) and (-webkit-min-device-pixel-ratio: 1.5)"><source srcset="https://static.ebayinc.com/static/assets/Uploads/Blog/Posts/_resampled/FillWzQzMCwyNDJd/221101-Fault-Injection-tech-blog-v1-inc-16x9-hero.jpg?fs=2eb29c6541633149" media="(max-width: 480px)"><source srcset="https://static.ebayinc.com/static/assets/Uploads/Blog/Posts/_resampled/FillWzgwMCw0NTBd/221101-Fault-Injection-tech-blog-v1-inc-16x9-hero.jpg?fs=200b57277fa6db14" media="(min-width: 1080px)"><source srcset="https://static.ebayinc.com/static/assets/Uploads/Blog/Posts/_resampled/FillWzc1MCw0MjJd/221101-Fault-Injection-tech-blog-v1-inc-16x9-hero.jpg?fs=2d0732922ce752ca" media="(min-width: 768px)"><source srcset="https://static.ebayinc.com/static/assets/Uploads/Blog/Posts/_resampled/FillWzcxOCw0MDRd/221101-Fault-Injection-tech-blog-v1-inc-16x9-hero.jpg?fs=72edd937ee2ccf8c" media="(min-width: 481px)"><img class="re__img " src="https://static.ebayinc.com/static/assets/Uploads/Blog/Posts/_resampled/FillWzgwMCw0NTBd/221101-Fault-Injection-tech-blog-v1-inc-16x9-hero.jpg?fs=200b57277fa6db14" alt="" height="450" width="800"></picture>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</figure>
|
||
|
||
|
||
|
||
<div class="content-container">
|
||
|
||
<div class="cms-content clear story-article-content no-gallery">
|
||
|
||
<h3 dir="ltr">Background</h3><p dir="ltr">It might sound paradoxical to deliberately break something we’re trying to fix, but sometimes, that’s the most efficient method to do it. Fault injection is the process by which we deliberately introduce faults into the system. We can observe the system behavior with the injected faults to identify the weakness of the system. Within the industry, fault injection is a common practice as a means to build fault-tolerant, resilient systems. </p><p dir="ltr">The eBay notification platform team practiced the idea of fault injection from a different perspective in eBay’s notification platform. This platform is responsible for pushing platform notifications to third party applications to provide the latest changes in item price, item stock status, payment status and more. It is a highly distributed and large-scale system relying on many external dependencies, including distributed store, message queue, push notification endpoints and others. Any faults in these dependent services will directly impact the stability of the system, so it’s quite valuable to run experiments in the system containing the failures of these dependencies, in order to understand the behaviors and mitigate any weaknesses. </p><p dir="ltr">To achieve this, the faults are injected by simulation in the application level by the code instrumentation. As far as we know, we are the very first in the industry to practice this idea officially and widely to experiment on the mission-critical system with different kinds of deliberately designed faults.</p><p dir="ltr"> </p><h3 dir="ltr">Application Level vs Infrastructure Level</h3><p dir="ltr">There is no unified way to inject these faults. The most straightforward and ubiquitous way to do so in the industry is to create real faults directly. For example, to introduce the <em>http</em> disconnection or timeout error, one option is to turn off the network or shut down the downstream services temporarily; to introduce the disk full error, one option is to create a bunch of files in the file system. We can think of this as fault injection at the infrastructure level, and if we think about it from this perspective, it will inevitably drive us to create the concrete faults for the infrastructure resources. Creating faults will directly harm the infrastructure resources; turning off the network obviously causes many issues, in the previous example. If the resource is shared, this will introduce extra impacts and risks to other services depending on it. If the resource is dedicated, it will increase the cost.</p><p dir="ltr">But there is a different way to approach this problem: Instead of creating faults at the infrastructure level, what if we created them at the application level? This would allow us to simulate the faults we’d like to use with the application API leveraged to talk with the infrastructure resources. For example, to inject the <em>http</em> timeout fault, we add the latency in the <em>http</em> client library; to simulate the internal service error, we simulate the response code with 500 <em>http</em> status code. The faults are restricted to the API level and do no harm to the underlying infrastructure resources. In this way, we’ve found an affordable, secure and reusable way to do fault injection.</p><p dir="ltr">As our service is a Java-based application, we’ve provided a Java agent. Within it, we instrumented the class files of the client libraries for the dependent services to introduce different kinds of faults we defined. The introduced faults are raised when our service communicates with the underlying resource through the instrumented API. The faults do not really happen in our dependent services, owing to the changed codes, but the effect is simulated, enabling us to experiment without risk.</p><p dir="ltr"><a class="js-story-modal-image story-modal-image" href="assets/Uploads/Editor/221101-Fault-Injection-tech-blog-v1-inc-1600x-image-1.jpg"><img class="leftAlone" title="" src="assets/Uploads/Editor/_resampled/ResizedImageWzEyMDAsNjc2XQ/221101-Fault-Injection-tech-blog-v1-inc-1600x-image-1.jpg" alt="221101 Fault Injection tech blog v1 inc 1600x image 1" width="600" height="338"></a></p><h3 dir="ltr">Instrumentation</h3><p dir="ltr">To simulate the faults for the client libraries by instrumentation is challenging. Our main task is to force the invoked methods to experience failures. One method to do this is to inject failure directly into the method, by, for example, throwing the exception in the method body. The other method calls for changing the value or the state of the input parameters to drive the method to go to the failure execution path. There are three instrumentation patterns in our project.</p><p dir="ltr"><strong>1. Block or interrupt the method logic</strong></p><p dir="ltr">This type of instrumentation is straightforward in that the API can throw exceptions or sleep for a specific period of time to simulate the error or timeout.</p><p><a class="js-story-modal-image story-modal-image" href="assets/Uploads/Editor/221101-Fault-Injection-tech-blog-v1-inc-1600x-image-2.jpg"><img class="leftAlone" title="" src="assets/Uploads/Editor/_resampled/ResizedImageWzEyMDAsMjc4XQ/221101-Fault-Injection-tech-blog-v1-inc-1600x-image-2.jpg" alt="221101 Fault Injection tech blog v1 inc 1600x image 2" width="600" height="139"></a></p><p dir="ltr"> </p><p dir="ltr"><strong>2. Change the state of method parameters </strong></p><p dir="ltr">Under some circumstances, the simulation of faults will depend on the specific state of the input parameters. For example, the below method logic is depending on the return value of the <em>response.getStatusCode()</em>. If the value does not equal 200, the failure logic will be triggered. So if we want to simulate the faults with failure code, then we need to find a way to change the state of response which will be returned from the <em>response.getStatusCode()</em>. </p><p dir="ltr">The way we achieved this is to add the instrumented code snippet to throw a specific defined exception and let the exception carry the response code we need to simulate. Meanwhile, we instrument the method by adding the try-catch block to the method to specifically catch the exception we throw and return the code in the catch block. By doing so, we’ve changed the method execution path to return the designated value.</p><p dir="ltr"><a class="js-story-modal-image story-modal-image" href="assets/Uploads/Editor/221101-Fault-Injection-tech-blog-v1-inc-1600x-image-3.jpg"><img class="leftAlone" title="" src="assets/Uploads/Editor/_resampled/ResizedImageWzEyMDAsNTk2XQ/221101-Fault-Injection-tech-blog-v1-inc-1600x-image-3.jpg" alt="221101 Fault Injection tech blog v1 inc 1600x image 3" width="600" height="298"></a></p><p dir="ltr"> </p><p dir="ltr"><strong>3. Replace the value of method parameters</strong></p><p dir="ltr"><span id="docs-internal-guid-e099d06f-7fff-5caa-b4ce-4dc2f65ee59f">In contrast to the above example, sometimes the method logic will depend on the value of the parameters. So if you want to simulate a fault, then you need to change the value of the input parameter. To change the value of the parameter, we need to know the name of the parameter first and inject the code to replace the value for the parameter with its name. This is not easy because the parameter name can only be known in the runtime. So we leverage the Java reflection to get the names of the parameters in the runtime.</span></p><p dir="ltr"><a class="js-story-modal-image story-modal-image" href="assets/Uploads/Editor/221101-Fault-Injection-tech-blog-v1-inc-1600x-image-4.jpg"><img class="leftAlone" title="" src="assets/Uploads/Editor/_resampled/ResizedImageWzEyMDAsMzY4XQ/221101-Fault-Injection-tech-blog-v1-inc-1600x-image-4.jpg" alt="221101 Fault Injection tech blog v1 inc 1600x image 4" width="600" height="184"></a><span id="docs-internal-guid-373e3361-7fff-e507-7064-f27c5d54d398">To implement the above three types of instrumentation, we have created a Java agent. In the agent, we have implemented a classloader which will instrument the code of the methods leveraged in the application code. We also created an annotation to indicate which method will be instrumented and put the instrumentation logic in the methods annotated. Here’s an example:</span></p><p dir="ltr"><a class="js-story-modal-image story-modal-image" href="assets/Uploads/Editor/221101-Fault-Injection-tech-blog-v1-inc-1600x-image-5.jpg"><img class="leftAlone" title="" src="assets/Uploads/Editor/_resampled/ResizedImageWzEyMDAsNDUwXQ/221101-Fault-Injection-tech-blog-v1-inc-1600x-image-5.jpg" alt="221101 Fault Injection tech blog v1 inc 1600x image 5" width="600" height="225"></a><span id="docs-internal-guid-71cbbbac-7fff-6513-cf3a-9e2d9ac9467b">In the above code snippet, we’d like to provide the instrumentation logic for <em>org.asynchttpclient.providers.netty.future.NettyResponseFuture.done()</em>. So what we do is to create a new method with the same signature, and make it be annotated by @Enforce, which is the user-defined Java annotation used to indicate the instrumentation logic for fault injection. The annotation has two fields: value and type. The value field is the class name of the method we want to instrument. (We’ll discuss the type field shortly.) When the agent is loaded, the defined class loader will find all the methods annotated by @Enforce and inject the instrumentation logic defined in the methods to the methods to be instrumented. The type field of @Enforce has the two values runtime(default value) and static. In the above example, we implement the instrumentation logic with the Java code. But there remains a chance that we might need to provide the instrumentation logic with a string literal. Here’s an example of that:</span></p><p dir="ltr"><a class="js-story-modal-image story-modal-image" href="assets/Uploads/Editor/221101-Fault-Injection-tech-blog-v1-inc-1600x-image-6.jpg"><img class="leftAlone" title="" src="assets/Uploads/Editor/_resampled/ResizedImageWzEyMDAsNDY0XQ/221101-Fault-Injection-tech-blog-v1-inc-1600x-image-6.jpg" alt="221101 Fault Injection tech blog v1 inc 1600x image 6" width="600" height="232"></a></p><h3 dir="ltr">Customized Class Loader</h3><p dir="ltr">Now we have implemented the instrumentation logic in the Java agent. However, we still need to create a customized class loader to inject the instrumentation logic in the target methods of the client libraries into which we want to inject faults. The class loader leverages Javassist, the instrument library, which can manipulate the Java bytecode to transform the class files of the target methods to include the defined faults.</p><p dir="ltr"><a class="js-story-modal-image story-modal-image" href="assets/Uploads/Editor/221101-Fault-Injection-tech-blog-v1-inc-1600x-image-8.jpg"><img class="leftAlone" title="" src="assets/Uploads/Editor/_resampled/ResizedImageWzEyMDAsNzYyXQ/221101-Fault-Injection-tech-blog-v1-inc-1600x-image-8.jpg" alt="221101 Fault Injection tech blog v1 inc 1600x image 8" width="600" height="381"></a></p><p dir="ltr">With the above implementation described, we have injected the faults by instrumentation for the below client libraries of the three resources we are depending on.</p><ul><li>
|
||
<p dir="ltr">Push Notification Endpoints: </p>
|
||
</li>
|
||
<ul><li>
|
||
<p dir="ltr">Client lib: async-http-client 1.8.3</p>
|
||
</li>
|
||
<li>
|
||
<p dir="ltr">Fault types:</p>
|
||
</li>
|
||
<ul><li>
|
||
<p dir="ltr">Timeout</p>
|
||
</li>
|
||
<li>
|
||
<p dir="ltr">Exception</p>
|
||
</li>
|
||
<li>
|
||
<p dir="ltr">Response status code</p>
|
||
</li>
|
||
</ul></ul><li>
|
||
<p dir="ltr">Message Queue:</p>
|
||
</li>
|
||
<ul><li>
|
||
<p dir="ltr">Client lib: kafka-client 2.5.1</p>
|
||
</li>
|
||
<li>
|
||
<p dir="ltr">Fault types:</p>
|
||
</li>
|
||
<ul><li>
|
||
<p dir="ltr">Timeout</p>
|
||
</li>
|
||
<li>
|
||
<p dir="ltr">Exception</p>
|
||
</li>
|
||
</ul></ul><li>
|
||
<p dir="ltr">Distributed Store (built in-house by eBay): </p>
|
||
</li>
|
||
<ul><li>
|
||
<p dir="ltr">Client lib: monster-java-client 3.4.4.2-RELEASE</p>
|
||
</li>
|
||
<li>
|
||
<p dir="ltr">Fault types:</p>
|
||
</li>
|
||
<ul><li>
|
||
<p dir="ltr">Timeout</p>
|
||
</li>
|
||
<li>
|
||
<p dir="ltr">Exception</p>
|
||
</li>
|
||
</ul></ul></ul><h3 dir="ltr">Configuration Management</h3><p><span id="docs-internal-guid-cdfcf5fd-7fff-fb09-2c59-e57f42f49f32">To dynamically change the configuration for the fault injections in the runtime, we have implemented a configure management console in the Java agent. As our service is a web application, we can instrument the<em> javax.servlet.http.HttpServlet.service(HttpServletRequest, HttpServletResponse)</em> to expose the endpoints for the configure management. The endpoint will render a configuration page to let developers configure the attributes of the fault injection in the runtime. For example, a developer could globally enable or disable the fault injection and other subtypes of the faults; for example, a timeout for <em>AyncHttpClient</em>.</span></p><p><a class="js-story-modal-image story-modal-image" href="assets/Uploads/Editor/221101-Fault-Injection-tech-blog-v1-inc-1600x-image-9.jpg"><img class="leftAlone" title="" src="assets/Uploads/Editor/_resampled/ResizedImageWzEyMDAsNzM2XQ/221101-Fault-Injection-tech-blog-v1-inc-1600x-image-9.jpg" alt="221101 Fault Injection tech blog v1 inc 1600x image 9" width="600" height="368"></a></p><h3 dir="ltr">What’s Next</h3><p dir="ltr">We will expand the scope of the application level fault injection in more client libraries and fault categories to diversify the scenarios of experiments for our services under different kinds of fault circumstances. Meanwhile, as the configuration of the faults setting through the configuration management console can only be triggered at the instance level, we will find a way to broadcast the changes across the cluster.</p>
|
||
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="bottom">
|
||
|
||
|
||
<share-icons title="How eBay’s Notification Platform Used Fault Injection in New Ways" link="https://ebayinc.to/4kyLmNL" twitter="https://ebayinc.to/4hKBv4G" linkedin="https://ebayinc.to/3FNUNJ3" facebook="https://ebayinc.to/3DSCQsl"></share-icons>
|
||
|
||
|
||
<p class="article-topics-holder" tabindex="0">
|
||
|
||
|
||
<span class="tiny-title">Tag:</span>
|
||
<a href="/stories/archive/?tag=testing" title="More articles in Testing">Testing</a>
|
||
|
||
</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<nav class="page-nav">
|
||
|
||
<a class="prev" href="/stories/why-and-how-ebay-pivoted-to-opentelemetry/"><span class="hover">Previous Article<span class="colon">:</span></span> <span class="title">Why and How eBay Pivoted to OpenTelemetry</span></a>
|
||
|
||
|
||
|
||
<a class="next" href="/stories/how-ebay-created-a-language-model-with-three-billion-item-titles/"><span class="hover">Next Article<span class="colon">:</span></span> <span class="title">How eBay Created a Language Model With Three Billion Item Titles</span></a>
|
||
|
||
</nav>
|
||
|
||
|
||
|
||
</div>
|
||
|
||
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
</article>
|
||
|
||
|
||
<aside class="side-col blog-sidebar">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<section class="blog-related-module blog-side-col-module white-bg">
|
||
<h3 class="stories-side-col-heading">Related Articles</h3>
|
||
|
||
<a class="block-level-link" href="/stories/increase-a-b-testing-power-by-combining-experiments/">
|
||
<div class="title">Increase A/B Testing Power by Combining Experiments</div>
|
||
<div class="sub clear"><time datetime="2022-10-24" class="article-date" aria-label="October 24, 2022" role="contentinfo"><span aria-hidden="true">Oct 24, 2022</span></time> </div>
|
||
</a>
|
||
|
||
<a class="block-level-link" href="/stories/openfeature-with-contributions-from-ebay-submitted-to-cncfs-sandbox-program/">
|
||
<div class="title">OpenFeature, With Contributions From eBay, Submitted to CNCF's Sandbox Program</div>
|
||
<div class="sub clear"><time datetime="2022-05-19" class="article-date" aria-label="May 19, 2022" role="contentinfo"><span aria-hidden="true">May 19, 2022</span></time> </div>
|
||
</a>
|
||
|
||
<a class="block-level-link" href="/stories/creating-high-quality-staging-data-with-a-nosql-data-migration-system/">
|
||
<div class="title">Creating High-Quality Staging Data with a NoSQL Data Migration System</div>
|
||
<div class="sub clear"><time datetime="2022-02-07" class="article-date" aria-label="February 7, 2022" role="contentinfo"><span aria-hidden="true">Feb 7, 2022</span></time> </div>
|
||
</a>
|
||
|
||
<a class="block-level-link" href="/stories/the-staging-dichotomy-part-two/">
|
||
<div class="title">The Staging Dichotomy: Part Two</div>
|
||
<div class="sub clear"><time datetime="2021-12-08" class="article-date" aria-label="December 8, 2021" role="contentinfo"><span aria-hidden="true">Dec 8, 2021</span></time> </div>
|
||
</a>
|
||
|
||
<a class="block-level-link" href="/stories/the-staging-dichotomy-part-one/">
|
||
<div class="title">The Staging Dichotomy: Part One</div>
|
||
<div class="sub clear"><time datetime="2021-12-06" class="article-date" aria-label="December 6, 2021" role="contentinfo"><span aria-hidden="true">Dec 6, 2021</span></time> </div>
|
||
</a>
|
||
|
||
</section>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</aside>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
</main>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<ebi-footer></ebi-footer>
|
||
|
||
|
||
|
||
|
||
<a class="back-to-top aria-only" href="/stories/how-ebays-notification-platform-used-fault-injection-in-new-ways/#skip-to-main">Back to Top</a>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script src="https://beep.ebayinc.to/beep.js" async></script>
|
||
|
||
|
||
|
||
|
||
<script>window.dataLayer = window.dataLayer || []; window.dataLayer.push({"event":"story_article","attributes":{"publish_date":"2022-12-15","story_topic":"","story_tag":"Testing"}});</script>
|
||
|
||
|
||
|
||
|
||
|
||
<script>
|
||
(function() {
|
||
var imgEl = new Image();
|
||
imgEl.src = '//' + location.host + location.pathname.replace(/\/+$/, '') + '/ebi.gif';
|
||
imgEl.style = 'position: absolute; top: 0';
|
||
imgEl.alt = '';
|
||
document.querySelector('body').appendChild(imgEl);
|
||
})();
|
||
</script>
|
||
|
||
|
||
|
||
|
||
<script>
|
||
var _plsubtInp = {
|
||
'eventFamily': 'EBAYINC',
|
||
'samplingRate': 100,
|
||
'pageLoadTime': new Date().getTime(),
|
||
'pageId': 4685387,
|
||
'disableImp': false
|
||
};
|
||
var _plsUBTTQ = [];
|
||
</script>
|
||
<script type="text/javascript" defer src="https://ir.ebaystatic.com/cr/v/c01/250701JZ48C6XWVCKCN2.js"></script>
|
||
<script>
|
||
window.addEventListener('load', () => {
|
||
_plsUBTTQ.push(['customEvts', 'EBAYINC', 'CLIENT_PAGE_VIEW', { 'imp': '4685387' }]);
|
||
});
|
||
</script>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script type="text/javascript">
|
||
_linkedin_partner_id = "3677196";
|
||
window._linkedin_data_partner_ids = window._linkedin_data_partner_ids || [];
|
||
window._linkedin_data_partner_ids.push(_linkedin_partner_id);
|
||
</script><script type="text/javascript">
|
||
(function(){var s = document.getElementsByTagName("script")[0];
|
||
var b = document.createElement("script");
|
||
b.type = "text/javascript";b.async = true;
|
||
b.src = "https://snap.licdn.com/li.lms-analytics/insight.min.js";
|
||
s.parentNode.insertBefore(b, s);})();
|
||
</script>
|
||
<noscript>
|
||
<img height="1" width="1" style="display:none;" alt="" src="https://px.ads.linkedin.com/collect/?pid=3677196&fmt=gif" />
|
||
</noscript>
|
||
|
||
|
||
|
||
|
||
<script>
|
||
!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
|
||
},s.version='1.1',s.queue=[],u=t.createElement(n),u.async=!0,u.src='//static.ads-twitter.com/uwt.js',
|
||
a=t.getElementsByTagName(n)[0],a.parentNode.insertBefore(u,a))}(window,document,'script');
|
||
|
||
|
||
|
||
|
||
|
||
twq('init','o2r23');
|
||
twq('track','PageView');
|
||
|
||
|
||
twq('init','o7ejw');
|
||
twq('track','PageView');
|
||
</script>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script type="text/javascript" src="https://static.ebayinc.com/static/theme/javascript/build/jquery.js?v=752435c"></script><script type="text/javascript" src="https://static.ebayinc.com/static/theme/javascript/build/foot.js?v=752435c"></script><script type="text/javascript" src="https://static.ebayinc.com/static/theme/javascript/foot/modules/special/article-image-modal.js?v=752435c"></script><script type="text/javascript" src="/xkv61/LGG/pIgd/G5QS/aNtZ/rt9bV25m/TARAAQE/XytqQGgq/Mg4c?v=89948ed2-aa13-69a9-761c-84d333de173b" defer></script></body>
|
||
</html>
|