Files
nexus/sreweekly/articles/191/04-behind-the-scenes-of-a-long-eve-online-downtime-2015.html
2026-09-12 17:23:01 +08:00

169 lines
103 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!doctype html><html lang="en" itemscope><head><script>dataLayer=[]</script><script>!function(e,t,a,n,c,g,o){e.GoogleAnalyticsObject=c,e.ga=e.ga||function(){(e.ga.q=e.ga.q||[]).push(arguments)},e.ga.l=1*new Date,g=t.createElement(a),o=t.getElementsByTagName(a)[0],g.async=1,g.src="https://www.google-analytics.com/analytics.js",o.parentNode.insertBefore(g,o)}(window,document,"script",0,"ga"),ga("create","UA-45583206-1","auto"),ga((function(e){window.gaClientId=e.get("clientId")}))</script><script>window.signupPersonalization=!1</script><meta charset="utf-8"><title data-rh="true">Behind the Scenes of a long EVE Online downtime | EVE Online</title><meta data-rh="true" property="og:type" content="game"/><meta data-rh="true" property="og:url" content="https://www.eveonline.com/news/view/behind-the-scenes-of-a-long-eve-online-downtime"/><meta data-rh="true" property="og:site_name" content="EVE Online"/><meta data-rh="true" itemprop="name" content="Behind the Scenes of a long EVE Online downtime | EVE Online"/><meta data-rh="true" property="og:title" content="Behind the Scenes of a long EVE Online downtime | EVE Online"/><meta data-rh="true" name="description" content="This dev blog was written collaboratively with CCP Masterplan. This dev blog is an account of what happened behind the scenes when EVE Online had one of its longest downtimes in years, on July 15th 2015\. We are sharing it with you because we know a lot of players work in the IT field and might"/><meta data-rh="true" itemprop="description" content="This dev blog was written collaboratively with CCP Masterplan. This dev blog is an account of what happened behind the scenes when EVE Online had one of its longest downtimes in years, on July 15th 2015\. We are sharing it with you because we know a lot of players work in the IT field and might"/><meta data-rh="true" property="og:description" content="This dev blog was written collaboratively with CCP Masterplan. This dev blog is an account of what happened behind the scenes when EVE Online had one of its longest downtimes in years, on July 15th 2015\. We are sharing it with you because we know a lot of players work in the IT field and might"/><meta data-rh="true" itemprop="image" content="https://content.eveonline.com/www/newssystem/media/67449/1/have-you-tried.jpg?fm=jpg&amp;w=1200&amp;h=630&amp;fit=fill"/><meta data-rh="true" property="og:image" content="https://content.eveonline.com/www/newssystem/media/67449/1/have-you-tried.jpg?fm=jpg&amp;w=1200&amp;h=630&amp;fit=fill"/><meta data-rh="true" property="og:image:width" content="1200"/><meta data-rh="true" property="og:image:height" content="630"/><meta data-rh="true" name="twitter:card" content="summary_large_image"/><meta data-rh="true" name="article:published_time" content="2015-08-07T00:00:00Z"/><meta data-rh="true" name="article:modified_time" content="2015-08-07T00:00:00Z"/><link data-rh="true" rel="canonical" href="https://www.eveonline.com/news/view/behind-the-scenes-of-a-long-eve-online-downtime"/><link data-rh="true" rel="alternate" hrefLang="en" href="https://www.eveonline.com/news/view/behind-the-scenes-of-a-long-eve-online-downtime"/><link data-rh="true" rel="alternate" hrefLang="fr" href="https://www.eveonline.com/fr/news/view/behind-the-scenes-of-a-long-eve-online-downtime"/><link data-rh="true" rel="alternate" hrefLang="de" href="https://www.eveonline.com/de/news/view/behind-the-scenes-of-a-long-eve-online-downtime"/><link data-rh="true" rel="alternate" hrefLang="ru" href="https://www.eveonline.com/ru/news/view/behind-the-scenes-of-a-long-eve-online-downtime"/><link data-rh="true" rel="alternate" hrefLang="ja" href="https://www.eveonline.com/ja/news/view/behind-the-scenes-of-a-long-eve-online-downtime"/><link data-rh="true" rel="alternate" hrefLang="ko" href="https://www.eveonline.com/ko/news/view/behind-the-scenes-of-a-long-eve-online-downtime"/><link data-rh="true" rel="alternate" hrefLang="es" href="https://www.eveonline.com/es/news/view/behind-the-scenes-of-a-long-eve-online-downtime"/><link data-rh="true" rel="alternate" hrefLang="zh" href="https://www.eveonline.com/zh/news/view/behind-the-scenes-of-a-long-eve-online-downtime"/><script data-rh="true" type="application/ld+json">
{
"@context": "http://schema.org",
"@graph": [
{
"@type": "NewsArticle",
"mainEntityOfPage": {
"@type": "WebPage",
"@id": "https://www.eveonline.com/news/view/undefined"
},
"headline": "Behind the Scenes of a long EVE Online downtime",
"image": ["//content.eveonline.com/www/newssystem/media/67449/1/have-you-tried.jpg"],
"datePublished": "2015-08-07T00:00:00Z",
"dateModified": "2015-08-07T00:00:00Z",
"author": {
"@type": "Person",
"name": "CCP Goliath"
},
"publisher": {
"@type": "Organization",
"name": "Fenris Creations",
"logo": {
"@type": "ImageObject",
"url": "https://web.ccpgamescdn.com/eveonlineassets/eveonline.png"
}
}
},
{
"@type": "BreadcrumbList",
"itemListElement": [{
"@type": "ListItem",
"position": 1,
"name": "EVE Online",
"item": "https://www.eveonline.com"
},{
"@type": "ListItem",
"position": 2,
"name": "News",
"item": "https://www.eveonline.com/news"
},{
"@type": "ListItem",
"position": 3,
"name": "Behind the Scenes of a long EVE Online downtime"
}]
}
]
}</script><meta name="color-scheme" content="dark"><meta name="viewport" content="width=device-width,initial-scale=1,minimum-scale=1,shrink-to-fit=no"><meta name="google-site-verification" content="V1MFuRc641J_fa6mTQ4WhHLYR5MLgLHIev1NE3huoFg"/><meta name="googlebot" content="index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1"/><meta property="fb:admins" content="336200018,1245262884"/><script>!function(e){"use strict";function s(s){if(s){var t=e.documentElement;t.classList?t.classList.add("webp"):t.className+=" webp",window.sessionStorage.setItem("webpSupport",!0)}}!function(e){if(window.sessionStorage&&window.sessionStorage.getItem("webpSupport"))s(!0);else{var t=new Image;t.onload=t.onerror=function(){e(2===t.height)},t.src="data:image/webp;base64,UklGRi4AAABXRUJQVlA4TCEAAAAvAUAAEB8wAiMwAgSSNtse/cXjxyCCmrYNWPwmHRH9jwMA"}}(s)}(document)</script><link rel="alternate" type="application/rss+xml" title="EVE News RSS Feed" href="https://www.eveonline.com/rss"><link rel="preconnect" href="https://www.google-analytics.com" crossorigin="anonymous"><link rel="preconnect" href="https://www.googletagmanager.com" crossorigin="anonymous"><link rel="preconnect" href="https://consent.cookiebot.com" crossorigin="anonymous"><link rel="preconnect" href="https://consentcdn.cookiebot.com" crossorigin="anonymous"><link rel="preconnect" href="https://web.ccpgamescdn.com" crossorigin="anonymous"><link rel="preload" href="https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Shentox-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Shentox-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Shentox-Light.woff2" as="font" type="font/woff2" crossorigin="anonymous"><link rel="preload" href="https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Shentox-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous"><style>@font-face{font-family:Shentox;font-display:swap;src:url(https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Shentox-SemiBold.woff2) format('woff2'),url(https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Shentox-SemiBold.woff) format('woff');font-weight:600;font-style:normal}@font-face{font-family:Shentox;font-display:swap;src:url(https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Shentox-Medium.woff2) format('woff2'),url(https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Shentox-Medium.woff) format('woff');font-weight:500;font-style:normal}@font-face{font-family:Shentox;font-display:swap;src:url(https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Shentox-Regular.woff2) format('woff2'),url(https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Shentox-Regular.woff) format('woff');font-weight:400;font-style:normal}@font-face{font-family:Shentox;font-display:swap;src:url(https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Shentox-Light.woff2) format('woff2'),url(https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Shentox-Light.woff) format('woff');font-weight:300;font-style:normal}@font-face{font-family:Rogan;font-display:swap;src:url(https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Rogan-SemiBold.woff2) format('woff2'),url(https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Rogan-SemiBold.woff) format('woff');font-weight:600;font-style:normal}@font-face{font-family:Rogan;font-display:swap;src:url(https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Rogan-Light.woff2) format('woff2'),url(https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Rogan-Light.woff) format('woff');font-weight:300;font-style:normal}@font-face{font-family:Rogan;font-display:swap;src:url(https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Rogan-Medium.woff2) format('woff2'),url(https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Rogan-Medium.woff) format('woff');font-weight:500;font-style:normal}@font-face{font-family:Rogan;font-display:swap;src:url(https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Rogan-Regular.woff2) format('woff2'),url(https://web.ccpgamescdn.com/aws/webfonts/shentox/webfonts-2.0/Rogan-Regular.woff) format('woff');font-weight:400;font-style:normal}</style><style data-styled="fxDDAl eZsQyb iwQzVe jGSGTM" data-styled-version="4.4.1">
/* sc-component-id: BackgroundImage__Bg-sc-xkyflv-0 */
.fxDDAl{-webkit-animation-name:fadeIn;animation-name:fadeIn;-webkit-animation-duration:2s;animation-duration:2s;-webkit-animation-iteration-count:1;animation-iteration-count:1;background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=2400&q=75);} @media only screen and (max-width:1920px){.fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=1920&q=75);}} @media only screen and (max-width:1700px){.fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=1700&q=75);}} @media only screen and (max-width:1500px){.fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=1500&q=75);}} @media only screen and (max-width:1300px){.fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=1300&q=75);}} @media only screen and (max-width:1100px){.fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=1100&q=75);}} @media only screen and (max-width:900px){.fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=900&q=75);}} @media only screen and (max-width:700px){.fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=700&q=75);}} @media only screen and (max-width:500px){.fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=500&q=75);}} @media only screen and (max-width:300px){.fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=300&q=75);}} html.webp .fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=2400&q=75&fm=webp);} @media only screen and (max-width:1920){html.webp .fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=1920&q=75&fm=webp);}} @media only screen and (max-width:1700px){html.webp .fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=1700&q=75&fm=webp);}} @media only screen and (max-width:1500px){html.webp .fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=1500&q=75&fm=webp);}} @media only screen and (max-width:1300px){html.webp .fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=1300&q=75&fm=webp);}} @media only screen and (max-width:1100px){html.webp .fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=1100&q=75&fm=webp);}} @media only screen and (max-width:900px){html.webp .fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=900&q=75&fm=webp);}} @media only screen and (max-width:700px){html.webp .fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=700&q=75&fm=webp);}} @media only screen and (max-width:500px){html.webp .fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=500&q=75&fm=webp);}} @media only screen and (max-width:300px){html.webp .fxDDAl{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/1mv6Tg8PUsMWyGauWckSOo/5be5f00d5fb6d445f420b7a1c669fd26/star-bg.jpg?w=300&q=75&fm=webp);}} @-webkit-keyframes fadeIn{0%{opacity:0;}20%{opacity:0;}60%{opacity:1;}100%{opacity:1;}} @keyframes fadeIn{0%{opacity:0;}20%{opacity:0;}60%{opacity:1;}100%{opacity:1;}}.eZsQyb{background-repeat:no-repeat;background-size:cover;background-position:center center;-webkit-animation-name:fadeIn;animation-name:fadeIn;-webkit-animation-duration:2s;animation-duration:2s;-webkit-animation-iteration-count:1;animation-iteration-count:1;background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=2400&q=75);} @media only screen and (max-width:1920px){.eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=1920&q=75);}} @media only screen and (max-width:1700px){.eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=1700&q=75);}} @media only screen and (max-width:1500px){.eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=1500&q=75);}} @media only screen and (max-width:1300px){.eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=1300&q=75);}} @media only screen and (max-width:1100px){.eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=1100&q=75);}} @media only screen and (max-width:900px){.eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=900&q=75);}} @media only screen and (max-width:700px){.eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=700&q=75);}} @media only screen and (max-width:500px){.eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=500&q=75);}} @media only screen and (max-width:300px){.eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=300&q=75);}} html.webp .eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=2400&q=75&fm=webp);} @media only screen and (max-width:1920){html.webp .eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=1920&q=75&fm=webp);}} @media only screen and (max-width:1700px){html.webp .eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=1700&q=75&fm=webp);}} @media only screen and (max-width:1500px){html.webp .eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=1500&q=75&fm=webp);}} @media only screen and (max-width:1300px){html.webp .eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=1300&q=75&fm=webp);}} @media only screen and (max-width:1100px){html.webp .eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=1100&q=75&fm=webp);}} @media only screen and (max-width:900px){html.webp .eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=900&q=75&fm=webp);}} @media only screen and (max-width:700px){html.webp .eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=700&q=75&fm=webp);}} @media only screen and (max-width:500px){html.webp .eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=500&q=75&fm=webp);}} @media only screen and (max-width:300px){html.webp .eZsQyb{background-image:url(https://images.ctfassets.net/7lhcm73ukv5p/3kBbL40e9KAKPMlM27jqRw/670eb7dd8cd76eab700ec8445373dd2c/news_detail_background.jpg?w=300&q=75&fm=webp);}} @-webkit-keyframes fadeIn{0%{opacity:0;}20%{opacity:0;}60%{opacity:1;}100%{opacity:1;}} @keyframes fadeIn{0%{opacity:0;}20%{opacity:0;}60%{opacity:1;}100%{opacity:1;}}
/* sc-component-id: Typography__DynamicComponent-sc-1bolua0-0 */
.iwQzVe{font-size:36px;font-weight:500;text-transform:unset;line-height:1.06;} @media (min-width:480px){.iwQzVe{font-size:calc(0.012371134020618556 * 100vw + 30.061855670103093px);}} @media (min-width:1450px){.iwQzVe{font-size:48px;}}.jGSGTM{font-size:26px;font-weight:500;text-transform:uppercase;line-height:1.08;} @media (min-width:480px){.jGSGTM{font-size:calc(0.010309278350515464 * 100vw + 21.051546391752577px);}} @media (min-width:1450px){.jGSGTM{font-size:36px;}}</style><style data-emotion="mantine "></style><link href="/static/css/npm-react-image-lightbox.67368503.chunk.css" rel="stylesheet"><link href="/static/css/main.b5e384e7.chunk.css" rel="stylesheet"></head><body><script>window.REDUX_STATE={"localize":{"languages":[],"translations":{},"options":{"renderToStaticMarkup":false,"renderInnerHtml":false,"ignoreTranslateChildren":false,"defaultLanguage":"","onMissingTranslation":function(e){e.translationId,e.languageCode;return"Missing translationId: ${ translationId } for language: ${ languageCode }"}}},"pages":{"hasContent":false,"isFetching":false,"article":{"title":"Behind the Scenes of a long EVE Online downtime","id":"behind-the-scenes-of-a-long-eve-online-downtime","publishingDate":"2015-08-07T00:00:00Z","author":"CCP Goliath","metaDescription":"This dev blog was written collaboratively with CCP Masterplan. This dev blog is an account of what happened behind the scenes when EVE Online had one of its longest downtimes in years, on July 15th 2015\\. We are sharing it with you because we know a lot of players work in the IT field and might","category":"dev-blogs","metaImageUrl":"\u002F\u002Fcontent.eveonline.com\u002Fwww\u002Fnewssystem\u002Fmedia\u002F67449\u002F1\u002Fhave-you-tried.jpg","content":"_This dev blog was written collaboratively with CCP Masterplan._\n\nThis dev blog is an account of what happened behind the scenes when EVE Online had one of its longest downtimes in years, on July 15\u003Csup\u003Eth\u003C\u002Fsup\u003E 2015\\. We are sharing it with you because we know a lot of players work in the IT field and might appreciate the war stories, and because it gives some interesting insight on the inner workings of a rather unique game cluster. If these things interest you, read on for an account of what happened on July 15\u003Csup\u003Eth\u003C\u002Fsup\u003E. If you are happy to be oblivious to these details, we recommend the more high level summary we posted [here](\u002F\u002Fcommunity.eveonline.com\u002Fnews\u002Fnews-channels\u002Feve-online-news\u002Fskillpoint-gift-for-extended-downtime-on-july-15th\u002F) shortly after the incident.\n\n**Background**\n\nThose of you who have been following us for a long time will doubtless have memories of downtimes extending over days rather than hours.  In recent years we have managed to, for the most part, eliminate these terribly long downtimes.  On July 15\u003Csup\u003Eth\u003C\u002Fsup\u003E we were inaccessible to players for 699 minutes, which is slightly longer than the downtimes that occurred on 2\u003Csup\u003End\u003C\u002Fsup\u003E and 3\u003Csup\u003Erd\u003C\u002Fsup\u003E of June 2013, immediately prior to the Odyssey deployment (which itself incidentally incurred a 342 minute downtime due to long running tiericide scripts).  Prior to that, Incarna in 2011 saw us down for 963 minutes, but really was the last of the very long deployments that followed our 6-month release cadence.  Right now, if we simply auto-reboot without deploying anything, we can be confident that that will take approximately 7 minutes, and if we are deploying we can normally expect to be up well within our allotted 30 minute downtime. \n\n**Startup**\n\nThe TQ cluster is made of approximately [250 server nodes](\u002F\u002Fwiki.eveonline.com\u002Fen\u002Fwiki\u002FTranquility#Node). To start up the cluster, all the nodes must perform a coordinated sequence of actions. These actions include assigning IDs to each node, making network connections between every pair of nodes, allocating which solar-systems will run on which node, and loading up the necessary data to allow each node to perform its tasks. Node tasks may include handling the region for a market, or a set of corps\u002Falliances, or skill training for a set of characters.\n\nDuring the start-up sequence, the cluster progresses through several stages. The cluster will not advance to the following stage until all nodes have reported that they have completed the current stage's actions. It begins at stage -4 and continues up to stage 0\\. Once all nodes have reported in at stage 0, the cluster is considered ready. One master node is chosen to orchestrate this sequence, nicknamed Polaris. The Polaris node is responsible for checking the stage of all other nodes, and sending out instructions to advance to the next stage once the appropriate conditions are met.\n\n-4: The node has started running and we have a connection to the database \n-3: The node has successfully opened a network connection to every other node \n-2: Address cache has been primed, this is basically the routing table so each node knows what every other node is supposed to be doing \n-1: All startup services have completed their initialization and now some pre-loading of data has begun (market, solar systems etc) \n0: The node has been told the cluster is ready, all startup data is loaded and services are ready to receive requests\n\n**Incident**\n\nWhen we took the server down on July 15\u003Csup\u003Eth\u003C\u002Fsup\u003E to deploy the first follow up patch to Aegis Sovereignty, we expected no incident – the test servers that we had previously applied the update to had started correctly and were behaving normally, and we assumed it would be a very standard 15-20 minute deployment.  Our first snag was our deployment tool taking a very long time to deploy the server package, eventually erroring out and being rebooted shortly before 11.30.  At this time we messaged a delay in startup but still didn’t expect anything out of the ordinary – tools can occasionally just fail, timeouts happen and we simply deployed again, this time successfully.  Server startup was first attempted at 11.42, by 11.46 we had 12 (out of ~250) nodes reporting that they were stuck, preventing a successful startup.  Not to be deterred, our first port of call was the oldest trick in the IT book – “turn it off and on again”. \n\n![IT](\u002F\u002Fcontent.eveonline.com\u002Fwww\u002Fnewssystem\u002Fmedia\u002F67449\u002F1\u002Fhave-you-tried.jpg)\n\nStartup went faster this time around, but we were still faced with 41 nodes stuck in their “-1” state.  We decided to set VIP access on the server, meaning that only accounts with special developer roles can access the server when it’s online, and do a full do-over of the deployment to rule out any locks, conflicts or human error from the first time around.  All our suspicions at this time were directly on the environment as being the cause, as the code was working on a test server and none of the data was giving off any red flags. \n\nUnfortunately, 2 startups later, we were no closer to having a healthy server online, so it was time to call in the cavalry.  EVE programmers joined Operations for further investigation into the code side of things, meanwhile some other developers discussed the feasibility and likely outcome of a rollback to the previous day’s build.  A rollback had not been considered up to this point as nothing had indicated that the code was an issue, but since we had some time while the investigation was ongoing, we figured we would test it. \n\nWhile EVE programmers worked through the start-up logs, attempting to figure out what errors might be related to the cause of the start-up issues, Ops worked through a number of hardware\u002FOS checks, as we had made various infrastructure changes in the previous days.  As the Polaris node is critical to orchestrating the start-up sequence, the server normally hosting the Polaris node was removed from the cluster. A node on a different physical server was chosen as a replacement. This change was an attempt to eliminate any hardware\u002Fsoftware failures that might be specific to a single server, however no improvement was seen in the next start-up attempt. Our test of the rollback was confirmed to work, but we still didn’t believe the code to be the issue – we were firm that it was either related to the build package itself, data, or the environment. \n\nTesting further, we decided to deploy our original TQ build to the Singularity test server.  The reasoning behind this was that Singularity has DUST data and services, whereas Multiplicity, our EVE Hotfix\u002FRelease test server, does not.  We considered this an unlikely scenario at best, but worth doing, and Sisi did indeed start up fine.  We noticed in out monitoring tools that the EVE process was trying to communicate out on an extra network card, IBM USB Remote NDIS Network Device, that was in the IBM blade servers.  This network interface is used to manage the server via IMM (Integrated Management Module) or AMM (Advanced Management Module).  Through this interface you can manage the server on a low level such as updating the firmware through the OS or getting information on the server.  The network interface gets assigned a Windows self-signed IP address that is not used in public IP version 4 space and has no routes to our private networks in the datacenter.  Although the EVE process is not supposed to use this network interface, to eliminate the possibility that it could cause issues we went ahead and disabled the interface.  On the TQ side, we disabled CREST and associated services to rule out everything we could, made a change to the name of the server package so we could ensure there hadn’t been bad overwrites, and tried again.  At 14.21 startup had completed, with 53 nodes stuck at “-1” status. \n\nWe tried a few long shots over the next hour as developers continued their investigations.  Our next idea came at 15.29, when EVE Dev presented a plan to empty some of the new sovereignty records from the database, backing them up in temporary tables, to see if that data was the cause of the issue.  The reasoning behind this was that at gameplay level, the most significant difference between start-up on Tuesday and on Wednesday was that on Tuesday all the sovereignty structures were in a fresh state, before any campaigns or vulnerable windows had been generated.  In reaction to this, CCP Lebowski and other QA began generating tremendously large amounts of Entosis campaigns on Singularity, to see if they could cause a comparable spike of errors, or a failed startup.  Despite loading Singularity with almost double the amount of sovereignty reinforcement events and vulnerability windows as TQ, it continued to start up perfectly every time.  We had the script tested on Duality by 15.48, confirming it worked as expected, and it was off to TQ with it.  The script ran, the tables were confirmed to be empty, and at 16.33 we anxiously watched the progression of the start-up sequence\n\n-4 \n-3 \n-2 \n-1 \n... \n0 \nShe's up!\n\nThis was great news. Now we had to figure out why clearing out the reinforcement and vulnerability data allowed the server to start up cleanly.\n\nOpening in this state was not an option, considering we had just effectively deleted a day’s worth of sovereignty campaigns, so investigations continued.  By this time, there were around 8 EVE programmers participating in the investigation, with several Operations staff conducting their own investigations in parallel, and practically every QA in the building trying to replicate the issue on Singularity.  By 17.10 we made the next change - the vulnerability window data only was re-added to the DB. The next start-up would help indicate if it was this or the reinforcement events that were blocking start-up.  We deployed the change, started up successfully, and had a vaguely smoking gun – the presence of campaigns themselves was somehow causing the issue.  EVE Dev focused their investigations accordingly. The remaining campaign data was then re-added back to the database.\n\n![Thanks \u002Fr\u002Feve](\u002F\u002Fcontent.eveonline.com\u002Fwww\u002Fnewssystem\u002Fmedia\u002F67449\u002F1\u002FIqy3lny.png)\n\nA theory about what might cause the stuck nodes had been formed: during start-up, the nodes running solar-systems with one or more sovereignty campaigns must talk to the nodes managing the related alliances. One of these queries is to look up the alliance's chosen capital system. According to the design for the new sovereignty system, changes to an alliance's capital system will take several days to come in to effect. During the days following the initial feature deployment we wanted alliances to more easily settle in to these new rules, and so we added a configuration option whereby we could temporarily override the 7-day timer with a much shorter delay. \n\nOur theory was that if these cross-node calls to lookup the alliance capital were interacting with the mechanism for loading configuration data (as used to override the default delay) in a particular way, it could lead to a cross-node deadlock. That is, the solar-system node that loaded the campaign is waiting for the alliance node to respond with the info it needs. Meanwhile the alliance node has loaded the capital configuration settings and is sending an update to that capital solar-system telling it \"You are the capital, you should use a +2 defense modifier\". The alliance node will not respond to any new requests until the solar-system node acknowledges the updated capital status, but the solar system node will not respond to any new requests until the alliance node answers the campaign query.  We submitted a code change that removed the configurable capital delay and should have eliminated the possibility of this causing a deadlock. During the next startup at 18.12 we were met with 51 nodes at “-1” – back to the drawing board (after one quick reboot just in case). \n\nSo what did we know by that point?\n\n* Starting up with no campaign data in the DB worked OK on TQ and on all test servers.\n* Starting up around 300 campaigns in the DB failed on TQ, but worked OK on all test servers.\n* Eliminating cross-node calls related to alliance capitals did not seem to make any difference.\n* We needed to do some experiments around loading campaigns on TQ to isolate what point in particular we experienced the problem.\n\nWe decided to completely remove campaign loading at start-up. The consequence of this was that the background spawning of command nodes wouldn't happen, but otherwise the sovereignty structures should load up as normal. Note that this is not intended as an actual fix, instead it is to give us a new data point about what conditions cause a successful\u002Ffailed start-up.  This experimental change (Hotfix#2) did indeed get us started up at 19.15, and led us to experiment #3 - what would happen if we now loaded the campaigns, but only after a brief delay, to allow everything else to finish starting up first?\n\nWe also tried re-enabling the campaign loading (that we had previously disabled in Hotfix#2), but delayed it and let it run asynchronously to the rest of the start-up.  At this point we were hit with some slight snags, with Perforce sync hangs causing delays in our build system, and some unrelated data storage issues, but with every hotfix we were getting closer and closer…\n\n![Pizza](\u002F\u002Fcontent.eveonline.com\u002Fwww\u002Fnewssystem\u002Fmedia\u002F67449\u002F1\u002FCJ-vErKWsAARyU8.jpg_large.jpg)\n\nto pizza, which mercifully arrived at 19.48 while our 3\u003Csup\u003Erd\u003C\u002Fsup\u003E hotfix was deploying.  TQ failed to start up again with this change, which was interesting because in Hotfix#3 we were loading the campaigns independently of the rest of the start-up tasks, and yet loading them was still able to break some nodes. \n\nReinvigorated by a ping-pong table full of pizza we redeployed Hotfix #2 for more experimentation and LIVE CONSOLE EXPERIMENTS.\n\n![Pro Dev](\u002F\u002Fcontent.eveonline.com\u002Fwww\u002Fnewssystem\u002Fmedia\u002F67449\u002F1\u002Fpics_026.jpg)\n\nOnce all nodes were up and we had verified that solar-systems had loaded ok, we opened up a python console on the server, allowing us to issue commands and observe the results in real time.  We performed a few checks to ensure that no campaigns were loaded on any nodes. As expected, they all reported that their campaigns were empty. We then requested all nodes to load up the campaigns that they were responsible for. Immediately the cluster started to become slow and unresponsive. After a few minutes, we started to see several nodes die and drop out of the cluster. As this happens, any solar systems on those dead nodes get remapped to remaining live nodes. We were now down to around 200 nodes out of 250.  We then instructed the cluster to run the campaign loading sequence again. What we expected to see was each node log out the campaigns it knows about, and then report that no further campaigns required loading (as they were all in memory). What actually happened is that again the cluster became unresponsive for a few minutes, and when everything settled down, a bunch more nodes had also died. This was most strange, as all that should have happened was the nodes logging out some info, as they had nothing new that required loading. A thought occurred: maybe it wasn’t actually the campaigns themselves that were causing the issues - what if it related to the logging that happens around the campaigns?  As the cluster was now in a bad state due all the dead nodes, we requested a reboot (staying on Hotfix#2) for some more tests.\n\nVia the console we modified the campaign loading functions so that they performed the load-from-DB operations normally, but all their logging operations were disabled.  We then repeated the same test from above - instruct all nodes to load their campaigns. This command completed almost instantly, and the cluster remained perfectly healthy. WTF! Further checks indicated that all campaigns had indeed successfully loaded.  We then repeated the second-time load test, again using the loading functions with the disabled logging. This also completed instantly, reporting that no new campaigns needed loading.\n\nAs a final test, we then switched the disabled logging back on, and issued another load-campaigns instruction. The cluster then regressed back to the earlier behaviour, showing excessive delays as nodes began to die off.  We seemed to have found the culprit (even if we didn’t know exactly how\u002Fwhy!).  For some reason, the logging channel used by the campaign system on TQ appeared to degrade node performance to the point that some of those nodes would actually drop out of the cluster.\n\n_Note: These channels refer to the logs used by developers for testing feature operation and investigating defects. They are independent of the activity logs that you might see in your character wallets, for example, or that GMs use in their customer service duties._\n\n![Logs](\u002F\u002Fcontent.eveonline.com\u002Fwww\u002Fnewssystem\u002Fmedia\u002F67449\u002F1\u002F53e7f9e6-fc03-4407-bd68-d7c9ad5791b2-A26675.jpg)\n\nHotfix #5 was requested at 21.48, containing what we believed would mitigate the issue. All logging within the campaign system was entirely removed, but otherwise the code was mostly unchanged. If this change led to a successful startup, we would be close to being able to re-open TQ.  It was deployed to TQ at 22.07.  We had a good startup by 22.22 and able to begin VIP checks to ensure that no sovereignty data had been damaged by our experiments.  One problem was found with a duplicate campaign, but that was easy enough to clean away.  We brought CREST back online at 22.38 and lifted VIP to open TQ to all players at 22.41. \n\n**Aftermath – Monday 20\u003Csup\u003Eth\u003C\u002Fsup\u003E July**\n\nFollowing our previous adventures in clusters failing to start, we needed to get to the bottom of why one particular log channel (the one used by sovereignty campaigns) on one particular server (Tranquillity) could cause extreme server stability issues. Therefore we scheduled some VIP time on TQ where we could perform a few experiments. Our goal was to find the simplest possible code that we could execute that would reproduce the symptoms in order to aid further investigations. We connected to a python console on a TQ node - from the console we could interactively 'poke at things' to see what we could break. (Prior to this, we had gone through the same steps on the Singularity and Duality test servers, and all tests had completed without triggering the error condition).\n\nWe had prepared a series of escalating steps that we intended to follow. Each one would progressively add more factors in to the mix, until the issue could be hopefully be reproduced. As it turned out, we didn't need to go very far at all! Here is what happened:\n\nFirstly we had two logging channels - a generic channel, and then the specific channel used within the sovereignty system for logging campaign activity. We then validated that both channels were working, by executing the following code on a single node.\n\n\u003Ctable border=\"0\" cellpadding=\"0\" cellspacing=\"0\"\u003E\n\n\u003Ctbody\u003E\n\n\u003Ctr\u003E\n\n\u003Ctd\u003E\n\n1\n\n2\n\n3\n\n4\n\n5\n\n6\n\n7\n\n8\n\n\u003C\u002Ftd\u003E\n\n\u003Ctd\u003E\n\n # generic_logger is a reference to a generic log channel\n\n # campaign_logger is a reference to the log channel used by sovereignty campaigns\n\n # Step 1a: Assert that output to the generic logger works correctly\n\n generic_logger.warn('The quick brown fox jumps over the lazy dog')\n\n # Step 1b: Assert that output to the campaign logger works correctly\n\n campaign_logger.warn('The quick brown fox jumps over the lazy dog')\n\n\u003C\u002Ftd\u003E\n\n\u003C\u002Ftr\u003E\n\n\u003C\u002Ftbody\u003E\n\n\u003C\u002Ftable\u003E\n\nAs we run this we were also observing the log output in real time via Splunk. We did indeed see each line appear once.\n\nNext we run step 1a on all 250 nodes in parallel at the same time. Immediately we see 250 entries pop up on the Splunk display. Then we do the same for step 1b, and see the same result - each node logs one instance of the line, and each line shows it came via the campaign channel.\n\nSo far so good. Now let's log a bit more data, but nothing that should cause any problems, right? Right? Hmm...\n\n\u003Ctable border=\"0\" cellpadding=\"0\" cellspacing=\"0\"\u003E\n\n\u003Ctbody\u003E\n\n\u003Ctr\u003E\n\n\u003Ctd\u003E\n\n1\n\n2\n\n3\n\n4\n\n5\n\n\u003C\u002Ftd\u003E\n\n\u003Ctd\u003E\n\n # Step 2a: Log the same line 500 times per node, on the generic log channel\n\n [generic_logger.warn('The quick brown fox jumps over the lazy dog') for i in range(500)]\n\n # Step 2b: Log the same line 500 times per node, on the campaign log channel\n\n [campaign_logger.warn('The quick brown fox jumps over the lazy dog') for i in range(500)]\n\n\u003C\u002Ftd\u003E\n\n\u003C\u002Ftr\u003E\n\n\u003C\u002Ftbody\u003E\n\n\u003C\u002Ftable\u003E\n\nFor those not familiar with Python, line 2 will loop 500 times, outputting the warning message once per loop via the generic log channel. Line 5 does the same thing, but to the campaign log channel.\n\nFirst we ran step 2a on all nodes in parallel. The command completed instantly, and we saw a spike of 125,000 lines (250*500) on the Splunk graph. That might seem like a lot of logging, but it isn't anything the system can't handle, especially in small bursts like this.  Next we ran step 2b in the same way. This was where something curious happened. The correct number of log lines did show up in Splunk (The logs **do** show something!), but the command did not appear to return as immediately as it did for step 2a. In fact it took a few minutes before the console became responsive again, and the returned data indicated that several nodes did not respond in time.  Looking over the status of the cluster, those nodes were now showing as dead. Somehow this innocent log line had managed to cause these nodes to time-out and drop out of the cluster.\n\nWell that was easier than we thought! We didn't even need to start doing anything with campaigns or the database to reproduce the dead cluster.  When a developer cannot rely on simple logging actions to not kill his production server, he is going to have a bad day. Much like we did on Wednesday 15th.  Having isolated the problem down even more, we restarted TQ and then gave the go ahead to drop VIP and opened up as normal.\n\nAs mentioned on a few occasions in the time-line above, we had never seen this happen on any other test server before or since. There is something unique to the way that TQ operates (or is configured). Investigations are continuing to try to track down why this logging behaviour happens only on TQ, and only on specific log channels. We know that TQ certainly operates at different scale in terms of the amount of hardware, but so far other experiments have suggested that this is not simply a matter of number of nodes.  The investigation continues…\n\n![Top. Men.](\u002F\u002Fcontent.eveonline.com\u002Fwww\u002Fnewssystem\u002Fmedia\u002F67449\u002F1\u002FRaidersWarehouse.jpg)"}},"language":"en","navigation":{"show":true,"mobileOpen":false,"mobileAccountOpen":false},"location":{"pathname":"\u002Fnews\u002Fview\u002Fbehind-the-scenes-of-a-long-eve-online-downtime","type":"PAGE","payload":{"lang":undefined,"page":"news","subpage":"view","id":"behind-the-scenes-of-a-long-eve-online-downtime","subid":undefined},"prev":{"pathname":"","type":"","payload":{}},"kind":"load","history":undefined,"hasSSR":true,"routesMap":{"ERROR":{"path":"\u002Ferror"},"PAGE":{"path":"\u002F:lang(|en|de|fr|ru|ja|ko|es|zh)?\u002F:page?\u002F:subpage?\u002F:id?\u002F:subid?","thunk":async function(e,t){const{payload:{lang:n,page:r,subpage:a,id:o}}=t().location;let i=Object(Ii.a)(t()),s=n;const l=Object(Ii.b)();if(null!=n&&-1!==l.indexOf(n)||(s="en"),s!==i&&(e((e=>({type:"LANGUAGE_CHANGE",payload:e}))(s)),i=s),"en"===n)return e(Object(ro.redirect)(Object(As.e)(r,a,o)));const c=function(e,t){const n=e||"home",r=io.a[n];if(r)return t&&r.children?r.children:r}(r,a);if(!c)return console.log("dispatch not found in actions.js"),e(Object(As.h)());if(c.disabled)return e(Object(As.h)());if("home"===r)return e("en"===s?Object(ro.redirect)(Object(As.e)()):Object(ro.redirect)(Object(As.e)(void 0,void 0,void 0,s)));if(c.redirect){const t=c.redirect;return t.payload.lang=i,e(Object(ro.redirect)(t))}if(e({type:"HIDE_NAVIGATION",payload:c.hideMenu}),c.populate){const o=await c.populate(e,t,i);if(o&&o.payload&&o.payload.page&&Object(Ls.c)(o.payload.page)&&a)return e(Object(ro.redirect)(Object(As.e)(r,null,null,n)))}switch(c.id){case"store":case"callback":case"login":break;default:e(Object(As.i)())}}}}},"auth":{"isLoggedIn":false,"isLoggingIn":false}}</script><script>window.APOLLO_STATE={"ROOT_QUERY":{"__typename":"Query","sectionCollection({\"limit\":20,\"locale\":\"en\",\"preview\":false,\"where\":{\"identifier\":\"twitch-stream-widget\"}})":{"__typename":"SectionCollection","items":[]},"sectionCollection({\"limit\":20,\"locale\":\"en\",\"preview\":false,\"where\":{\"identifier\":\"footer\"}})":{"__typename":"SectionCollection","items":[{"__typename":"Section","name":"EVE Online Footer (turbo old model - most of www)","sys":{"__ref":"Sys:6CYsNLR3hb071Z2uei68rC"},"headline":"Follow EVE Online","teaser":"EVE Online® and Fenris Creations™ and all related logos and other elements are trademarks of Fenris Creations.\n\n©##YEAR## Fenris Creations. All rights reserved.\n","body":null,"buttonText":null,"buttonUrl":null,"template":null,"videoId":null,"backgroundImage":null,"imageFile":null,"theme":null,"identifier":"footer","contentCollection({\"limit\":20})":{"__typename":"SectionContentCollection","total":4,"items":[{"__typename":"Content","name":"Footer Content 1","sys":{"__ref":"Sys:41m44JCpIu3teu1nZv0GAp"},"headline":"Play EVE Online","body":"\u003Cul\u003E\n \u003Cli\u003EPlay EVE Online\u003C\u002Fli\u003E\n \u003Cli\u003E\u003Ca href=\"\u002Fsignup\" target=\"_blank\"\u003EPlay for Free\u003C\u002Fa\u003E\u003C\u002Fli\u003E\n \u003Cli\u003E\u003Ca href=\"\u002Fdownload\"\u003EDownload the Game\u003C\u002Fa\u003E\u003C\u002Fli\u003E\n \u003Cli\u003E\u003Ca href=\"https:\u002F\u002Fstore.eveonline.com\u002F#omega\" target=\"_blank\"\u003EUpgrade to Omega\u003C\u002Fa\u003E\u003C\u002Fli\u003E\n \u003Cli\u003E\u003Ca href=\"https:\u002F\u002Fsupport.eveonline.com\u002Fhc\u002Fen-us\u002Farticles\u002F5885219196828-System-Requirements\" target=\"_blank\"\u003ESystem Requirements\u003C\u002Fa\u003E\u003C\u002Fli\u003E\n \u003Cli\u003E\u003Ca href=\"\u002Feve-academy\"\u003EStarter Guide for Beginners\u003C\u002Fa\u003E\u003C\u002Fli\u003E\n \u003Cli\u003E\u003Ca href=\"\u002Feve-academy\u002Fexcel-add-in\"\u003EEVE Online add-in for Excel\u003C\u002Fa\u003E\u003C\u002Fli\u003E\n\u003C\u002Ful\u003E","buttonText":null,"buttonUrl":null,"buttonTheme":null,"date":null,"imageFile":null},{"__typename":"Content","name":"Footer Content 2","sys":{"__ref":"Sys:4o8PeJSfSZO41AnOa1bjCY"},"headline":null,"body":"\u003Cul\u003E\u003Cli\u003ECurrent Players\u003C\u002Fli\u003E\u003Cli\u003E\u003Ca href=\"https:\u002F\u002Fsecure.eveonline.com\u002Faccount\" target=\"_blank\" rel=\"noopener noreferrer\"\u003EAccount Management\u003C\u002Fa\u003E\u003C\u002Fli\u003E\u003Cli\u003E\u003Ca href=\"https:\u002F\u002Fsupport.eveonline.com\u002F\" target=\"_blank\" rel=\"noopener noreferrer\"\u003ESupport\u003C\u002Fa\u003E\u003C\u002Fli\u003E\u003Cli\u003E\u003Ca href=\"https:\u002F\u002Fcommunity.eveonline.com\u002F\" target=\"_blank\" rel=\"noopener noreferrer\"\u003ECommunity\u003C\u002Fa\u003E\u003C\u002Fli\u003E\u003Cli\u003E\u003Ca href=\"https:\u002F\u002Fforums.eveonline.com\u002F\" target=\"_blank\" rel=\"noopener noreferrer\"\u003EForums\u003C\u002Fa\u003E\u003C\u002Fli\u003E\u003Cli\u003E\u003Ca href=\"https:\u002F\u002Fdevelopers.eveonline.com\u002F\" target=\"_blank\" rel=\"noopener noreferrer\"\u003EDevelopers\u003C\u002Fa\u003E\u003C\u002Fli\u003E\u003Cli\u003E\u003Ca href=\"\u002Frecruit\" data-id=\"a4c097ec\"\u003ERecruit a Friend\u003C\u002Fa\u003E\u003C\u002Fli\u003E\n\u003Cli\u003E\u003Ca href=\"\u002Freturn\"\u003EReturn to EVE\u003C\u002Fa\u003E\u003C\u002Fli\u003E\n \u003Cli\u003E\u003Ca href=\"\u002Fpartners\"\u003EPartnership Program\u003C\u002Fa\u003E\u003C\u002Fli\u003E\n \u003Cli\u003E\u003Ca href=\"\u002Fplex-for-good\"\u003EPLEX For Good\u003C\u002Fa\u003E\u003C\u002Fli\u003E\n\u003C\u002Ful\u003E\n","buttonText":null,"buttonUrl":null,"buttonTheme":null,"date":null,"imageFile":null},{"__typename":"Content","name":"Footer Content 3","sys":{"__ref":"Sys:179WqmemY0dx2FRfP9r7Fj"},"headline":null,"body":"\u003Cul\u003E\u003Cli\u003EWorlds of EVE\u003C\u002Fli\u003E\u003Cli\u003E\u003Ca href=\"https:\u002F\u002Funiverse.eveonline.com\u002F\" target=\"_blank\" rel=\"noopener noreferrer\"\u003EEVE Universe\u003C\u002Fa\u003E\u003C\u002Fli\u003E\u003Cli\u003E\u003Ca href=\"https:\u002F\u002Ffanfest.eveonline.com\u002F\" target=\"_blank\" rel=\"noopener noreferrer\"\u003EEVE Fanfest\u003C\u002Fa\u003E\u003C\u002Fli\u003E\u003Cli\u003E\u003Ca href=\"https:\u002F\u002Feveonline-merchandise-store.myshopify.com\" target=\"_blank\" rel=\"noopener noreferrer\"\u003EEVE Merchandise\u003C\u002Fa\u003E\u003C\u002Fli\u003E\u003Cli\u003E\u003Ca href=\"https:\u002F\u002Fwww.eveonline.com\u002Fdiscovery\u002F\" target=\"_blank\" rel=\"noopener noreferrer\"\u003EProject Discovery\u003C\u002Fa\u003E\u003C\u002Fli\u003E\u003C\u002Ful\u003E","buttonText":null,"buttonUrl":null,"buttonTheme":null,"date":null,"imageFile":null},{"__typename":"Content","name":"Footer Content 4","sys":{"__ref":"Sys:4ICpIi3SRiTPLxiFO0LSS0"},"headline":null,"body":"\u003Cul\u003E\u003Cli data-simpleFooter=\"remove\"\u003ESite Info\u003C\u002Fli\u003E\u003Cli data-simpleFooter=\"remove\"\u003E\u003Ca href=\"https:\u002F\u002Fwww.fenris.com\u002Fcompany\" target=\"_blank\" rel=\"noopener noreferrer\"\u003EAbout Fenris\u003C\u002Fa\u003E\u003C\u002Fli\u003E\u003Cli\u003E\u003Ca href=\"https:\u002F\u002Fsupport.eveonline.com\u002Fhc\u002Fen-us\u002Farticles\u002F8414770561948-EVE-Online-Terms-of-Service\" target=\"_blank\" rel=\"noopener noreferrer\"\u003ETerms of Service\u003C\u002Fa\u003E\u003C\u002Fli\u003E\u003Cli\u003E\u003Ca href=\"https:\u002F\u002Fwww.fenris.com\u002Fprivacy-policy\" target=\"_blank\" rel=\"noopener noreferrer\"\u003EPrivacy Policy\u003C\u002Fa\u003E\u003C\u002Fli\u003E\u003Cli data-simpleFooter=\"remove\"\u003E\u003Ca href=\"https:\u002F\u002Fwww.eveonline.com\u002Frss\" target=\"_blank\" rel=\"noopener noreferrer\"\u003ERSS Feed\u003C\u002Fa\u003E\u003C\u002Fli\u003E\u003C\u002Ful\u003E","buttonText":null,"buttonUrl":null,"buttonTheme":null,"date":null,"imageFile":null}]}}]},"page({\"id\":\"3mtu854CmybqjgenaEpQMP\",\"locale\":\"en\",\"preview\":false})":{"__typename":"Page","siteName":"News","slug":"news","metaTitle":"EVE Online News - Updates, blogs, events, patch notes & more","metaDescription":"Stay fully up-to-date with all the latest news, updates, events, and developments from EVE Online, the #1 space MMO game!","metaImage":{"__typename":"Asset","url":"https:\u002F\u002Fimages.ctfassets.net\u002F7lhcm73ukv5p\u002F57vEzM9swcMaXVAgB5dXyB\u002F0d1dfe6b6de9dcdb97ccad6482592e4b\u002Feve-news-meta-image_en.jpg"},"pageBackground":null,"body":null,"config":{"pageType":"single"},"sectionsCollection({\"limit\":20})":{"__typename":"PageSectionsCollection","total":5,"items":[{"__typename":"Section","name":"EVE News - Patch Notes Featured (Featured Carousel)","headline":null,"teaser":null,"body":null,"buttonText":null,"buttonUrl":null,"template":null,"videoId":null,"backgroundImage":null,"imageFile":null,"theme":null,"identifier":"news-page-patch-notes-carousel","sys":{"__typename":"Sys","publishedVersion":559},"contentCollection({\"limit\":10})":{"__typename":"SectionContentCollection","total":2,"items":[{"__typename":"Content","name":"News carousel - Force Projection: Ansiblex Capacitor Update","headline":"Operation Avalon: FAQ","body":"## Ansiblex Capacitor Update\nGame Design Director FC Okami is back with more details on force projection and the coming changes to Ansiblex Jump Bridges in the September Major Update.","buttonText":"LEARN MORE","buttonUrl":"https:\u002F\u002Fwww.eveonline.com\u002Fnews\u002Fview\u002Fforce-projection-ansiblex-capacitor-update","buttonTheme":null,"date":null,"imageFile":{"__typename":"Asset","url":"https:\u002F\u002Fimages.ctfassets.net\u002F7lhcm73ukv5p\u002F3gC6pCWgelclxHwLHyDLsy\u002Fbab9532b8c269f431b3bac2a6992a8ce\u002FScreenshot_EN_No_Copy_1920x1080.jpg","description":"","width":1920,"height":1080}},{"__typename":"Content","name":"News carousel- The Cradle of War Expansion is Here ","headline":"The Cradle of War Expansion is Here ","body":"### You can turn the tide of the war, whatever your playstyle. \nCradle of War, the new expansion for EVE Online, is live, bringing Military Campaigns, titles and achievements, a new starter space for rookie capsuleers, eight new ships, a historical epic arc, and more. ","buttonText":"Learn more","buttonUrl":"https:\u002F\u002Fwww.eveonline.com\u002Fnews\u002Fview\u002Fthe-cradle-of-war-expansion-is-here","buttonTheme":null,"date":null,"imageFile":{"__typename":"Asset","url":"https:\u002F\u002Fimages.ctfassets.net\u002F7lhcm73ukv5p\u002FOt7AtVQ0JR8CvEvzUfBZd\u002Fb7e24ce92444f1ff8f8c5e0072c50f3e\u002FEN_News-EmailHeader_Copy-LiveNow_1920x1080.jpg","description":"","width":1920,"height":1080}}]}},{"__typename":"Section","name":"EVE News - Page Header","headline":"EVE Online News","teaser":"Updates, blogs, events, patch notes & more","body":"Updates, blogs, events, patch notes & more","buttonText":null,"buttonUrl":null,"template":null,"videoId":null,"backgroundImage":null,"imageFile":{"__typename":"Asset","url":"https:\u002F\u002Fimages.ctfassets.net\u002F7lhcm73ukv5p\u002F68YBGWYEpcvrgSkO7atGwn\u002Fb6507f9999d3815631a360e0e9c3ed06\u002FEVE_Updates_Header_1920x701.jpg","description":"","width":1920,"height":701},"theme":null,"identifier":"news-page-header","sys":{"__typename":"Sys","publishedVersion":170},"contentCollection({\"limit\":10})":{"__typename":"SectionContentCollection","total":0,"items":[]}},{"__typename":"Section","name":"EVE News - Page Footer","headline":"Buy PLEX, get wealthy","teaser":"Sell for millions in EVE","body":null,"buttonText":"Buy now","buttonUrl":"https:\u002F\u002Fstore.eveonline.com\u002F#plex","template":null,"videoId":null,"backgroundImage":null,"imageFile":{"__typename":"Asset","url":"https:\u002F\u002Fimages.ctfassets.net\u002F7lhcm73ukv5p\u002F3NfcBBAG2I6i5p98MvMUJk\u002Fbcfd1eb23666b69b507efd4100b96c0b\u002Fbottom_plex.jpg","description":null,"width":1920,"height":1080},"theme":"Yellow","identifier":"news-page-footer","sys":{"__typename":"Sys","publishedVersion":69},"contentCollection({\"limit\":10})":{"__typename":"SectionContentCollection","total":0,"items":[]}},{"__typename":"Section","name":"EVE News - Essential Posts","headline":"Essential Posts","teaser":null,"body":null,"buttonText":null,"buttonUrl":null,"template":null,"videoId":null,"backgroundImage":null,"imageFile":null,"theme":null,"identifier":"news-featured-posts","sys":{"__typename":"Sys","publishedVersion":595},"contentCollection({\"limit\":10})":{"__typename":"SectionContentCollection","total":4,"items":[{"__typename":"Article"},{"__typename":"Article"},{"__typename":"Article"},{"__typename":"Article"}]}},null]}},"resourceSet({\"id\":\"68QQ8b0caBddJvZ4zTMEgQ\",\"locale\":\"en\",\"preview\":false})":{"__typename":"ResourceSet","name":"Tags for the news page","resourcesCollection({\"limit\":40,\"locale\":\"en\"})":{"__typename":"ResourceSetResourcesCollection","total":19,"items":[{"__typename":"Resource","key":"in-game-events","val":"In-game Events"},{"__typename":"Resource","key":"patch-notes","val":"Patch Notes"},{"__typename":"Resource","key":"expansion","val":"Expansion"},{"__typename":"Resource","key":"ccptv","val":"CCPTV"},{"__typename":"Resource","key":"monthly-economic-reports","val":"Monthly Economic Reports"},{"__typename":"Resource","key":"balance-changes","val":"Balance Changes"},{"__typename":"Resource","key":"the-scope","val":"The Scope"},{"__typename":"Resource","key":"pvp","val":"PVP"},{"__typename":"Resource","key":"development-updates","val":"Development Updates"},{"__typename":"Resource","key":"battle-reports","val":"Battle Reports"},{"__typename":"Resource","key":"community","val":"Community"},{"__typename":"Resource","key":"new-features","val":"New features"},{"__typename":"Resource","key":"economy","val":"Economy"},{"__typename":"Resource","key":"eve-pulse","val":"EVE Pulse"},{"__typename":"Resource","key":"offers","val":"Offers"},{"__typename":"Resource","key":"exploits","val":"Exploits"},{"__typename":"Resource","key":"test-servers","val":"Test Servers"},{"__typename":"Resource","key":"eve-vanguard","val":"EVE Vanguard"},{"__typename":"Resource","key":"carbon","val":"Carbon"}]}},"section({\"id\":\"4Czq2QMVGpes5srijmEYrK\",\"locale\":\"en\",\"preview\":false})":{"__typename":"Section","headline":"Essential Posts","contentCollection":{"__typename":"SectionContentCollection","total":4,"items":[{"__typename":"Article","title":"The Cradle of War Expansion is Here ","slug":"the-cradle-of-war-expansion-is-here","metaImageUrl":{"__typename":"Asset","url":"https:\u002F\u002Fimages.ctfassets.net\u002F7lhcm73ukv5p\u002FOt7AtVQ0JR8CvEvzUfBZd\u002Fb7e24ce92444f1ff8f8c5e0072c50f3e\u002FEN_News-EmailHeader_Copy-LiveNow_1920x1080.jpg"}},{"__typename":"Article","title":"Cradle of War: Expansion Notes","slug":"cradle-of-war-expansion-notes","metaImageUrl":{"__typename":"Asset","url":"https:\u002F\u002Fimages.ctfassets.net\u002F7lhcm73ukv5p\u002F5R8eXvpNPvAVuGWz3N6UdI\u002F01c0f8f71648164ecb1c49c0b41e3d4a\u002FCradle_of_War_Header_Image.png"}},{"__typename":"Article","title":"Patch Notes - Version 23.02","slug":"patch-notes-version-23-02","metaImageUrl":{"__typename":"Asset","url":"https:\u002F\u002Fimages.ctfassets.net\u002F7lhcm73ukv5p\u002F3QemA9HYWdHRMBgr0GwH4V\u002F4d5b17bda999714195eb33a34f0515b6\u002FCata_Expan.png"}},{"__typename":"Article","title":"Cradle of War In Focus ","slug":"cradle-of-war-in-focus","metaImageUrl":{"__typename":"Asset","url":"https:\u002F\u002Fimages.ctfassets.net\u002F7lhcm73ukv5p\u002F6IgNmMMcEYkkTH50Ab9fbA\u002Ff817635db2700489a10c5f23210713cc\u002F1_Screenshot_Copy_3840x2160.jpg"}}]}},"articleCollection({\"limit\":1,\"locale\":\"en\",\"order\":[\"publishingDate_DESC\"],\"preview\":false,\"where\":{\"tags_contains_all\":[\"patch-notes\"]}})":{"__typename":"ArticleCollection","items":[{"__typename":"Article","sys":{"__typename":"Sys","publishedAt":"2026-09-08T11:00:10.142Z","firstPublishedAt":"2026-06-09T11:00:05.498Z"},"slug":"patch-notes-version-24-01"}]},"articleCollection({\"limit\":1,\"locale\":\"en\",\"preview\":false,\"where\":{\"slug\":\"behind-the-scenes-of-a-long-eve-online-downtime\"}})":{"__typename":"ArticleCollection","items":[]}},"Sys:6CYsNLR3hb071Z2uei68rC":{"__typename":"Sys","id":"6CYsNLR3hb071Z2uei68rC"},"Sys:41m44JCpIu3teu1nZv0GAp":{"__typename":"Sys","id":"41m44JCpIu3teu1nZv0GAp"},"Sys:4o8PeJSfSZO41AnOa1bjCY":{"__typename":"Sys","id":"4o8PeJSfSZO41AnOa1bjCY"},"Sys:179WqmemY0dx2FRfP9r7Fj":{"__typename":"Sys","id":"179WqmemY0dx2FRfP9r7Fj"},"Sys:4ICpIi3SRiTPLxiFO0LSS0":{"__typename":"Sys","id":"4ICpIi3SRiTPLxiFO0LSS0"}}</script><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"><div class="AppLayout_layout__dcmAX"><div class="en"><div></div><main class="Content_content__MA3nS"><div><div class="BackgroundImage__Bg-sc-xkyflv-0 fxDDAl"><div class="BackgroundImage__Bg-sc-xkyflv-0 eZsQyb NewsDetailContainer_header__DBf+T"></div><div class="NewsDetail_newsDetail__soqD+"><div class="Container_container__fmk4w "><div class="NewsDetail_grid__alWQI"><div><h1 font-size="36,48" font-weight="500" class="Typography__DynamicComponent-sc-1bolua0-0 iwQzVe NewsDetail_title__piOn3">Behind the Scenes of a long EVE Online downtime</h1><div class="NewsDetail_authAndShare__8dCvb"><span class="DateAndAuthor_author_date__sXdb2">2015-08-07<span> <!-- -->- <!-- -->By<!-- --> <!-- -->CCP Goliath<!-- --> </span></span><div class="SocialShare_socialShare__RZFYE"><div class="SocialShare_inner__MfFOr"><a href="https://www.facebook.com/sharer/sharer.php?u=https://www.eveonline.com/news/view/undefined" rel="noopener noreferrer" target="_blank"></a><a href="https://twitter.com/share?url=https://www.eveonline.com/news/view/undefined" rel="noopener noreferrer" target="_blank"></a><a href="https://vk.com/share.php?url=https://www.eveonline.com/news/view/undefined" rel="noopener noreferrer" target="_blank"></a><a href="https://www.reddit.com/submit?url=https://www.eveonline.com/news/view/undefined" rel="noopener noreferrer" target="_blank"></a></div><div class="SocialShare_copyLink__RBtoN" role="presentation"><div class="SocialShare_copied__L-e4a">Link copied</div></div></div></div><div class="lazyload-wrapper "><div style="height:475px" class="lazyload-placeholder"></div></div><div class="NewsDetail_content__N4TV-"><p><em>This dev blog was written collaboratively with CCP Masterplan.</em></p>
<p>This dev blog is an account of what happened behind the scenes when EVE Online had one of its longest downtimes in years, on July 15<sup>th</sup> 2015. We are sharing it with you because we know a lot of players work in the IT field and might appreciate the war stories, and because it gives some interesting insight on the inner workings of a rather unique game cluster. If these things interest you, read on for an account of what happened on July 15<sup>th</sup>. If you are happy to be oblivious to these details, we recommend the more high level summary we posted <a href="//community.eveonline.com/news/news-channels/eve-online-news/skillpoint-gift-for-extended-downtime-on-july-15th/">here</a> shortly after the incident.</p>
<p><strong>Background</strong></p>
<p>Those of you who have been following us for a long time will doubtless have memories of downtimes extending over days rather than hours.  In recent years we have managed to, for the most part, eliminate these terribly long downtimes.  On July 15<sup>th</sup> we were inaccessible to players for 699 minutes, which is slightly longer than the downtimes that occurred on 2<sup>nd</sup> and 3<sup>rd</sup> of June 2013, immediately prior to the Odyssey deployment (which itself incidentally incurred a 342 minute downtime due to long running tiericide scripts).  Prior to that, Incarna in 2011 saw us down for 963 minutes, but really was the last of the very long deployments that followed our 6-month release cadence.  Right now, if we simply auto-reboot without deploying anything, we can be confident that that will take approximately 7 minutes, and if we are deploying we can normally expect to be up well within our allotted 30 minute downtime. </p>
<p><strong>Startup</strong></p>
<p>The TQ cluster is made of approximately <a href="//wiki.eveonline.com/en/wiki/Tranquility#Node">250 server nodes</a>. To start up the cluster, all the nodes must perform a coordinated sequence of actions. These actions include assigning IDs to each node, making network connections between every pair of nodes, allocating which solar-systems will run on which node, and loading up the necessary data to allow each node to perform its tasks. Node tasks may include handling the region for a market, or a set of corps/alliances, or skill training for a set of characters.</p>
<p>During the start-up sequence, the cluster progresses through several stages. The cluster will not advance to the following stage until all nodes have reported that they have completed the current stage&#x27;s actions. It begins at stage -4 and continues up to stage 0. Once all nodes have reported in at stage 0, the cluster is considered ready. One master node is chosen to orchestrate this sequence, nicknamed Polaris. The Polaris node is responsible for checking the stage of all other nodes, and sending out instructions to advance to the next stage once the appropriate conditions are met.</p>
<p>-4: The node has started running and we have a connection to the database<br/>
-3: The node has successfully opened a network connection to every other node<br/>
-2: Address cache has been primed, this is basically the routing table so each node knows what every other node is supposed to be doing<br/>
-1: All startup services have completed their initialization and now some pre-loading of data has begun (market, solar systems etc)<br/>
0: The node has been told the cluster is ready, all startup data is loaded and services are ready to receive requests</p>
<p><strong>Incident</strong></p>
<p>When we took the server down on July 15<sup>th</sup> to deploy the first follow up patch to Aegis Sovereignty, we expected no incident – the test servers that we had previously applied the update to had started correctly and were behaving normally, and we assumed it would be a very standard 15-20 minute deployment.  Our first snag was our deployment tool taking a very long time to deploy the server package, eventually erroring out and being rebooted shortly before 11.30.  At this time we messaged a delay in startup but still didn’t expect anything out of the ordinary – tools can occasionally just fail, timeouts happen and we simply deployed again, this time successfully.  Server startup was first attempted at 11.42, by 11.46 we had 12 (out of ~250) nodes reporting that they were stuck, preventing a successful startup.  Not to be deterred, our first port of call was the oldest trick in the IT book – “turn it off and on again”. </p>
<p><img src="//content.eveonline.com/www/newssystem/media/67449/1/have-you-tried.jpg?w=900&amp;fm=jpg&amp;fl=progressive" loading="lazy" alt=""/></p>
<p>Startup went faster this time around, but we were still faced with 41 nodes stuck in their “-1” state.  We decided to set VIP access on the server, meaning that only accounts with special developer roles can access the server when it’s online, and do a full do-over of the deployment to rule out any locks, conflicts or human error from the first time around.  All our suspicions at this time were directly on the environment as being the cause, as the code was working on a test server and none of the data was giving off any red flags. </p>
<p>Unfortunately, 2 startups later, we were no closer to having a healthy server online, so it was time to call in the cavalry.  EVE programmers joined Operations for further investigation into the code side of things, meanwhile some other developers discussed the feasibility and likely outcome of a rollback to the previous day’s build.  A rollback had not been considered up to this point as nothing had indicated that the code was an issue, but since we had some time while the investigation was ongoing, we figured we would test it. </p>
<p>While EVE programmers worked through the start-up logs, attempting to figure out what errors might be related to the cause of the start-up issues, Ops worked through a number of hardware/OS checks, as we had made various infrastructure changes in the previous days.  As the Polaris node is critical to orchestrating the start-up sequence, the server normally hosting the Polaris node was removed from the cluster. A node on a different physical server was chosen as a replacement. This change was an attempt to eliminate any hardware/software failures that might be specific to a single server, however no improvement was seen in the next start-up attempt. Our test of the rollback was confirmed to work, but we still didn’t believe the code to be the issue – we were firm that it was either related to the build package itself, data, or the environment. </p>
<p>Testing further, we decided to deploy our original TQ build to the Singularity test server.  The reasoning behind this was that Singularity has DUST data and services, whereas Multiplicity, our EVE Hotfix/Release test server, does not.  We considered this an unlikely scenario at best, but worth doing, and Sisi did indeed start up fine.  We noticed in out monitoring tools that the EVE process was trying to communicate out on an extra network card, IBM USB Remote NDIS Network Device, that was in the IBM blade servers.  This network interface is used to manage the server via IMM (Integrated Management Module) or AMM (Advanced Management Module).  Through this interface you can manage the server on a low level such as updating the firmware through the OS or getting information on the server.  The network interface gets assigned a Windows self-signed IP address that is not used in public IP version 4 space and has no routes to our private networks in the datacenter.  Although the EVE process is not supposed to use this network interface, to eliminate the possibility that it could cause issues we went ahead and disabled the interface.  On the TQ side, we disabled CREST and associated services to rule out everything we could, made a change to the name of the server package so we could ensure there hadn’t been bad overwrites, and tried again.  At 14.21 startup had completed, with 53 nodes stuck at “-1” status. </p>
<p>We tried a few long shots over the next hour as developers continued their investigations.  Our next idea came at 15.29, when EVE Dev presented a plan to empty some of the new sovereignty records from the database, backing them up in temporary tables, to see if that data was the cause of the issue.  The reasoning behind this was that at gameplay level, the most significant difference between start-up on Tuesday and on Wednesday was that on Tuesday all the sovereignty structures were in a fresh state, before any campaigns or vulnerable windows had been generated.  In reaction to this, CCP Lebowski and other QA began generating tremendously large amounts of Entosis campaigns on Singularity, to see if they could cause a comparable spike of errors, or a failed startup.  Despite loading Singularity with almost double the amount of sovereignty reinforcement events and vulnerability windows as TQ, it continued to start up perfectly every time.  We had the script tested on Duality by 15.48, confirming it worked as expected, and it was off to TQ with it.  The script ran, the tables were confirmed to be empty, and at 16.33 we anxiously watched the progression of the start-up sequence</p>
<p>-4<br/>
-3<br/>
-2<br/>
-1<br/>
...<br/>
0<br/>
She&#x27;s up!</p>
<p>This was great news. Now we had to figure out why clearing out the reinforcement and vulnerability data allowed the server to start up cleanly.</p>
<p>Opening in this state was not an option, considering we had just effectively deleted a day’s worth of sovereignty campaigns, so investigations continued.  By this time, there were around 8 EVE programmers participating in the investigation, with several Operations staff conducting their own investigations in parallel, and practically every QA in the building trying to replicate the issue on Singularity.  By 17.10 we made the next change - the vulnerability window data only was re-added to the DB. The next start-up would help indicate if it was this or the reinforcement events that were blocking start-up.  We deployed the change, started up successfully, and had a vaguely smoking gun – the presence of campaigns themselves was somehow causing the issue.  EVE Dev focused their investigations accordingly. The remaining campaign data was then re-added back to the database.</p>
<p><img src="//content.eveonline.com/www/newssystem/media/67449/1/Iqy3lny.png" loading="lazy" alt=""/></p>
<p>A theory about what might cause the stuck nodes had been formed: during start-up, the nodes running solar-systems with one or more sovereignty campaigns must talk to the nodes managing the related alliances. One of these queries is to look up the alliance&#x27;s chosen capital system. According to the design for the new sovereignty system, changes to an alliance&#x27;s capital system will take several days to come in to effect. During the days following the initial feature deployment we wanted alliances to more easily settle in to these new rules, and so we added a configuration option whereby we could temporarily override the 7-day timer with a much shorter delay. </p>
<p>Our theory was that if these cross-node calls to lookup the alliance capital were interacting with the mechanism for loading configuration data (as used to override the default delay) in a particular way, it could lead to a cross-node deadlock. That is, the solar-system node that loaded the campaign is waiting for the alliance node to respond with the info it needs. Meanwhile the alliance node has loaded the capital configuration settings and is sending an update to that capital solar-system telling it &quot;You are the capital, you should use a +2 defense modifier&quot;. The alliance node will not respond to any new requests until the solar-system node acknowledges the updated capital status, but the solar system node will not respond to any new requests until the alliance node answers the campaign query.  We submitted a code change that removed the configurable capital delay and should have eliminated the possibility of this causing a deadlock. During the next startup at 18.12 we were met with 51 nodes at “-1” – back to the drawing board (after one quick reboot just in case). </p>
<p>So what did we know by that point?</p>
<ul>
<li>Starting up with no campaign data in the DB worked OK on TQ and on all test servers.</li>
<li>Starting up around 300 campaigns in the DB failed on TQ, but worked OK on all test servers.</li>
<li>Eliminating cross-node calls related to alliance capitals did not seem to make any difference.</li>
<li>We needed to do some experiments around loading campaigns on TQ to isolate what point in particular we experienced the problem.</li>
</ul>
<p>We decided to completely remove campaign loading at start-up. The consequence of this was that the background spawning of command nodes wouldn&#x27;t happen, but otherwise the sovereignty structures should load up as normal. Note that this is not intended as an actual fix, instead it is to give us a new data point about what conditions cause a successful/failed start-up.  This experimental change (Hotfix#2) did indeed get us started up at 19.15, and led us to experiment #3 - what would happen if we now loaded the campaigns, but only after a brief delay, to allow everything else to finish starting up first?</p>
<p>We also tried re-enabling the campaign loading (that we had previously disabled in Hotfix#2), but delayed it and let it run asynchronously to the rest of the start-up.  At this point we were hit with some slight snags, with Perforce sync hangs causing delays in our build system, and some unrelated data storage issues, but with every hotfix we were getting closer and closer…</p>
<p><img src="//content.eveonline.com/www/newssystem/media/67449/1/CJ-vErKWsAARyU8.jpg?w=900&amp;fm=jpg&amp;fl=progressive_large.jpg?w=900&amp;fm=jpg&amp;fl=progressive" loading="lazy" alt=""/></p>
<p>to pizza, which mercifully arrived at 19.48 while our 3<sup>rd</sup> hotfix was deploying.  TQ failed to start up again with this change, which was interesting because in Hotfix#3 we were loading the campaigns independently of the rest of the start-up tasks, and yet loading them was still able to break some nodes. </p>
<p>Reinvigorated by a ping-pong table full of pizza we redeployed Hotfix #2 for more experimentation and LIVE CONSOLE EXPERIMENTS.</p>
<p><img src="//content.eveonline.com/www/newssystem/media/67449/1/pics_026.jpg?w=900&amp;fm=jpg&amp;fl=progressive" loading="lazy" alt=""/></p>
<p>Once all nodes were up and we had verified that solar-systems had loaded ok, we opened up a python console on the server, allowing us to issue commands and observe the results in real time.  We performed a few checks to ensure that no campaigns were loaded on any nodes. As expected, they all reported that their campaigns were empty. We then requested all nodes to load up the campaigns that they were responsible for. Immediately the cluster started to become slow and unresponsive. After a few minutes, we started to see several nodes die and drop out of the cluster. As this happens, any solar systems on those dead nodes get remapped to remaining live nodes. We were now down to around 200 nodes out of 250.  We then instructed the cluster to run the campaign loading sequence again. What we expected to see was each node log out the campaigns it knows about, and then report that no further campaigns required loading (as they were all in memory). What actually happened is that again the cluster became unresponsive for a few minutes, and when everything settled down, a bunch more nodes had also died. This was most strange, as all that should have happened was the nodes logging out some info, as they had nothing new that required loading. A thought occurred: maybe it wasn’t actually the campaigns themselves that were causing the issues - what if it related to the logging that happens around the campaigns?  As the cluster was now in a bad state due all the dead nodes, we requested a reboot (staying on Hotfix#2) for some more tests.</p>
<p>Via the console we modified the campaign loading functions so that they performed the load-from-DB operations normally, but all their logging operations were disabled.  We then repeated the same test from above - instruct all nodes to load their campaigns. This command completed almost instantly, and the cluster remained perfectly healthy. WTF! Further checks indicated that all campaigns had indeed successfully loaded.  We then repeated the second-time load test, again using the loading functions with the disabled logging. This also completed instantly, reporting that no new campaigns needed loading.</p>
<p>As a final test, we then switched the disabled logging back on, and issued another load-campaigns instruction. The cluster then regressed back to the earlier behaviour, showing excessive delays as nodes began to die off.  We seemed to have found the culprit (even if we didn’t know exactly how/why!).  For some reason, the logging channel used by the campaign system on TQ appeared to degrade node performance to the point that some of those nodes would actually drop out of the cluster.</p>
<p><em>Note: These channels refer to the logs used by developers for testing feature operation and investigating defects. They are independent of the activity logs that you might see in your character wallets, for example, or that GMs use in their customer service duties.</em></p>
<p><img src="//content.eveonline.com/www/newssystem/media/67449/1/53e7f9e6-fc03-4407-bd68-d7c9ad5791b2-A26675.jpg?w=900&amp;fm=jpg&amp;fl=progressive" loading="lazy" alt=""/></p>
<p>Hotfix #5 was requested at 21.48, containing what we believed would mitigate the issue. All logging within the campaign system was entirely removed, but otherwise the code was mostly unchanged. If this change led to a successful startup, we would be close to being able to re-open TQ.  It was deployed to TQ at 22.07.  We had a good startup by 22.22 and able to begin VIP checks to ensure that no sovereignty data had been damaged by our experiments.  One problem was found with a duplicate campaign, but that was easy enough to clean away.  We brought CREST back online at 22.38 and lifted VIP to open TQ to all players at 22.41. </p>
<p><strong>Aftermath – Monday 20<sup>th</sup> July</strong></p>
<p>Following our previous adventures in clusters failing to start, we needed to get to the bottom of why one particular log channel (the one used by sovereignty campaigns) on one particular server (Tranquillity) could cause extreme server stability issues. Therefore we scheduled some VIP time on TQ where we could perform a few experiments. Our goal was to find the simplest possible code that we could execute that would reproduce the symptoms in order to aid further investigations. We connected to a python console on a TQ node - from the console we could interactively &#x27;poke at things&#x27; to see what we could break. (Prior to this, we had gone through the same steps on the Singularity and Duality test servers, and all tests had completed without triggering the error condition).</p>
<p>We had prepared a series of escalating steps that we intended to follow. Each one would progressively add more factors in to the mix, until the issue could be hopefully be reproduced. As it turned out, we didn&#x27;t need to go very far at all! Here is what happened:</p>
<p>Firstly we had two logging channels - a generic channel, and then the specific channel used within the sovereignty system for logging campaign activity. We then validated that both channels were working, by executing the following code on a single node.</p>
<div class="NewsDetail_table__+3sW8"><table><tbody><tr><td>
<p>1</p>
<p>2</p>
<p>3</p>
<p>4</p>
<p>5</p>
<p>6</p>
<p>7</p>
<p>8</p>
</td><td>
<p> # generic_logger is a reference to a generic log channel</p>
<p> # campaign_logger is a reference to the log channel used by sovereignty campaigns</p>
<p> # Step 1a: Assert that output to the generic logger works correctly</p>
<p> generic_logger.warn(&#x27;The quick brown fox jumps over the lazy dog&#x27;)</p>
<p> # Step 1b: Assert that output to the campaign logger works correctly</p>
<p> campaign_logger.warn(&#x27;The quick brown fox jumps over the lazy dog&#x27;)</p>
</td></tr></tbody></table></div>
<p>As we run this we were also observing the log output in real time via Splunk. We did indeed see each line appear once.</p>
<p>Next we run step 1a on all 250 nodes in parallel at the same time. Immediately we see 250 entries pop up on the Splunk display. Then we do the same for step 1b, and see the same result - each node logs one instance of the line, and each line shows it came via the campaign channel.</p>
<p>So far so good. Now let&#x27;s log a bit more data, but nothing that should cause any problems, right? Right? Hmm...</p>
<div class="NewsDetail_table__+3sW8"><table><tbody><tr><td>
<p>1</p>
<p>2</p>
<p>3</p>
<p>4</p>
<p>5</p>
</td><td>
<p> # Step 2a: Log the same line 500 times per node, on the generic log channel</p>
<p> [generic_logger.warn(&#x27;The quick brown fox jumps over the lazy dog&#x27;) for i in range(500)]</p>
<p> # Step 2b: Log the same line 500 times per node, on the campaign log channel</p>
<p> [campaign_logger.warn(&#x27;The quick brown fox jumps over the lazy dog&#x27;) for i in range(500)]</p>
</td></tr></tbody></table></div>
<p>For those not familiar with Python, line 2 will loop 500 times, outputting the warning message once per loop via the generic log channel. Line 5 does the same thing, but to the campaign log channel.</p>
<p>First we ran step 2a on all nodes in parallel. The command completed instantly, and we saw a spike of 125,000 lines (250*500) on the Splunk graph. That might seem like a lot of logging, but it isn&#x27;t anything the system can&#x27;t handle, especially in small bursts like this.  Next we ran step 2b in the same way. This was where something curious happened. The correct number of log lines did show up in Splunk (The logs <strong>do</strong> show something!), but the command did not appear to return as immediately as it did for step 2a. In fact it took a few minutes before the console became responsive again, and the returned data indicated that several nodes did not respond in time.  Looking over the status of the cluster, those nodes were now showing as dead. Somehow this innocent log line had managed to cause these nodes to time-out and drop out of the cluster.</p>
<p>Well that was easier than we thought! We didn&#x27;t even need to start doing anything with campaigns or the database to reproduce the dead cluster.  When a developer cannot rely on simple logging actions to not kill his production server, he is going to have a bad day. Much like we did on Wednesday 15th.  Having isolated the problem down even more, we restarted TQ and then gave the go ahead to drop VIP and opened up as normal.</p>
<p>As mentioned on a few occasions in the time-line above, we had never seen this happen on any other test server before or since. There is something unique to the way that TQ operates (or is configured). Investigations are continuing to try to track down why this logging behaviour happens only on TQ, and only on specific log channels. We know that TQ certainly operates at different scale in terms of the amount of hardware, but so far other experiments have suggested that this is not simply a matter of number of nodes.  The investigation continues…</p>
<p><img src="//content.eveonline.com/www/newssystem/media/67449/1/RaidersWarehouse.jpg?w=900&amp;fm=jpg&amp;fl=progressive" loading="lazy" alt=""/></p></div><div class="NewsDetail_content__N4TV-"></div></div><div class="NewsDetail_sidebar__wOoXe"><div class="adGlare_adsWrapper__eMUBH adGlare_large__w2Trr"><section class="Section_section__itmgp"><div class="slick-slider slick-initialized"><div class="slick-list"><div class="slick-track" style="width:100%;left:0%"><div data-index="0" class="slick-slide slick-active slick-current" tabindex="-1" aria-hidden="false" style="outline:none;width:100%;position:relative;left:0;opacity:1;transition:opacity 500ms ease, visibility 500ms ease"><div><a href="https://www.eveonline.com/recruit?utm_source=www&amp;utm_medium=banner&amp;utm_campaign=recruitment&amp;utm_content=defaultcreative" tabindex="-1" style="width:100%;display:inline-block"><img alt="" src="https://web.ccpgamescdn.com/aws/eveonline/images/fallback/recruit-fallback-en.png"/></a></div></div></div></div></div></section></div><div class="Search_search__SSzQT"><form><input type="text" placeholder="Search EVE News" value=""/><button type="button" name="Search"></button></form></div><div class="Frame_box__p0fm0"><div class="Frame_content__JDasz FeaturedPosts_featured__AXh3n Frame_noCorner__SQJqb Frame_smallBottomCut__2J2So Frame_regular__A3N-e"><h2 font-size="26,36" font-weight="500" class="Typography__DynamicComponent-sc-1bolua0-0 jGSGTM Heading_heading__creO2 Heading_small__ArtnA Heading_colorTheme__KhZnY"><span>Essential Posts</span></h2><article class="FeaturedPosts_featured__content__sO2EH"><a href="/news/view/the-cradle-of-war-expansion-is-here" data-id="8170174a"><div class="FeaturedPosts_featured__imgwrapper__YoMEb"><div class="lazyload-wrapper "><div style="height:73px" class="lazyload-placeholder"></div></div></div></a><a href="/news/view/the-cradle-of-war-expansion-is-here" data-id="8170174a">The Cradle of War Expansion is Here </a></article><article class="FeaturedPosts_featured__content__sO2EH"><a href="/news/view/cradle-of-war-expansion-notes" data-id="4e207410"><div class="FeaturedPosts_featured__imgwrapper__YoMEb"><div class="lazyload-wrapper "><div style="height:73px" class="lazyload-placeholder"></div></div></div></a><a href="/news/view/cradle-of-war-expansion-notes" data-id="4e207410">Cradle of War: Expansion Notes</a></article><article class="FeaturedPosts_featured__content__sO2EH"><a href="/news/view/patch-notes-version-23-02" data-id="e6e8e0ce"><div class="FeaturedPosts_featured__imgwrapper__YoMEb"><div class="lazyload-wrapper "><div style="height:73px" class="lazyload-placeholder"></div></div></div></a><a href="/news/view/patch-notes-version-23-02" data-id="e6e8e0ce">Patch Notes - Version 23.02</a></article><article class="FeaturedPosts_featured__content__sO2EH"><a href="/news/view/cradle-of-war-in-focus" data-id="88010b01"><div class="FeaturedPosts_featured__imgwrapper__YoMEb"><div class="lazyload-wrapper "><div style="height:73px" class="lazyload-placeholder"></div></div></div></a><a href="/news/view/cradle-of-war-in-focus" data-id="88010b01">Cradle of War In Focus </a></article></div></div></div></div></div></div></div></div></main><footer class="Footer_footer__C-rC9"><div><div class="Container_container__fmk4w "><div data-simplefooter="remove" class="Footer_row__rtpKa Footer_top__Amz49"><h3>Follow EVE Online</h3><div class="styles_social__R2Qy6"><div class="styles_socialIcon__zlwbv styles_facebook__z4uyv"><a class="idm-share-facebook" href="//www.facebook.com/eveonline/" title="Facebook" data-id="" target="_blank" aria-label="Facebook" rel="noopener noreferrer"></a></div><div class="styles_socialIcon__zlwbv styles_twitter__wrj69"><a class="idm-share-twitter" href="//twitter.com/eveonline/" title="Twitter" data-id="" target="_blank" aria-label="Twitter" rel="noopener noreferrer"></a></div><div class="styles_socialIcon__zlwbv styles_youtube__-RSrH"><a class="idm-share-youtube" href="//www.youtube.com/channel/UCwF3VyalTHzL0L-GDlwtbRw/" title="YouTube" data-id="" target="_blank" aria-label="YouTube" rel="noopener noreferrer"></a></div><div class="styles_socialIcon__zlwbv styles_twitch__vsWUx"><a class="idm-share-twitch" href="//www.twitch.tv/ccp" title="Twitch" data-id="" target="_blank" aria-label="Twitch" rel="noopener noreferrer"></a></div><div class="styles_socialIcon__zlwbv undefined"><a class="idm-share-discord" href="//eveonline.com/discord" title="Discord" data-id="" target="_blank" aria-label="Discord" rel="noopener noreferrer"></a></div><div class="styles_socialIcon__zlwbv styles_instagram__p-MPU"><a class="idm-share-instagram" href="//www.instagram.com/eveonline/" title="Instagram" data-id="" target="_blank" aria-label="Instagram" rel="noopener noreferrer"></a></div><div class="styles_socialIcon__zlwbv styles_vk__Eaz1y"><a class="idm-share-vk" href="//vk.com/eveonline" title="VK" data-id="" target="_blank" aria-label="VK" rel="noopener noreferrer"></a></div></div></div><div class="Footer_row__rtpKa Footer_below__iHje+"><div class="Footer_language__W5m6r"><div class="SelectLanguage_language__KhPHU SelectLanguage_fpTheme__CkFYu" data-locale="en"><svg width="20" fill="#5ccbcb" height="100%" viewBox="0 0 20 20" preserveAspectRatio="xMidYMid meet" xmlns="http://www.w3.org/2000/svg" class="SelectLanguage_globe__vXWXl" xmlns:xlink="http://www.w3.org/1999/xlink"><path d="M10.5859 19.1171V15.4726H13.5789C13.4031 15.9343 13.2063 16.3726 12.9836 16.7781C12.2969 18.0367 11.4461 18.857 10.5859 19.1171ZM0.8125 10.5859C0.896875 11.9218 1.26719 13.1781 1.8625 14.3007H4.83438C4.54844 13.157 4.37266 11.9054 4.33281 10.5859H0.8125ZM2.60078 15.4726C3.67188 16.9164 5.15313 18.0367 6.87344 18.6601C6.19141 17.8328 5.6125 16.7453 5.17891 15.4726H2.60078ZM7.01406 16.7781C7.70312 18.0367 8.55156 18.857 9.41406 19.1195V15.4726H6.42109C6.59688 15.9343 6.79375 16.3726 7.01406 16.7781ZM1.82969 5.76246C1.25313 6.86871 0.896875 8.10387 0.8125 9.41403H4.33281C4.37266 8.11793 4.54141 6.88746 4.81797 5.76246H1.82969ZM2.55625 4.59059H5.15547C5.59141 3.28981 6.17734 2.18121 6.87109 1.33981C5.12734 1.97262 3.62969 3.11637 2.55625 4.59059ZM12.9859 3.22184C12.2969 1.96325 11.4484 1.14293 10.5859 0.880432V4.58824H13.6047C13.4219 4.10309 13.2156 3.64606 12.9859 3.22184ZM17.4438 4.59059C16.3703 3.11637 14.8703 1.97262 13.1266 1.33981C13.8203 2.17887 14.4086 3.28981 14.8422 4.59059H17.4438ZM17.3992 15.4726H14.8211C14.3875 16.7453 13.8086 17.8328 13.1266 18.6601C14.8469 18.0367 16.3305 16.9164 17.3992 15.4726ZM19.1875 9.41403C19.1055 8.10387 18.7469 6.86871 18.1703 5.76246H15.182C15.4586 6.88746 15.6273 8.11793 15.6672 9.41403H19.1875ZM14.493 10.5859H10.5859V14.3007H13.9586C14.268 13.157 14.4508 11.8984 14.493 10.5859ZM13.975 5.76246H10.5859V9.41403H14.493C14.4531 8.12496 14.275 6.88981 13.975 5.76246ZM6.04141 14.3007H9.41406V10.5859H5.50703C5.54922 11.8984 5.73203 13.157 6.04141 14.3007ZM15.1656 14.3007H18.1375C18.7328 13.1781 19.1031 11.9218 19.1875 10.5859H15.6672C15.6273 11.9054 15.4516 13.157 15.1656 14.3007ZM5.50703 9.41403H9.41406V5.76246H6.025C5.725 6.88981 5.54688 8.12496 5.50703 9.41403ZM7.01406 3.22184C6.78203 3.64606 6.57578 4.10309 6.39531 4.59059H9.41406V0.882776C8.55391 1.14293 7.70312 1.96325 7.01406 3.22184Z" fill="#5ccbcb"></path></svg><div class="SelectLanguage_selected__cjbJs"><span>en</span></div><ul data-anchor="top"><li class="SelectLanguage_active__-gdeR SelectLanguage_fpTheme__CkFYu"><a href="/news/view/behind-the-scenes-of-a-long-eve-online-downtime" hrefLang="en" data-id="52d81be"><span class="SelectLanguage_lang_text__Sn7Ce">English</span><span class="SelectLanguage_lang_code__VxoBD">en</span></a></li><li class="SelectLanguage_fpTheme__CkFYu"><a href="/fr/news/view/behind-the-scenes-of-a-long-eve-online-downtime" hrefLang="fr" data-id="aacdecde"><span class="SelectLanguage_lang_text__Sn7Ce">Français</span><span class="SelectLanguage_lang_code__VxoBD">fr</span></a></li><li class="SelectLanguage_fpTheme__CkFYu"><a href="/de/news/view/behind-the-scenes-of-a-long-eve-online-downtime" hrefLang="de" data-id="dcd2b2be"><span class="SelectLanguage_lang_text__Sn7Ce">Deutsch</span><span class="SelectLanguage_lang_code__VxoBD">de</span></a></li><li class="SelectLanguage_fpTheme__CkFYu"><a href="/ru/news/view/behind-the-scenes-of-a-long-eve-online-downtime" hrefLang="ru" data-id="a31bbf7e"><span class="SelectLanguage_lang_text__Sn7Ce">Русский</span><span class="SelectLanguage_lang_code__VxoBD">ru</span></a></li><li class="SelectLanguage_fpTheme__CkFYu"><a href="/ja/news/view/behind-the-scenes-of-a-long-eve-online-downtime" hrefLang="ja" data-id="235332fe"><span class="SelectLanguage_lang_text__Sn7Ce">日本語</span><span class="SelectLanguage_lang_code__VxoBD">ja</span></a></li><li class="SelectLanguage_fpTheme__CkFYu"><a href="/ko/news/view/behind-the-scenes-of-a-long-eve-online-downtime" hrefLang="ko" data-id="6aad795e"><span class="SelectLanguage_lang_text__Sn7Ce">한국어</span><span class="SelectLanguage_lang_code__VxoBD">ko</span></a></li><li class="SelectLanguage_fpTheme__CkFYu"><a href="/es/news/view/behind-the-scenes-of-a-long-eve-online-downtime" hrefLang="es" data-id="377af71e"><span class="SelectLanguage_lang_text__Sn7Ce">Español</span><span class="SelectLanguage_lang_code__VxoBD">es</span></a></li><li class="SelectLanguage_fpTheme__CkFYu"><a href="/zh/news/view/behind-the-scenes-of-a-long-eve-online-downtime" hrefLang="zh" data-id="e72eb29e"><span class="SelectLanguage_lang_text__Sn7Ce">简体中文</span><span class="SelectLanguage_lang_code__VxoBD">zh</span></a></li></ul></div></div><div data-simplefooter="remove" class="Footer_col__9Khe7"><ul>
<li>Play EVE Online</li>
<li><a href="/signup" target="_blank">Play for Free</a></li>
<li><a href="/download">Download the Game</a></li>
<li><a href="https://store.eveonline.com/#omega" target="_blank">Upgrade to Omega</a></li>
<li><a href="https://support.eveonline.com/hc/en-us/articles/5885219196828-System-Requirements" target="_blank">System Requirements</a></li>
<li><a href="/eve-academy">Starter Guide for Beginners</a></li>
<li><a href="/eve-academy/excel-add-in">EVE Online add-in for Excel</a></li>
</ul></div><div data-simplefooter="remove" class="Footer_col__9Khe7"><ul><li>Current Players</li><li><a href="https://secure.eveonline.com/account" target="_blank" rel="noopener noreferrer">Account Management</a></li><li><a href="https://support.eveonline.com/" target="_blank" rel="noopener noreferrer">Support</a></li><li><a href="https://community.eveonline.com/" target="_blank" rel="noopener noreferrer">Community</a></li><li><a href="https://forums.eveonline.com/" target="_blank" rel="noopener noreferrer">Forums</a></li><li><a href="https://developers.eveonline.com/" target="_blank" rel="noopener noreferrer">Developers</a></li><li><a href="/recruit" data-id="a4c097ec">Recruit a Friend</a></li>
<li><a href="/return">Return to EVE</a></li>
<li><a href="/partners">Partnership Program</a></li>
<li><a href="/plex-for-good">PLEX For Good</a></li>
</ul></div><div data-simplefooter="remove" class="Footer_col__9Khe7"><ul><li>Worlds of EVE</li><li><a href="https://universe.eveonline.com/" target="_blank" rel="noopener noreferrer">EVE Universe</a></li><li><a href="https://fanfest.eveonline.com/" target="_blank" rel="noopener noreferrer">EVE Fanfest</a></li><li><a href="https://eveonline-merchandise-store.myshopify.com" target="_blank" rel="noopener noreferrer">EVE Merchandise</a></li><li><a href="https://www.eveonline.com/discovery/" target="_blank" rel="noopener noreferrer">Project Discovery</a></li></ul></div><div data-simplefooter="keep" class="Footer_col__9Khe7"><div data-simplefooter="keep" class="Footer_col__9Khe7"><ul><li data-simplefooter="remove">Site Info</li><li data-simplefooter="remove"><a href="https://www.fenris.com/company" target="_blank" rel="noopener noreferrer">About Fenris</a></li><li><a href="https://support.eveonline.com/hc/en-us/articles/8414770561948-EVE-Online-Terms-of-Service" target="_blank" rel="noopener noreferrer">Terms of Service</a></li><li><a href="https://www.fenris.com/privacy-policy" target="_blank" rel="noopener noreferrer">Privacy Policy</a></li><li data-simplefooter="remove"><a href="https://www.eveonline.com/rss" target="_blank" rel="noopener noreferrer">RSS Feed</a></li></ul></div><ul class="Footer_cookieLink__gCl4A"><li><a style="cursor:pointer" href="#">Manage Cookies</a></li></ul></div></div><div id="rating_container"><div id="pegi_rating"><div class="Footer_row__rtpKa Footer_required__SoH29"><div class="Footer_copyright__irBHz Footer_copyrightText__SNkO-"><p>EVE Online® and Fenris Creations™ and all related logos and other elements are trademarks of Fenris Creations.</p>
<p>©2026 Fenris Creations. All rights reserved.</p></div><div class="Footer_rating__dQ8j1"><div class="Footer_pegi__hkr5j"><a id="pegi-info" href="http://www.pegi.info/" title="Visit the PEGI webpage"><div class="lazyload-wrapper "><div style="height:50px" class="lazyload-placeholder"></div></div></a></div></div></div></div></div></div></div></footer></div></div></div><script src="https://embed.twitch.tv/embed/v1.js" defer="defer"></script><script src="/static/js/runtime.5d2a5f82.js"></script><script src="/static/js/npm-lodash.95845f68.chunk.js"></script><script src="/static/js/npm-mantine.c4278e93.chunk.js"></script><script src="/static/js/npm-ccpgames.b82e6891.chunk.js"></script><script src="/static/js/npm-date-fns.3f9689aa.chunk.js"></script><script src="/static/js/npm-sentry.02e9b96f.chunk.js"></script><script src="/static/js/npm-lodash-es.247c47c6.chunk.js"></script><script src="/static/js/npm-es-abstract.e78b559c.chunk.js"></script><script src="/static/js/npm-fortawesome.171b8386.chunk.js"></script><script src="/static/js/npm-apollo.88e1cd85.chunk.js"></script><script src="/static/js/npm-axios.76bb4275.chunk.js"></script><script src="/static/js/npm-react-spinners.86f3431d.chunk.js"></script><script src="/static/js/npm-ramda.d2f6fbae.chunk.js"></script><script src="/static/js/npm-mdast-util-to-hast.f18bd1d9.chunk.js"></script><script src="/static/js/npm-react-redux.53fc4f0d.chunk.js"></script><script src="/static/js/npm-sentry-internal.836a6023.chunk.js"></script><script src="/static/js/npm-yup.73545278.chunk.js"></script><script src="/static/js/npm-browserify-sign.c64a1fd5.chunk.js"></script><script src="/static/js/npm-react-query.4bf798ee.chunk.js"></script><script src="/static/js/npm-react-select-search.89327f01.chunk.js"></script><script src="/static/js/npm-emotion.13989c97.chunk.js"></script><script src="/static/js/npm-hast-util-raw.1b41e856.chunk.js"></script><script src="/static/js/npm-redux-first-router.ba0f78ed.chunk.js"></script><script src="/static/js/npm-graphql.d5afdced.chunk.js"></script><script src="/static/js/npm-micromark-core-commonmark.eb770a5a.chunk.js"></script><script src="/static/js/npm-pbkdf2.1292c8a6.chunk.js"></script><script src="/static/js/npm-parse-asn1.07b4f78b.chunk.js"></script><script src="/static/js/npm-property-information.89ec3ca6.chunk.js"></script><script src="/static/js/npm-browserify-aes.9ec6171d.chunk.js"></script><script src="/static/js/npm-elliptic.f11b1682.chunk.js"></script><script src="/static/js/npm-hash-base.637d28d3.chunk.js"></script><script src="/static/js/npm-react-scroll.e260e812.chunk.js"></script><script src="/static/js/npm-react-image-lightbox.8e5f5429.chunk.js"></script><script src="/static/js/npm-readable-stream.c477a26d.chunk.js"></script><script src="/static/js/npm-asn1.js.776d4f75.chunk.js"></script><script src="/static/js/npm-hash.js.93e37078.chunk.js"></script><script src="/static/js/npm-radix-ui.f81be333.chunk.js"></script><script src="/static/js/npm-react-modal.90cfada5.chunk.js"></script><script src="/static/js/npm-contentful.862c3276.chunk.js"></script><script src="/static/js/npm-react-slick.2a67dea1.chunk.js"></script><script src="/static/js/npm-micromark.9c5bc720.chunk.js"></script><script src="/static/js/npm-react-dom.c8230956.chunk.js"></script><script src="/static/js/npm-react-localize-redux.ee78414f.chunk.js"></script><script src="/static/js/npm-rudy-history.3c9948fe.chunk.js"></script><script src="/static/js/npm-qs.e60238ff.chunk.js"></script><script src="/static/js/npm-react-transition-group.62459257.chunk.js"></script><script src="/static/js/npm-vfile.c294201d.chunk.js"></script><script src="/static/js/npm-floating-ui.a8bcfdad.chunk.js"></script><script src="/static/js/npm-react-tooltip.cd30ddbf.chunk.js"></script><script src="/static/js/npm-optimism.ec1774c3.chunk.js"></script><script src="/static/js/npm-styled-components.f5a76eec.chunk.js"></script><script src="/static/js/npm-browserify-rsa.3fed19ea.chunk.js"></script><script src="/static/js/npm-buffer.1886d793.chunk.js"></script><script src="/static/js/npm-es5-shim.8a1dbb37.chunk.js"></script><script src="/static/js/npm-formik.2e5f5375.chunk.js"></script><script src="/static/js/npm-react-device-detect.527d211a.chunk.js"></script><script src="/static/js/npm-react-helmet-async.8b3f9017.chunk.js"></script><script src="/static/js/npm-bn.js.f1197972.chunk.js"></script><script src="/static/js/npm-character-entities.1b7baca5.chunk.js"></script><script src="/static/js/npm-es6-shim.c3de8a62.chunk.js"></script><script src="/static/js/npm-fuse.js.f146089b.chunk.js"></script><script src="/static/js/npm-launchdarkly-js-client-sdk.25b3ca9f.chunk.js"></script><script src="/static/js/npm-mdast-util-from-markdown.f5b7a0ed.chunk.js"></script><script src="/static/js/npm-platform.7c507ae8.chunk.js"></script><script src="/static/js/npm-react-visibility-sensor.6f4818cb.chunk.js"></script><script src="/static/js/npm-react-zoom-pan-pinch.c49bd4bd.chunk.js"></script><script src="/static/js/npm-redux.eccfc802.chunk.js"></script><script src="/static/js/npm-resize-observer-polyfill.4e82588c.chunk.js"></script><script src="/static/js/npm-sjcl.524ae378.chunk.js"></script><script src="/static/js/npm-ua-parser-js.63b6f0f9.chunk.js"></script><script src="/static/js/vendors~main.9fcc19bf.chunk.js"></script><script src="/static/js/main.545433fb.chunk.js"></script></body></html>