Files
nexus/sreweekly/articles/215/06-hosted-pools-availability-degradation.html
2026-09-12 17:23:01 +08:00

32 lines
60 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" dir="ltr" lang="en-US" >
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="msapplication-config" content="none" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="robots" content="none" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link id="page-icon" rel="shortcut icon" href="https://cdn.vsassets.io/content/icons/favicon.ico" />
<meta name="referrer" content="strict-origin-when-cross-origin" /><script data-description="contentInit" nonce="s2/kszieM8kz7XUSWF0tmA==">window.__IS_BUNDLING_ENABLED = true;window.__IS_CDN_ENABLED = true;window.$DEBUG = false;window.__scriptType = "ES2017";window.__moduleNamespaces = ["create-react-class","prop-types","react","react-dom","react-dom-factories","react-transition-group","tslib","VSS/Core","mousetrap","VSS/Platform","@azure/msal-browser","msal-browser","VSS/Msal","VSSUI","VSS/Legacy","VSS/Features/PlatformUI","markdown-it","markdown-it-emoji","xss","js-yaml","markdown-it-container","markdown-it-imsize","markdown-it-task-lists","VSS/Features/Markdown","Status/Core","Status/LiveSiteEventPage","Status/Msal"];window.__moduleAliases = {"react-dom/client":"react-dom"};window.__asyncModules = [];window.__extensionsStaticRoot = "https://cdn.vsassets.io/ext/";window.__extensionsLocalStaticRoot = "/_static/_ext/";window.__useEntraToken = true;window.__isCookieless = true;window.__redirectToSignout = false;window.resourceLoadStrategy = 'scriptsFirst';var preTraceServiceStartErrors = []; var onPreTraceServiceError = function (event) { preTraceServiceStartErrors.push(event) }; window.addEventListener('error', onPreTraceServiceError);var previousResourceLoadCompletion = window.performance.now(); var resourceLoadTimes = {}; window.logResourceLoad = function (id) {var now = window.performance.now(); resourceLoadTimes[id] = { current: now, previous: previousResourceLoadCompletion }; previousResourceLoadCompletion = now; }; </script><script data-description="disableCdn" nonce="s2/kszieM8kz7XUSWF0tmA==">function cdnReload(fromTimeout) {
document.cookie = 'TFS-CDN=disabled;max-age=28800;path=/;secure';
document.cookie = 'TFS-CDNTRACE=report;path=/;secure';
if(fromTimeout){document.cookie = 'TFS-CDNTIMEOUT=report;path=/;secure'; }
document.location.reload(true);
}
var cdnTimeout = setTimeout(function () { cdnReload(true); }, 10000);</script><script data-description="trackLoadedBundles" nonce="s2/kszieM8kz7XUSWF0tmA=="> var loadedScripts = {}; document.addEventListener('scriptLoaded', function (e) { if (e && e.detail) { loadedScripts[e.detail.id] = true;} });</script><script src="https://cdn.vsassets.io/ext/ms.vss-web/core-content/ms.vss-web.core-content.es6.9bQ5_t.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="196857" data-clientid="ms.vss-web.core-content/core-content/ms.vss-web.core-content.es6.min.js" data-sourcecontribution="ms.vss-web.core-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-web.core-content/core-content/ms.vss-web.core-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-web/platform-content/ms.vss-web.platform-content.es6.jYYCy0.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="153235" data-clientid="ms.vss-web.platform-content/platform-content/ms.vss-web.platform-content.es6.min.js" data-sourcecontribution="ms.vss-web.platform-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-web.platform-content/platform-content/ms.vss-web.platform-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-web/msal-browser/ms.vss-web.msal-browser.es6.s4Fhb4.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="244636" data-clientid="ms.vss-web.msal-browser/msal-browser/ms.vss-web.msal-browser.es6.min.js" data-sourcecontribution="ms.vss-web.msal-browser" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-web.msal-browser/msal-browser/ms.vss-web.msal-browser.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-web/msal-content/ms.vss-web.msal-content.es6.wy_rif.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="33102" data-clientid="ms.vss-web.msal-content/msal-content/ms.vss-web.msal-content.es6.min.js" data-sourcecontribution="ms.vss-web.msal-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-web.msal-content/msal-content/ms.vss-web.msal-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-features/ui-core-content/ms.vss-features.ui-core-content.es6.g7itcZ.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="81530" data-clientid="ms.vss-features.ui-core-content/ui-core-content/ms.vss-features.ui-core-content.es6.min.js" data-sourcecontribution="ms.vss-features.ui-core-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-core-content/ui-core-content/ms.vss-features.ui-core-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-features/ui-spinner-content/ms.vss-features.ui-spinner-content.es6.KJ2rk5.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="1301" data-clientid="ms.vss-features.ui-spinner-content/ui-spinner-content/ms.vss-features.ui-spinner-content.es6.min.js" data-sourcecontribution="ms.vss-features.ui-spinner-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-spinner-content/ui-spinner-content/ms.vss-features.ui-spinner-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-features/ui-widgets-content/ms.vss-features.ui-widgets-content.es6.mF8Cpk.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="161364" data-clientid="ms.vss-features.ui-widgets-content/ui-widgets-content/ms.vss-features.ui-widgets-content.es6.min.js" data-sourcecontribution="ms.vss-features.ui-widgets-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-widgets-content/ui-widgets-content/ms.vss-features.ui-widgets-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-web/legacy-content/ms.vss-web.legacy-content.es6.SHdynN.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="7302" data-clientid="ms.vss-web.legacy-content/legacy-content/ms.vss-web.legacy-content.es6.min.js" data-sourcecontribution="ms.vss-web.legacy-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-web.legacy-content/legacy-content/ms.vss-web.legacy-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-features/platform-ui-content/ms.vss-features.platform-ui-content.es6.GHwneN.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="23801" data-clientid="ms.vss-features.platform-ui-content/platform-ui-content/ms.vss-features.platform-ui-content.es6.min.js" data-sourcecontribution="ms.vss-features.platform-ui-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.platform-ui-content/platform-ui-content/ms.vss-features.platform-ui-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-features/ui-input-content/ms.vss-features.ui-input-content.es6.Er0kZm.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="14160" data-clientid="ms.vss-features.ui-input-content/ui-input-content/ms.vss-features.ui-input-content.es6.min.js" data-sourcecontribution="ms.vss-features.ui-input-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-input-content/ui-input-content/ms.vss-features.ui-input-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-features/ui-page-content/ms.vss-features.ui-page-content.es6.SzxLJL.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="24146" data-clientid="ms.vss-features.ui-page-content/ui-page-content/ms.vss-features.ui-page-content.es6.min.js" data-sourcecontribution="ms.vss-features.ui-page-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-page-content/ui-page-content/ms.vss-features.ui-page-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-features/ui-layer-content/ms.vss-features.ui-layer-content.es6.4pRjuB.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="19603" data-clientid="ms.vss-features.ui-layer-content/ui-layer-content/ms.vss-features.ui-layer-content.es6.min.js" data-sourcecontribution="ms.vss-features.ui-layer-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-layer-content/ui-layer-content/ms.vss-features.ui-layer-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-features/ui-breadcrumb-content/ms.vss-features.ui-breadcrumb-content.es6.K6q0fx.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="4366" data-clientid="ms.vss-features.ui-breadcrumb-content/ui-breadcrumb-content/ms.vss-features.ui-breadcrumb-content.es6.min.js" data-sourcecontribution="ms.vss-features.ui-breadcrumb-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-breadcrumb-content/ui-breadcrumb-content/ms.vss-features.ui-breadcrumb-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-features/ui-card-content/ms.vss-features.ui-card-content.es6.ZoDWZB.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="3379" data-clientid="ms.vss-features.ui-card-content/ui-card-content/ms.vss-features.ui-card-content.es6.min.js" data-sourcecontribution="ms.vss-features.ui-card-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-card-content/ui-card-content/ms.vss-features.ui-card-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-features/ui-tree-content/ms.vss-features.ui-tree-content.es6.eDUFoi.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="16051" data-clientid="ms.vss-features.ui-tree-content/ui-tree-content/ms.vss-features.ui-tree-content.es6.min.js" data-sourcecontribution="ms.vss-features.ui-tree-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-tree-content/ui-tree-content/ms.vss-features.ui-tree-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-features/ui-dropdown-content/ms.vss-features.ui-dropdown-content.es6.i6hcl5.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="62431" data-clientid="ms.vss-features.ui-dropdown-content/ui-dropdown-content/ms.vss-features.ui-dropdown-content.es6.min.js" data-sourcecontribution="ms.vss-features.ui-dropdown-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-dropdown-content/ui-dropdown-content/ms.vss-features.ui-dropdown-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-features/ui-messagebar-content/ms.vss-features.ui-messagebar-content.es6._lgV0b.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="3606" data-clientid="ms.vss-features.ui-messagebar-content/ui-messagebar-content/ms.vss-features.ui-messagebar-content.es6.min.js" data-sourcecontribution="ms.vss-features.ui-messagebar-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-messagebar-content/ui-messagebar-content/ms.vss-features.ui-messagebar-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-features/ui-progress-bar-content/ms.vss-features.ui-progress-bar-content.es6.BLjpRm.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="3256" data-clientid="ms.vss-features.ui-progress-bar-content/ui-progress-bar-content/ms.vss-features.ui-progress-bar-content.es6.min.js" data-sourcecontribution="ms.vss-features.ui-progress-bar-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-progress-bar-content/ui-progress-bar-content/ms.vss-features.ui-progress-bar-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-features/ui-filterbar-content/ms.vss-features.ui-filterbar-content.es6.4N_Clh.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="9071" data-clientid="ms.vss-features.ui-filterbar-content/ui-filterbar-content/ms.vss-features.ui-filterbar-content.es6.min.js" data-sourcecontribution="ms.vss-features.ui-filterbar-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-filterbar-content/ui-filterbar-content/ms.vss-features.ui-filterbar-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-features/ui-status-content/ms.vss-features.ui-status-content.es6.vpW5td.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="14615" data-clientid="ms.vss-features.ui-status-content/ui-status-content/ms.vss-features.ui-status-content.es6.min.js" data-sourcecontribution="ms.vss-features.ui-status-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-status-content/ui-status-content/ms.vss-features.ui-status-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-web/markdown-it-content/ms.vss-web.markdown-it-content.es6.EXkGYi.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="144396" data-clientid="ms.vss-web.markdown-it-content/markdown-it-content/ms.vss-web.markdown-it-content.es6.min.js" data-sourcecontribution="ms.vss-web.markdown-it-content" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-web.markdown-it-content/markdown-it-content/ms.vss-web.markdown-it-content.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-web/xss/ms.vss-web.xss.es6.CDup_P.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="19625" data-clientid="ms.vss-web.xss/xss/ms.vss-web.xss.es6.min.js" data-sourcecontribution="ms.vss-web.xss" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-web.xss/xss/ms.vss-web.xss.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-features/markdown/ms.vss-features.markdown.es6.v8xh1Z.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="126428" data-clientid="ms.vss-features.markdown/markdown/ms.vss-features.markdown.es6.min.js" data-sourcecontribution="ms.vss-features.markdown" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.markdown/markdown/ms.vss-features.markdown.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-status-web/core/ms.vss-status-web.core.es6.r3cikU.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="104670" data-clientid="ms.vss-status-web.core/core/ms.vss-status-web.core.es6.min.js" data-sourcecontribution="ms.vss-status-web.core" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-status-web.core/core/ms.vss-status-web.core.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-status-web/live-site-event-page/ms.vss-status-web.live-site-event-page.es6.mN6YhD.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="6693" data-clientid="ms.vss-status-web.live-site-event-page/live-site-event-page/ms.vss-status-web.live-site-event-page.es6.min.js" data-sourcecontribution="ms.vss-status-web.live-site-event-page" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-status-web.live-site-event-page/live-site-event-page/ms.vss-status-web.live-site-event-page.es6.min.js');</script><script src="https://cdn.vsassets.io/ext/ms.vss-status-web/msal/ms.vss-status-web.msal.es6.1SX21I.min.js" nonce="s2/kszieM8kz7XUSWF0tmA==" data-contentlength="789" data-clientid="ms.vss-status-web.msal/msal/ms.vss-status-web.msal.es6.min.js" data-sourcecontribution="ms.vss-status-web.msal" data-loadingcontent="true"></script><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-status-web.msal/msal/ms.vss-status-web.msal.es6.min.js');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-web/core-content/ms.vss-web.core-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="13803" data-clientid="ms.vss-web.core-content/core-content/ms.vss-web.core-content.min.css" data-sourcecontribution="ms.vss-web.core-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-web.core-content/core-content/ms.vss-web.core-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-web/platform-content/ms.vss-web.platform-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="4114" data-clientid="ms.vss-web.platform-content/platform-content/ms.vss-web.platform-content.min.css" data-sourcecontribution="ms.vss-web.platform-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-web.platform-content/platform-content/ms.vss-web.platform-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-features/ui-core-content/ms.vss-features.ui-core-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="62808" data-clientid="ms.vss-features.ui-core-content/ui-core-content/ms.vss-features.ui-core-content.min.css" data-sourcecontribution="ms.vss-features.ui-core-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-core-content/ui-core-content/ms.vss-features.ui-core-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-features/ui-spinner-content/ms.vss-features.ui-spinner-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="846" data-clientid="ms.vss-features.ui-spinner-content/ui-spinner-content/ms.vss-features.ui-spinner-content.min.css" data-sourcecontribution="ms.vss-features.ui-spinner-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-spinner-content/ui-spinner-content/ms.vss-features.ui-spinner-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-features/ui-widgets-content/ms.vss-features.ui-widgets-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="63950" data-clientid="ms.vss-features.ui-widgets-content/ui-widgets-content/ms.vss-features.ui-widgets-content.min.css" data-sourcecontribution="ms.vss-features.ui-widgets-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-widgets-content/ui-widgets-content/ms.vss-features.ui-widgets-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-features/platform-ui-content/ms.vss-features.platform-ui-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="303" data-clientid="ms.vss-features.platform-ui-content/platform-ui-content/ms.vss-features.platform-ui-content.min.css" data-sourcecontribution="ms.vss-features.platform-ui-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.platform-ui-content/platform-ui-content/ms.vss-features.platform-ui-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-features/ui-input-content/ms.vss-features.ui-input-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="9539" data-clientid="ms.vss-features.ui-input-content/ui-input-content/ms.vss-features.ui-input-content.min.css" data-sourcecontribution="ms.vss-features.ui-input-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-input-content/ui-input-content/ms.vss-features.ui-input-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-features/ui-page-content/ms.vss-features.ui-page-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="15193" data-clientid="ms.vss-features.ui-page-content/ui-page-content/ms.vss-features.ui-page-content.min.css" data-sourcecontribution="ms.vss-features.ui-page-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-page-content/ui-page-content/ms.vss-features.ui-page-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-features/ui-layer-content/ms.vss-features.ui-layer-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="6906" data-clientid="ms.vss-features.ui-layer-content/ui-layer-content/ms.vss-features.ui-layer-content.min.css" data-sourcecontribution="ms.vss-features.ui-layer-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-layer-content/ui-layer-content/ms.vss-features.ui-layer-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-features/ui-breadcrumb-content/ms.vss-features.ui-breadcrumb-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="2462" data-clientid="ms.vss-features.ui-breadcrumb-content/ui-breadcrumb-content/ms.vss-features.ui-breadcrumb-content.min.css" data-sourcecontribution="ms.vss-features.ui-breadcrumb-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-breadcrumb-content/ui-breadcrumb-content/ms.vss-features.ui-breadcrumb-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-features/ui-card-content/ms.vss-features.ui-card-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="1227" data-clientid="ms.vss-features.ui-card-content/ui-card-content/ms.vss-features.ui-card-content.min.css" data-sourcecontribution="ms.vss-features.ui-card-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-card-content/ui-card-content/ms.vss-features.ui-card-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-features/ui-tree-content/ms.vss-features.ui-tree-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="482" data-clientid="ms.vss-features.ui-tree-content/ui-tree-content/ms.vss-features.ui-tree-content.min.css" data-sourcecontribution="ms.vss-features.ui-tree-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-tree-content/ui-tree-content/ms.vss-features.ui-tree-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-features/ui-dropdown-content/ms.vss-features.ui-dropdown-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="9506" data-clientid="ms.vss-features.ui-dropdown-content/ui-dropdown-content/ms.vss-features.ui-dropdown-content.min.css" data-sourcecontribution="ms.vss-features.ui-dropdown-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-dropdown-content/ui-dropdown-content/ms.vss-features.ui-dropdown-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-features/ui-messagebar-content/ms.vss-features.ui-messagebar-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="2607" data-clientid="ms.vss-features.ui-messagebar-content/ui-messagebar-content/ms.vss-features.ui-messagebar-content.min.css" data-sourcecontribution="ms.vss-features.ui-messagebar-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-messagebar-content/ui-messagebar-content/ms.vss-features.ui-messagebar-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-features/ui-progress-bar-content/ms.vss-features.ui-progress-bar-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="988" data-clientid="ms.vss-features.ui-progress-bar-content/ui-progress-bar-content/ms.vss-features.ui-progress-bar-content.min.css" data-sourcecontribution="ms.vss-features.ui-progress-bar-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-progress-bar-content/ui-progress-bar-content/ms.vss-features.ui-progress-bar-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-features/ui-filterbar-content/ms.vss-features.ui-filterbar-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="5206" data-clientid="ms.vss-features.ui-filterbar-content/ui-filterbar-content/ms.vss-features.ui-filterbar-content.min.css" data-sourcecontribution="ms.vss-features.ui-filterbar-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-filterbar-content/ui-filterbar-content/ms.vss-features.ui-filterbar-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-features/ui-status-content/ms.vss-features.ui-status-content.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="2670" data-clientid="ms.vss-features.ui-status-content/ui-status-content/ms.vss-features.ui-status-content.min.css" data-sourcecontribution="ms.vss-features.ui-status-content" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.ui-status-content/ui-status-content/ms.vss-features.ui-status-content.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-features/markdown/ms.vss-features.markdown.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="8685" data-clientid="ms.vss-features.markdown/markdown/ms.vss-features.markdown.min.css" data-sourcecontribution="ms.vss-features.markdown" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-features.markdown/markdown/ms.vss-features.markdown.min.css');</script><link href="https://cdn.vsassets.io/v/M279_20260901.13/_ext/ms.vss-status-web/core/ms.vss-status-web.core.min.css" rel="stylesheet" crossorigin="anonymous" data-contentlength="9266" data-clientid="ms.vss-status-web.core/core/ms.vss-status-web.core.min.css" data-sourcecontribution="ms.vss-status-web.core" data-loadingcontent="true" /><script data-description="logResourceLoad" nonce="s2/kszieM8kz7XUSWF0tmA==">window.logResourceLoad('ms.vss-status-web.core/core/ms.vss-status-web.core.min.css');</script><script data-description="cdnFallback" nonce="s2/kszieM8kz7XUSWF0tmA==">if (!window.LWL) { cdnReload(false); }</script><script data-description="clearCdnTimeout" nonce="s2/kszieM8kz7XUSWF0tmA==">window.clearTimeout(cdnTimeout);</script><script data-description="fireLoadEvents" nonce="s2/kszieM8kz7XUSWF0tmA==">var contentElements = document.querySelectorAll("[data-loadingcontent=true]"); for (var i=0; i<contentElements.length; i++) { var loadTimes = resourceLoadTimes[contentElements[i].getAttribute('data-clientid')]; window.__contentLoaded(contentElements[i], true, loadTimes && loadTimes.current, loadTimes && loadTimes.previous); }</script>
<script id="dataProviders" type="application/json">{"data":{"ms.vss-web.component-data":{"header":[{"componentId":"ms.vss-status-web.deployment-level-public-page-header-component","componentType":"publicHeader"}],"footer":[{"componentId":"ms.vss-status-web.deployment-level-public-page-footer-component","componentType":"publicFooter"}],"content":[{"componentId":"ms.vss-status-web.public-live-site-event-page-view-component","componentType":"publicLiveSiteEventPage","componentProperties":{"pageTitle":"Azure DevOps - Event"}}],"page":[{"componentId":"ms.vss-status-web.deployment-level-page-component","componentType":"layout","componentProperties":{"className":"flex-grow","orientation":"column","components":[{"componentRegion":"header","componentProperties":{"className":"header","role":"banner"}},{"componentType":"layout","componentProperties":{"className":"flex-grow","orientation":"column","components":[{"componentRegion":"content","componentProperties":{"className":"flex-column flex-grow content","role":"main","id":"skip-to-main-content"}}]}},{"componentRegion":"footer"}]}}]},"ms.vss-web.page-data":{"esNextSupported":true,"hostId":"89e99151-66bb-d9ae-a3dc-a9caa390d5fa","hostType":3,"hostName":"TEAM FOUNDATION","hostPath":"/","serviceInstanceType":"00000060-0000-8888-8000-000000000000","isBundlingEnabled":true,"isCdnEnabled":true,"isCdnAvailable":true,"isHosted":true,"isProduction":true,"isDebugMode":false,"isDevfabric":false,"isSslOnly":true,"isTracePointCollectionEnabled":false,"scriptType":"ES2017","serviceVersion":"Dev20.M279.1 (AzureDevOps_M279_20260901.13)","sessionId":"0d86a024-9e37-4a91-9919-73efdb48fd00","styleType":"stylesheet","user":{"name":"Anonymous","displayName":"Anonymous","url":"https://vssps.dev.azure.com/_apis/Identities/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","id":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","uniqueName":null,"imageUrl":null},"webPlatformVersion":2},"ms.vss-web.shared-data":null,"ms.vss-web.theme-data":{"requestedThemeId":"ms.vss-status-web.vsts-theme-status"},"ms.vss-web.user-claims-data":{"anonymous":true},"ms.vss-status-web.public-status-metadata-data-provider":{"services":[{"id":"Core services"},{"id":"Boards"},{"id":"Repos"},{"id":"Pipelines"},{"id":"Test Plans"},{"id":"Artifacts"},{"id":"Other services"}],"geographies":[{"id":"US","name":"United States"},{"id":"CA","name":"Canada"},{"id":"BR","name":"Brazil"},{"id":"EU","name":"Europe"},{"id":"UK","name":"United Kingdom"},{"id":"APAC","name":"Asia Pacific"},{"id":"AU","name":"Australia"},{"id":"IN","name":"India"}]},"ms.vss-status-web.public-live-site-event-data-provider":{"liveSiteEvent":{"id":181122901,"severity":2,"state":2,"startTime":"\/Date(1585043477027)\/","endTime":"\/Date(1585083288627)\/","nextUpdateTime":"\/Date(1586301433453)\/","title":"Hosted Pools Availability Degradation","impact":[{"serviceId":"Pipelines","scopeType":1,"scopeName":"Europe"}],"logs":[{"id":2310,"type":2,"scopeType":1,"creationDateTime":"\/Date(1586298339523)\/","lastUpdatedDateTime":"\/Date(1586298339523)\/","author":{"id":"9ac156b6-7efa-6c7d-abbc-123a989f9709","displayName":"Chad Kimes"},"description":"\u003cp\u003eFrom March 24th - 26th, 2020 many customers in Europe and the United Kingdom experienced delays in their builds and releases targeting our hosted Windows and Linux agents. This incident was caused by VM capacity constraints arising from the global health pandemic that led to increased machine reimage times and then increased wait times for available agents. Many customers experienced significant delays in their pipelines over multiple days. We sincerely apologize for the impact of this incident.\u003c/p\u003e\n\u003ch1\u003eCustomer Impact\u003c/h1\u003e\n\u003cp\u003eThe period of highest impact was 09:00-18:00 UTC in Europe on March 24th, with additional disruptions from 07:00-17:00 UTC on March 25th, and 07:00-16:00 UTC on March 26th. These periods correspond to peak usage in the affected regions. Customers experienced delays of 21 minutes on average and as high as 9 hours. Across these three periods, 6,136 organizations experienced delays. Here is a graph showing the number of impacted organizations during this period:\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\u0022https://status.dev.azure.com/_apis/status/livesiteevents/181122901/attachments/7138fdf0-031a-41f7-a535-85d1a8f52309.png\u0022 alt=\u0022Chart depicting data described in this report.\u0022 /\u003e\u003c/p\u003e\n\u003ch1\u003eWhat Happened\u003c/h1\u003e\n\u003cp\u003eAs Azure has documented at \u003ca href=\u0022http://aka.ms/cloudCovidResponseFAQ\u0022 rel=\u0022nofollow\u0022\u003ehttp://aka.ms/cloudCovidResponseFAQ\u003c/a\u003e, a significant surge in use has led to deployment success rates for some compute resource types falling below their normal rates in a number of geographies, including Europe and the United Kingdom. For each Azure Pipelines job that is run, the \u003ca href=\u0022https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted\u0022 rel=\u0022nofollow\u0022\u003ehosted agent pool\u003c/a\u003e allocates a fresh virtual machine based on the Azure Pipelines custom image - a process performed over 30,000 times per hour in Europe and the United Kingdom during business hours. This makes Azure Pipelines especially sensitive to increases in the compute allocation failure rate. While Azure Pipelines does retry allocation on failure, the increased failure rate causes our average time to spin up a new agent substantially. Here is a view of the number of minutes required for the reimaging step shown for the 95th percentile:\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\u0022https://status.dev.azure.com/_apis/status/livesiteevents/181122901/attachments/b8b11b4b-6a83-411a-bd65-5ce258a58d8d.png\u0022 alt=\u0022Chart depicting data described in this report.\u0022 /\u003e\u003c/p\u003e\n\u003cp\u003eThis increase in the time to spin up new agents then led to the total number of agents in the hosted pool being too low to service the number of jobs, which then led to queueing and pipeline delays. Here’s a view of the wait time, in minutes, for the 95th percentile:\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\u0022https://status.dev.azure.com/_apis/status/livesiteevents/181122901/attachments/ec027afa-8835-4ba0-96d6-b189ba60e076.png\u0022 alt=\u0022Chart depicting data described in this report.\u0022 /\u003e\u003c/p\u003e\n\u003cp\u003eWe’ve been working on architectural changes to our hosted agent pools to mitigate the potential for issues of this type, and we sped up the deployment of these fixes in Europe and the United Kingdom as a result of these incidents. For Linux agents, the change was to use \u003ca href=\u0022https://docs.microsoft.com/en-us/azure/virtual-machines/linux/ephemeral-os-disks\u0022 rel=\u0022nofollow\u0022\u003eephemeral OS disks\u003c/a\u003e. This replaces VM allocation/deallocation operations per pipeline job with a re-image operation, thus avoiding allocations altogether. For Windows agents, ephemeral disks were not an option due to the size restrictions - our agents have lots of software, including Visual Studio Enterprise, which requires a large amount of disk storage. As a result, the change for Windows involves using larger Azure VMs and \u003ca href=\u0022https://docs.microsoft.com/en-us/azure/virtual-machines/windows/nested-virtualization\u0022 rel=\u0022nofollow\u0022\u003enested virtualization\u003c/a\u003e. The primary reason these incidents took so long to mitigate is that we were rolling these changes out much faster than we otherwise would have wanted to, and needed to ensure they didn’t introduce new issues or make things even worse.\u003c/p\u003e\n\u003ch1\u003eCommunication problems\u003c/h1\u003e\n\u003cp\u003eIn addition to the technical problems discussed above, our communication during this incident was also problematic. On the first day, when the impact was most severe, we didn’t acknowledge the incident for approximately five hours, which is substantially worse than our target of 10 minutes. This lack of acknowledgement leads to frustration and confusion, and we apologize for that as well.\u003c/p\u003e\n\u003cp\u003eThe problem here is that our live-site processes have a gap for these types of incidents. When incidents involve customer request failures or performance impacts, we have automated tooling that starts an incident and loops in both a DRI (designated responsible individual) and what we call a PIM (primary incident manager). The PIM is typically the person responsible for posting external communications acknowledging the incident.\u003c/p\u003e\n\u003cp\u003ePipeline delays are detected by different tooling, and the PIM is not currently paged for these types of incidents. As a result, while the DRI was hard at work understanding the technical issues and looking for potential mitigations, the PIM was still asleep. Only when the PIM joined the incident bridge at roughly the beginning of business hours in the Eastern United States was the incident finally acknowledged.\u003c/p\u003e\n\u003ch1\u003eNext Steps\u003c/h1\u003e\n\u003cp\u003eTo avoid incidents like this in the future, we are taking several steps:\u003c/p\u003e\n\u003col\u003e\n\u003cli\u003eWe are improving our monitoring and alerting by keeping a closer watch on compute allocation failure rates and agent reimage times. This supplements the monitoring and alerting we already had in place for agent availability and pipeline wait times.\u003c/li\u003e\n\u003cli\u003eIf we notice impending issues, we will quickly expand our use of ephemeral OS disks for Linux agents, nested virtualization for Windows agents, or both. In the absence of issues, we will continue rolling these changes out slowly, following our normal safe deployment practices.\u003c/li\u003e\n\u003cli\u003eWe are improving our live-site processes to ensure that initial communication of pipeline delay incidents happens on the same schedule as other incident types.\u003c/li\u003e\n\u003c/ol\u003e\n\u003cp\u003eAgain, we understand the impact that these types of pipeline delays have, especially in combination with a delay in incident communication, and we sincerely apologize for the incident.\u003c/p\u003e\n\u003cp\u003eSincerely,\u003cbr /\u003e\nChad Kimes\u003cbr /\u003e\nDirector of Engineering\u003c/p\u003e\n","descriptionMd":"\nFrom March 24th - 26th, 2020 many customers in Europe and the United Kingdom experienced delays in their builds and releases targeting our hosted Windows and Linux agents. This incident was caused by VM capacity constraints arising from the global health pandemic that led to increased machine reimage times and then increased wait times for available agents. Many customers experienced significant delays in their pipelines over multiple days. We sincerely apologize for the impact of this incident. \n\n# Customer Impact\n\nThe period of highest impact was 09:00-18:00 UTC in Europe on March 24th, with additional disruptions from 07:00-17:00 UTC on March 25th, and 07:00-16:00 UTC on March 26th. These periods correspond to peak usage in the affected regions. Customers experienced delays of 21 minutes on average and as high as 9 hours. Across these three periods, 6,136 organizations experienced delays. Here is a graph showing the number of impacted organizations during this period: \n\n![Chart depicting data described in this report.](https://status.dev.azure.com/_apis/status/livesiteevents/181122901/attachments/7138fdf0-031a-41f7-a535-85d1a8f52309.png)\n\n# What Happened\n\nAs Azure has documented at [http://aka.ms/cloudCovidResponseFAQ](http://aka.ms/cloudCovidResponseFAQ), a significant surge in use has led to deployment success rates for some compute resource types falling below their normal rates in a number of geographies, including Europe and the United Kingdom. For each Azure Pipelines job that is run, the [hosted agent pool](https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted) allocates a fresh virtual machine based on the Azure Pipelines custom image - a process performed over 30,000 times per hour in Europe and the United Kingdom during business hours. This makes Azure Pipelines especially sensitive to increases in the compute allocation failure rate. While Azure Pipelines does retry allocation on failure, the increased failure rate causes our average time to spin up a new agent substantially. Here is a view of the number of minutes required for the reimaging step shown for the 95th percentile: \n\n![Chart depicting data described in this report.](https://status.dev.azure.com/_apis/status/livesiteevents/181122901/attachments/b8b11b4b-6a83-411a-bd65-5ce258a58d8d.png)\n\nThis increase in the time to spin up new agents then led to the total number of agents in the hosted pool being too low to service the number of jobs, which then led to queueing and pipeline delays. Here’s a view of the wait time, in minutes, for the 95th percentile:\n\n![Chart depicting data described in this report.](https://status.dev.azure.com/_apis/status/livesiteevents/181122901/attachments/ec027afa-8835-4ba0-96d6-b189ba60e076.png)\n\nWe’ve been working on architectural changes to our hosted agent pools to mitigate the potential for issues of this type, and we sped up the deployment of these fixes in Europe and the United Kingdom as a result of these incidents. For Linux agents, the change was to use [ephemeral OS disks](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/ephemeral-os-disks). This replaces VM allocation/deallocation operations per pipeline job with a re-image operation, thus avoiding allocations altogether. For Windows agents, ephemeral disks were not an option due to the size restrictions - our agents have lots of software, including Visual Studio Enterprise, which requires a large amount of disk storage. As a result, the change for Windows involves using larger Azure VMs and [nested virtualization](https://docs.microsoft.com/en-us/azure/virtual-machines/windows/nested-virtualization). The primary reason these incidents took so long to mitigate is that we were rolling these changes out much faster than we otherwise would have wanted to, and needed to ensure they didn’t introduce new issues or make things even worse. \n\n# Communication problems\n\nIn addition to the technical problems discussed above, our communication during this incident was also problematic. On the first day, when the impact was most severe, we didn’t acknowledge the incident for approximately five hours, which is substantially worse than our target of 10 minutes. This lack of acknowledgement leads to frustration and confusion, and we apologize for that as well.\n\nThe problem here is that our live-site processes have a gap for these types of incidents. When incidents involve customer request failures or performance impacts, we have automated tooling that starts an incident and loops in both a DRI (designated responsible individual) and what we call a PIM (primary incident manager). The PIM is typically the person responsible for posting external communications acknowledging the incident.\n\nPipeline delays are detected by different tooling, and the PIM is not currently paged for these types of incidents. As a result, while the DRI was hard at work understanding the technical issues and looking for potential mitigations, the PIM was still asleep. Only when the PIM joined the incident bridge at roughly the beginning of business hours in the Eastern United States was the incident finally acknowledged. \n\n# Next Steps\n\nTo avoid incidents like this in the future, we are taking several steps:\n\n1. We are improving our monitoring and alerting by keeping a closer watch on compute allocation failure rates and agent reimage times. This supplements the monitoring and alerting we already had in place for agent availability and pipeline wait times. \n2. If we notice impending issues, we will quickly expand our use of ephemeral OS disks for Linux agents, nested virtualization for Windows agents, or both. In the absence of issues, we will continue rolling these changes out slowly, following our normal safe deployment practices. \n3. We are improving our live-site processes to ensure that initial communication of pipeline delay incidents happens on the same schedule as other incident types. \n\nAgain, we understand the impact that these types of pipeline delays have, especially in combination with a delay in incident communication, and we sincerely apologize for the incident. \n\nSincerely, \nChad Kimes\nDirector of Engineering\n"},{"id":2270,"type":1,"scopeType":1,"creationDateTime":"\/Date(1585083329487)\/","lastUpdatedDateTime":"\/Date(1585083329487)\/","author":{"id":"f170e8c5-65dd-4bda-8d27-42faeeda864a","displayName":"Jiange Sun"},"description":"\u003cp\u003eThe issue is now fully mitigated. We apologize for the impact this had on our customers.\u003c/p\u003e\n","descriptionMd":"The issue is now fully mitigated. We apologize for the impact this had on our customers."},{"id":2269,"type":1,"scopeType":1,"creationDateTime":"\/Date(1585070937100)\/","lastUpdatedDateTime":"\/Date(1585070937100)\/","author":{"id":"9ac156b6-7efa-6c7d-abbc-123a989f9709","displayName":"Chad Kimes"},"description":"\u003cp\u003eOur engineers understand the root cause and are working on mitigations. Queue times are currently expected to be up to 20 minutes for Windows and Ubuntu hosted pools.\u003c/p\u003e\n","descriptionMd":"Our engineers understand the root cause and are working on mitigations. Queue times are currently expected to be up to 20 minutes for Windows and Ubuntu hosted pools."},{"id":2268,"type":1,"scopeType":1,"creationDateTime":"\/Date(1585058582670)\/","lastUpdatedDateTime":"\/Date(1585058582670)\/","author":{"id":"9ac156b6-7efa-6c7d-abbc-123a989f9709","displayName":"Chad Kimes"},"description":"\u003cp\u003eOur engineers are working to understand and mitigate the issue.\u003c/p\u003e\n","descriptionMd":"Our engineers are working to understand and mitigate the issue."},{"id":2267,"type":1,"scopeType":1,"creationDateTime":"\/Date(1585053974210)\/","lastUpdatedDateTime":"\/Date(1585053974210)\/","author":{"id":"9ac156b6-7efa-6c7d-abbc-123a989f9709","displayName":"Chad Kimes"},"description":"\u003cp\u003eOur engineers are currently investigating an event impacting Azure DevOps hosted pools for Ubuntu and Windows in Europe. The event is being triaged and we will post an update as soon as we know more.\u003c/p\u003e\n","descriptionMd":"Our engineers are currently investigating an event impacting Azure DevOps hosted pools for Ubuntu and Windows in Europe. The event is being triaged and we will post an update as soon as we know more."}],"isDeleted":false}},"ms.vss-web.msal-config-data":{"accessMapping":"ServiceDomainMapping","authenticatedUser":{"tenantId":"00000000-0000-0000-0000-000000000000","oid":""},"cacheLookupPolicy":0,"redirectResponseMode":"","scopes":["499b84ac-1321-427f-aa17-267ca6975798/user_impersonation"],"fatalErrorCodes":["multiple_matching_tokens"],"spaAuthorizationCode":null,"fpsTarget":null,"loginTrackingId":"7fb18e2c-57d0-4030-beba-9dedeadfdb42","configuration":{"auth":{"authority":"https://login.microsoftonline.com/organizations/","clientId":"499b84ac-1321-427f-aa17-267ca6975798","clientCapabilities":["CP1"],"redirectUri":"https://app.vssps.visualstudio.com/_msal/_popup"},"cache":{"cacheLocation":"localStorage"},"system":{"loggerOptions":{"logLevel":1}}},"cacheWatermark":null,"serviceWorkerEnabled":true,"extraQueryParameters":{}},"ms.vss-web.navigation-data":{"displayed":["ms.vss-status-web.deployment-level-page","ms.vss-status-web.deployment-level-public-page","ms.vss-status-web.public-live-site-event-page-view"],"hierarchy":{"ms.vss-status-web.deployment-level-public-page":["ms.vss-status-web.public-health-check-page-view","ms.vss-status-web.public-live-site-event-page-view","ms.vss-status-web.public-status-history-page-view","ms.vss-status-web.public-status-page-view"],"ms.vss-status-web.deployment-level-page":["ms.vss-status-web.deployment-level-public-page"]},"navigation":{"ms.vss-status-web.deployment-level-public-page":{"id":"ms.vss-status-web.deployment-level-public-page","name":null,"flags":3,"order":0.0,"supportsMobile":true},"ms.vss-status-web.deployment-level-page":{"id":"ms.vss-status-web.deployment-level-page","name":null,"flags":3,"order":0.0},"ms.vss-status-web.public-health-check-page-view":{"id":"ms.vss-status-web.public-health-check-page-view","name":null,"flags":3,"order":0.0},"ms.vss-status-web.public-live-site-event-page-view":{"id":"ms.vss-status-web.public-live-site-event-page-view","name":null,"flags":3,"order":0.0},"ms.vss-status-web.public-status-history-page-view":{"id":"ms.vss-status-web.public-status-history-page-view","name":null,"flags":3,"order":0.0},"ms.vss-status-web.public-status-page-view":{"id":"ms.vss-status-web.public-status-page-view","name":null,"flags":3,"order":0.0}},"location":{"ms.vss-status-web.deployment-level-public-page":null,"ms.vss-status-web.deployment-level-page":null,"ms.vss-status-web.public-health-check-page-view":null,"ms.vss-status-web.public-live-site-event-page-view":"/_event/181122901","ms.vss-status-web.public-status-history-page-view":null,"ms.vss-status-web.public-status-page-view":null},"routeId":"ms.vss-status-web.public-live-site-event-page-view-route","routeTemplates":["_event/{id}","_event/{id}/{parameters}"],"routeValues":{"id":"181122901","parameters":"post-mortem","controller":"ContributedPage","action":"Execute"}}},"sharedData":{"_identityDetails":{"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa":{"displayName":"Anonymous"}},"_assets":{"ms.vss-status-web/core/Content/AzDevOps.svg":"https://cdn.vsassets.io/ext/ms.vss-status-web/core/Content/AzDevOps.m5PeHm.svg","ms.vss-status-web/core/Content/AzDevOpsArtifacts.svg":"https://cdn.vsassets.io/ext/ms.vss-status-web/core/Content/AzDevOpsArtifacts.GFoqIT.svg","ms.vss-status-web/core/Content/AzDevOpsBoards.svg":"https://cdn.vsassets.io/ext/ms.vss-status-web/core/Content/AzDevOpsBoards.Gfto6L.svg","ms.vss-status-web/core/Content/AzDevOpsGeneric.svg":"https://cdn.vsassets.io/ext/ms.vss-status-web/core/Content/AzDevOpsGeneric.z0zDcJ.svg","ms.vss-status-web/core/Content/AzDevOpsPipelines.svg":"https://cdn.vsassets.io/ext/ms.vss-status-web/core/Content/AzDevOpsPipelines.ZoHcfU.svg","ms.vss-status-web/core/Content/AzDevOpsRepos.svg":"https://cdn.vsassets.io/ext/ms.vss-status-web/core/Content/AzDevOpsRepos.TyuQQr.svg","ms.vss-status-web/core/Content/AzDevOpsTestPlans.svg":"https://cdn.vsassets.io/ext/ms.vss-status-web/core/Content/AzDevOpsTestPlans.tJF4ss.svg","ms.vss-status-web/core/Content/Error404.png":"https://cdn.vsassets.io/ext/ms.vss-status-web/core/Content/Error404.x9Co4W.png","ms.vss-status-web/core/Content/Error500.png":"https://cdn.vsassets.io/ext/ms.vss-status-web/core/Content/Error500.X7KDFc.png"},"_featureFlags":{"Microsoft.Azure.DevOps.Status.AdminLogCheckboxesEnabled":true,"WebPlatform.SessionTokens.UseEntraTokens":false,"WebPlatform.Authentication.AllowMsalInIFramesForSignout":false,"VisualStudio.Services.WebPlatform.ServiceWorkerTargetedTokenRefreshFixEnabled":false,"VisualStudio.Services.WebPlatform.RedirectFlow.QueryResourceAreaByInstanceId":false,"WebPlatform.Authentication.EnableCAE":true},"_routes":{"ms.vss-status-web.public-status-page-view-route":[""],"ms.vss-status-web.public-status-history-page-view-route":["_history"],"ms.vss-status-web.public-live-site-event-page-view-route":["_event/{id}","_event/{id}/{parameters}"],"ms.vss-status-web.public-health-check-page-view-route":["_ipcheck"]},"_contributions":{"contributions":{"ms.vss-status-web.live-site-event-breadcrumb-provider":{"serviceName":"live-site-event-breadcrumb-provider","breadcrumbId":"ms.vss-status-web-header-breadcrumb","::Attributes":83,"::Hashcode":"l3qsfNEgtqivkqrQ64RJ2Q=="},"ms.vss-status-web.status-msal-login-provider":{"serviceName":"IMsalLoginProviderService","::Attributes":83,"::Hashcode":"l3qsfNEgtqivkqrQ64RJ2Q=="},"ms.vss-web.msal-js-token-provider":{"serviceName":"IMsalTokenProviderService","::Attributes":83,"::Hashcode":"iRRoEKshdDO/fMuVoQWQSA==","::BaseUri":"{{ServiceUrl \u002200025394-6065-48CA-87D9-7F5672854EF7\u0022 \u0022ProjectCollection\u0022}}"},"ms.vss-web.proof-of-presence-redirect-handler":{"serviceName":"IProofOfPresenceRedirectHandler","::Attributes":83,"::Hashcode":"iRRoEKshdDO/fMuVoQWQSA==","::BaseUri":"{{ServiceUrl \u002200025394-6065-48CA-87D9-7F5672854EF7\u0022 \u0022ProjectCollection\u0022}}"}},"relationships":{"ms.vss-status-web.live-site-event-breadcrumb-provider":{"parents":{"ms.vss-status-web.public-live-site-event-page-view":true}},"ms.vss-status-web.status-msal-login-provider":{"children":{"ms.vss-status-web.msal":true},"parents":{"ms.vss-status-web.status-website":true}},"ms.vss-web.msal-js-token-provider":{"children":{"ms.vss-web.msal-content":true,"ms.vss-web.msal-config-data":true,"ms.vss-web.msal-redirect-callback-handlers":true,"ms.vss-web.msal-signout-signal-container":true},"parents":{"ms.vss-web.token-provider-service-root":true}},"ms.vss-web.proof-of-presence-redirect-handler":{"parents":{"ms.vss-web.msal-redirect-callback-handlers":true}}},"types":{"ms.vss-web.breadcrumb-provider":{"ms.vss-status-web.live-site-event-breadcrumb-provider":true},"ms.vss-web.service":{"ms.vss-status-web.status-msal-login-provider":true,"ms.vss-web.msal-js-token-provider":true,"ms.vss-web.proof-of-presence-redirect-handler":true}},"providers":{}},"_themes":{"ms.vss-status-web.vsts-theme-status":{"id":"ms.vss-status-web.vsts-theme-status","name":"Status","extends":"ms.vss-web.vsts-theme","data":{"palette-primary-darken-6":"rgba(0, 103, 181, 1)","palette-primary-darken-10":"rgba(0, 91, 161, 1)","palette-primary-darkened-6":"0, 103, 181","palette-primary-darkened-10":"0, 91, 161","palette-primary-shade-30":"0, 69, 120","palette-primary-shade-20":"0, 90, 158","palette-primary-shade-10":"16, 110, 190","palette-primary":"0, 120, 212","palette-primary-tint-10":"43, 136, 216","palette-primary-tint-20":"199, 224, 244","palette-primary-tint-30":"222, 236, 249","palette-primary-tint-40":"239, 246, 252","palette-neutral-100":"0, 0, 0","palette-neutral-80":"51, 51, 51","palette-neutral-70":"76, 76, 76","palette-neutral-60":"102, 102, 102","palette-neutral-30":"166, 166, 166","palette-neutral-20":"200, 200, 200","palette-neutral-10":"218, 218, 218","palette-neutral-8":"234, 234, 234","palette-neutral-6":"239, 239, 239","palette-neutral-4":"244, 244, 244","palette-neutral-2":"248, 248, 248","palette-neutral-0":"255, 255, 255","palette-error":"rgba(232, 17, 35, 1)","palette-error-6":"rgba(203, 15, 31, 1)","palette-error-10":"rgba(184, 14, 28, 1)","palette-black-alpha-0":"rgba(var(--palette-neutral-100), 0)","palette-black-alpha-2":"rgba(var(--palette-neutral-100), 0.02)","palette-black-alpha-4":"rgba(var(--palette-neutral-100), 0.04)","palette-black-alpha-6":"rgba(var(--palette-neutral-100), 0.06)","palette-black-alpha-8":"rgba(var(--palette-neutral-100), 0.08)","palette-black-alpha-10":"rgba(var(--palette-neutral-100), 0.10)","palette-black-alpha-20":"rgba(var(--palette-neutral-100), 0.20)","palette-black-alpha-30":"rgba(var(--palette-neutral-100), 0.30)","palette-black-alpha-60":"rgba(var(--palette-neutral-100), 0.60)","palette-black-alpha-70":"rgba(var(--palette-neutral-100), 0.70)","palette-black-alpha-80":"rgba(var(--palette-neutral-100), 0.80)","palette-black-alpha-100":"rgba(var(--palette-neutral-100), 1)","palette-accent1-light":"249, 235, 235","palette-accent1":"218, 10, 0","palette-accent1-dark":"168, 0, 0","palette-accent2-light":"223, 246, 221","palette-accent2":"186, 216, 10","palette-accent2-dark":"16, 124, 16","palette-accent3-light":"251, 242, 236","palette-accent3":"214, 127, 60","palette-accent3-dark":"171, 102, 48","background-color":"rgba(var(--palette-neutral-0), 1)","communication-foreground":"rgba(var(--palette-primary-shade-20), 1)","communication-background":"rgba(var(--palette-primary), 1)","status-info-foreground":"rgba(0, 120, 212, 1)","status-info-background":"rgba(0, 120, 212, 1)","status-error-foreground":"rgba(205, 74, 69, 1)","status-error-background":"rgba(var(--palette-accent1-light), 1)","status-error-text":"rgba(var(--palette-accent1), 1)","status-error-strong":"rgba(var(--palette-accent1-dark), 1)","status-success-foreground":"rgba(var(--palette-accent2-dark), 1)","status-success-background":"rgba(var(--palette-accent2-light), 1)","status-success-text":"rgba(127, 184, 0, 1)","status-warning-icon-foreground":"rgba(177, 133, 37, 1)","status-warning-foreground":"rgba(214, 127, 60, 1)","status-warning-background":"rgba(var(--palette-accent3-light), 1)","status-warning-text":"rgba(184, 94, 6, 1)","text-primary-color":"rgba(var(--palette-neutral-100), .9)","text-secondary-color":"rgba(var(--palette-neutral-100), .55)","text-disabled-color":"rgba(var(--palette-neutral-100), .38)","text-on-communication-background":"var(--background-color)","border-subtle-color":"rgba(var(--palette-neutral-100), .08)","splitter-border-color":"rgba(var(--palette-neutral-100), .42)","callout-background-color":"var(--background-color)","callout-filtered-background-color":"rgba(var(--palette-neutral-0), 0.86)","callout-shadow-color":"rgba(0, 0, 0, .132)","callout-shadow-secondary-color":"rgba(0, 0, 0, .108)","panel-shadow-color":"rgba(0, 0, 0, .22)","panel-shadow-secondary-color":"rgba(0, 0, 0, .18)","focus-pulse-max-color":"rgba(var(--palette-primary), 0.23)","focus-pulse-min-color":"rgba(var(--palette-primary), 0.15)","focus-border-color":"rgba(var(--palette-primary), 1)","third-party-icon-filter":"none","component-status-success":"rgba(81, 156, 93, 1)","component-status-warning":"rgba(207, 122, 58, 1)","component-status-info":"rgba(0, 120, 212, 1)","component-status-neutral":"rgba(102, 102, 102, 1)","component-status-error":"rgba(205, 74, 69, 1)"}}}},"resolvedProviders":[{"id":"ms.vss-web.component-data"},{"id":"ms.vss-web.page-data"},{"id":"ms.vss-web.shared-data"},{"id":"ms.vss-web.theme-data"},{"id":"ms.vss-web.user-claims-data"},{"id":"ms.vss-status-web.public-status-metadata-data-provider"},{"id":"ms.vss-status-web.public-live-site-event-data-provider"},{"id":"ms.vss-web.msal-config-data"},{"id":"ms.vss-web.navigation-data"}],"exceptions":null,"clientProviders":null,"scopeName":"status","scopeValue":"ms.vss-status-web.public-live-site-event-page-view-route"}</script>
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title></title>
</head>
<body class="lwp fluent-icons-enabled">
<noscript><div class="absolute-fill flex-column flex-center"><h2>JavaScript is Disabled</h2><p>Please enable javascript and refresh the page</p></div></noscript>
<input type="hidden" name="__RequestVerificationToken" value="LCzay5jIB4vOnjUtqHeugOQoePttugRmev7n8rUU4ZtNiMEGRlQ8De3yKCVzEpewP-nrq-34j51Ik7q0jL9bOfVQ7xW3NZEgKHtMvWTAJIo1" /><input type="hidden" name="__RequestVerificationToken2" value="__RequestVerificationToken203783f70-f6b1-4272-a50d-36b9ed660c80" />
<div data-componentregion="page" class="full-size"></div>
<div class="bolt-portal-host absolute-fill no-events scroll-hidden"></div>
</body>
</html>