124 lines
41 KiB
HTML
124 lines
41 KiB
HTML
<html lang="en"> <head> <meta charset="utf-8"> <title>Incident details | Google Cloud Service Health</title> <link href="https://status.cloud.google.com/images/google-cloud-favicon.png" rel="icon"> <link href="//fonts.googleapis.com/css?family=Open+Sans:300,400,600,700&lang=en" rel="stylesheet" nonce="RKc-31Ij6yXhNLXEN5pJGw"> <style>table.psd-table{border-collapse:collapse;font-family:"open sans",arial,sans-serif;font-size:13px;width:100%}table.psd-table thead th{color:#767676;font-weight:normal;padding:0 5px 10px 8px;text-align:left;text-transform:uppercase}table.psd-table td,table.psd-table tbody th{background-color:#fcfcfc;border-bottom:1px solid #eee;border-top:1px solid #eee;font-weight:normal;padding:15px 5px 15px 8px;text-align:left}</style> <script nonce="RKc-31Ij6yXhNLXEN5pJGw">(function(){var trackingId='UA-177437757-6';(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');ga('create',trackingId,{'storage':'none' });ga('set','anonymizeIp',true);ga('set','referrer',document.referrer.split('?')[0]);ga('set','location',window.location.href.split("?")[0]);ga('send','pageview');})();</script> <style>body{font-family:"open sans",arial,sans-serif}main{margin:auto;max-width:978px}p{font-size:13px}h2{color:#444;font-size:16px;font-weight:600}h3{color:#444;font-size:14px;font-weight:600}table.status-updates td.status{padding-right:0}table.status-updates .date{padding-right:20px;white-space:nowrap}table.status-updates td.time{padding-right:20px;white-space:nowrap}.location{padding-right:8px}.location::after{content:','}.locations .location:last-of-type::after{content:''}</style> </head> <body> <psd-dashboard-top-bar class="mPVexkcpuTk__intro"><style>header{min-height:64px;display:flex;align-items:center;margin:auto;max-width:978px;justify-content:space-between}.mPVexkcpuTk__header-border{border-bottom:1px solid #eee}header .mPVexkcpuTk__dashboard-logo{float:left;height:40px}</style> <header class="mPVexkcpuTk__header-border" style="max-width:1200px"> <a class="mPVexkcpuTk__logo-link" href="https://status.cloud.google.com/" aria-label="Google Cloud"> <img class="mPVexkcpuTk__dashboard-logo" src="https://status.cloud.google.com/images/google-cloud-logo.svg" alt="Google Cloud"> </a> <div class="mPVexkcpuTk__upper-right-anchor jv6boZDWPRM__widget-container"><style>.jv6boZDWPRM__widget-container{margin:8px 0}.jv6boZDWPRM__widget-container.jv6boZDWPRM__empty{display:none}.jv6boZDWPRM__widget-container.jv6boZDWPRM__no-margin,.jv6boZDWPRM__widget-container.jv6boZDWPRM__no-widget{margin:0}</style> <div> <div> <div><style>.uoGMeAS7ink__widget-link{-webkit-box-align:center;-webkit-align-items:center;-moz-box-align:center;-ms-flex-align:center;align-items:center;border-radius:4px;display:-webkit-box;display:-webkit-flex;display:-moz-box;display:-ms-flexbox;display:flex;-webkit-box-flex:0;-webkit-flex:none;-moz-box-flex:0;-ms-flex:none;flex:none;-webkit-flex-grow:0;-ms-flex-positive:0;flex-grow:0;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-moz-box-orient:horizontal;-moz-box-direction:normal;-ms-flex-direction:row;flex-direction:row;gap:8px;-webkit-box-ordinal-group:2;-webkit-order:1;-moz-box-ordinal-group:2;-ms-flex-order:1;order:1;padding:8px}.uoGMeAS7ink__visually-hidden{border:0;clip:rect(0,0,0,0);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.uoGMeAS7ink__widget-link-text{-webkit-box-align:center;-webkit-align-items:center;-moz-box-align:center;-ms-flex-align:center;align-items:center;color:#5f6368;display:-webkit-box;display:-webkit-flex;display:-moz-box;display:-ms-flexbox;display:flex;-webkit-box-flex:0;-webkit-flex:none;-moz-box-flex:0;-ms-flex:none;flex:none;-webkit-flex-grow:0;-ms-flex-positive:0;flex-grow:0;font-family:Google Sans,open sans,arial,sans-serif;font-style:normal;font-weight:500;font-size:14px;letter-spacing:.25px;line-height:20px;-webkit-box-ordinal-group:1;-webkit-order:0;-moz-box-ordinal-group:1;-ms-flex-order:0;order:0;text-align:center}</style> <a class="uoGMeAS7ink__widget-link" href="https://status.cloud.google.com/redirect/console" target="_blank"> <p class="uoGMeAS7ink__widget-link-text"> Console </p> <svg class="uoGMeAS7ink__widget-link-icon" aria-hidden="true" fill="#5F6368" height="20" viewBox="0 -960 960 960" width="20" xmlns="http://www.w3.org/2000/svg"> <path d="M216-144q-29.7 0-50.85-21.15Q144-186.3 144-216v-528q0-29.7 21.15-50.85Q186.3-816 216-816h264v72H216v528h528v-264h72v264q0 29.7-21.15 50.85Q773.7-144 744-144H216Zm171-192-51-51 357-357H576v-72h240v240h-72v-117L387-336Z"> </path> </svg> <span class="uoGMeAS7ink__visually-hidden">Opens in a new tab</span> </a> </div> </div> </div> </div> </header> </psd-dashboard-top-bar> <main style="max-width:1200px"> <ul class="vQsaEtosvhM__breadcrumbs"><style>.vQsaEtosvhM__breadcrumbs{font-size:11px;margin:15px 0 20px 0;padding:0}.vQsaEtosvhM__breadcrumbs li{display:inline-block;margin:12px 0 0 0;padding:0}.vQsaEtosvhM__breadcrumbs li a{color:black;text-decoration:none}.vQsaEtosvhM__breadcrumbs li a:after{content:"\00203a";font-size:18px;line-height:13px;padding:0 8px;position:relative;top:1px}.vQsaEtosvhM__breadcrumbs li:last-child a:after{content:""}.vQsaEtosvhM__breadcrumbs li.vQsaEtosvhM__incident-description{max-width:340px;overflow:hidden;text-overflow:ellipsis;vertical-align:bottom;white-space:nowrap}</style> <li> <a href="https://status.cloud.google.com/">Google Cloud Service Health</a></li> <li><a href="https://status.cloud.google.com/summary">Incidents</a></li> <li class="vQsaEtosvhM__incident-description">Google Cloud Interconnect in Montreal, Canada is experiencing availability issues.</li> </ul> <div class="seE0LugIuG5__header-container"><style>h1{color:#444;font-size:18px;font-weight:600}.seE0LugIuG5__dashboard-title{margin:0}.seE0LugIuG5__header-container{-webkit-box-align:center;-webkit-align-items:center;-moz-box-align:center;-ms-flex-align:center;align-items:center;display:-webkit-box;display:-webkit-flex;display:-moz-box;display:-ms-flexbox;display:flex;margin:.67em 0}</style> <h1 class="seE0LugIuG5__dashboard-title dashboard-header">Service Health</h1> <div class="jv6boZDWPRM__widget-container jv6boZDWPRM__empty"> </div> </div> <div class="aNxXV4vcNrg__intro"><style>a{color:#245dc1;text-decoration:none}a:hover{text-decoration:underline}a:visited{color:#7759ae}.aNxXV4vcNrg__intro{color:#767676;font-size:13px}</style> <div> <span> This page provides status information on the services that are part of Google Cloud. Check back here to view the current status of the services listed below. If you are experiencing an issue not listed here, please <a href="https://cloud.google.com/support-hub/">contact Support</a>. Learn more about what's posted on the dashboard in <a href="https://cloud.google.com/support/docs/dashboard">this FAQ</a>. For additional information on these services, please visit <a href="https://cloud.google.com/">https://cloud.google.com/</a>. </span> <p>For incidents related to Google Security Products, visit <a href="https://status.cloud.google.com/security">https://status.cloud.google.com/security</a>. For incidents related to Looker (original), visit <a href="https://status.cloud.google.com/looker">https://status.cloud.google.com/looker</a>.</p>
|
||
</div> <div class="jv6boZDWPRM__widget-container jv6boZDWPRM__empty"> </div> </div> <div class="jv6boZDWPRM__widget-container"> <div> <ul class="nqqp3ZlxVUB__service-status-documentation"><style>ul.nqqp3ZlxVUB__service-status-documentation{display:flex;color:#767676;font-size:13px;list-style:none;padding:0}ul.nqqp3ZlxVUB__service-status-documentation .nqqp3ZlxVUB__with-margin{margin:5px}ul .nqqp3ZlxVUB__with-flex{display:inline-flex}.nqqp3ZlxVUB__with-flex span{margin-left:9px}psd-status-icon{display:inline-block}</style> <li class="nqqp3ZlxVUB__with-margin nqqp3ZlxVUB__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon><span>Available</span> </li> <li class="nqqp3ZlxVUB__with-margin nqqp3ZlxVUB__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__information" fill="none" aria-label="Informational status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm-.833 4.167v1.666h1.667V4.833H8.168Zm0 3.333v5h1.667v-5H8.168Zm-5.833.833A6.676 6.676 0 0 0 9 15.666a6.676 6.676 0 0 0 6.667-6.667A6.676 6.676 0 0 0 9 2.333a6.676 6.676 0 0 0-6.666 6.666Z" fill="#1967D2" fill-rule="evenodd"></path></svg> </psd-status-icon><span>Service information</span> </li> <li class="nqqp3ZlxVUB__with-margin nqqp3ZlxVUB__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__disruption" fill="none" aria-label="Disruption status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M8.168 11.5h1.667v1.666H8.168v-1.667Zm0-6.667h1.667v5H8.168v-5ZM8.993.666C4.393.666.668 4.399.668 8.999s3.725 8.334 8.325 8.334c4.608 0 8.342-3.734 8.342-8.334S13.6.666 8.993.666Zm.008 15a6.665 6.665 0 0 1-6.666-6.667A6.665 6.665 0 0 1 9 2.333a6.665 6.665 0 0 1 6.667 6.666A6.665 6.665 0 0 1 9 15.666Z" fill="#E37400" fill-rule="evenodd"></path></svg> </psd-status-icon><span>Service disruption</span> </li> <li class="nqqp3ZlxVUB__with-margin nqqp3ZlxVUB__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__outage" fill="none" aria-label="Outage status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M11.16 5.666 9 7.824 6.843 5.666 5.668 6.841l2.158 2.158-2.158 2.159 1.175 1.175 2.158-2.159 2.159 2.159 1.175-1.175-2.159-2.159 2.159-2.158-1.175-1.175ZM9 .666A8.326 8.326 0 0 0 .668 8.999a8.326 8.326 0 0 0 8.333 8.334 8.326 8.326 0 0 0 8.334-8.334A8.326 8.326 0 0 0 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Z" fill="#D93025" fill-rule="evenodd"></path></svg> </psd-status-icon><span>Service outage</span> </li> </ul> </div> </div> <p class="incident-header"> Incident affecting Hybrid Connectivity </p> <h2 class="incident-description">Google Cloud Interconnect in Montreal, Canada is experiencing availability issues.</h2> <p> Incident began at <strong class="start-time">2023-08-14 16:11</strong> and ended at <strong class="end-time">2023-08-15 11:22</strong> <span>(all times are <strong class="timezone">US/Pacific</strong>).</span> </p> <h3> Previously affected location(s) </h3> <p class="past-locations locations"> <span class="location">Montréal (northamerica-northeast1)</span> </p> <table class="status-updates psd-table"> <thead> <tr> <th class="status" scope="col"></th> <th class="date" scope="col">Date</th> <th class="time" scope="col">Time</th> <th class="description" scope="col">Description</th> </tr> </thead> <tbody> <tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">17 Aug 2023</th> <td class="time">13:40 PDT</td> <td class="description"><h1>Incident Report</h1>
|
||
<h2>Summary</h2>
|
||
<p>On Monday, 14 August 2023 starting at 16:11 US/Pacific, Google Cloud Interconnect experienced an outage with underground fiber cables in Montreal, Canada. Customers in the impacted location experienced availability issues and were unable to access Google Cloud services via physical connections for a period of 19 hours and 11 minutes. To our customers whose businesses were impacted during this outage, we sincerely apologize. This is not the level of quality and reliability we strive to offer, and we are taking immediate steps to improve the platform’s performance and availability.</p>
|
||
<h2>Root Cause</h2>
|
||
<p>The outage occurred due to a fire in an underground, third-party owned cable chamber that affected a larger leased fiber cable below street level, impacting the availability of the only physical path to the Edge Availability Domains (EADs) <a href="https://cloud.google.com/network-connectivity/docs/interconnect/concepts/choosing-colocation-facilities#locations-table">yul-zone1-1944 and yul-zone2-1944</a>. The fire affected a fiber conduit that is leveraged by Google to provide inter-zone connectivity.</p>
|
||
<p>At the time of the fire, this was the only physical path providing interconnect to customers from these EADs. This is not the expected redundancy.</p>
|
||
<p>The Edge Availability Domains, <a href="https://cloud.google.com/network-connectivity/docs/interconnect/concepts/choosing-colocation-facilities#locations-table">yul-zone1-1944 and yul-zone2-1944</a>, are planned and built for full redundancy of fiber connections back to the Google backbone network, with no single points of failure in equipment, and fiber paths are verified to be physically diverse along the whole route between Google network locations.</p>
|
||
<p>The EADs were operating without a usable redundant path due to previous fiber cuts that remained unrepaired. The repairs were not completed due to prematurely updated documentation in our systems to reflect our plans for future migrations, which caused us to believe the inactive fiber status was intentional.</p>
|
||
<p>Other than the impact to Cloud Interconnect, the northamerica-northeast1 (YUL) region was unaffected.</p>
|
||
<h2>Remediation and Prevention</h2>
|
||
<p>Google engineers were alerted to the outage via our internal systems on 14 August, 16:11 US/Pacific and immediately started an investigation.</p>
|
||
<p>Google engineering teams worked with the third-party fiber vendor and restored service for Cloud Interconnect location <a href="https://cloud.google.com/network-connectivity/docs/interconnect/concepts/choosing-colocation-facilities#locations-table">yul-zone1-1944</a>, on Tuesday, 15 August 2023 at 01:06 US/Pacific. Service for <a href="https://cloud.google.com/network-connectivity/docs/interconnect/concepts/choosing-colocation-facilities#locations-table">yul-zone2-1944</a>, was restored on Tuesday, 15 August at 11:22 US/Pacific.</p>
|
||
<p>The fire-damaged fiber could not be repaired in place, so the damaged underground cable chamber had to be bypassed physically. This required splicing several hundred fiber links, including the ones leased by Google, which extended the repair timeline.</p>
|
||
<p>Simultaneously, Google attempted to return to service the redundant physical path not affected by the fire. However, this was not immediately successful, and the incident was resolved by the repair of the primary path.</p>
|
||
<p>Google restored full redundancy for both EADs (both primary and redundant paths) on 16 August, at 14:55 US/Pacific.</p>
|
||
<p>Google Cloud Interconnect offers a 99.99% reliability SLA, for customers who deploy at least two connections in each of two interconnect regions. Customers that had deployed this architecture were not affected by this outage. For more information on this feature, see the following: <a href="https://cloud.google.com/network-connectivity/docs/interconnect/tutorials/production-level-overview">Topology for production-level applications overview</a></p>
|
||
<p>Additionally, Google is committed to preventing future recurrence by taking the following actions:</p>
|
||
<ul>
|
||
<li>Improve rules within Google's internal repair prioritization system so that any issue that results in a loss of redundancy for more than one EAD in a region is treated at the highest repair priority, where Google's target is a 24 hour return to service.</li>
|
||
<li>Ensure that exceptions that occur between the migration and repair process for fibers have a documented escalation process, and that migration processes do not change fiber documentation until immediately after the migration is complete.</li>
|
||
<li>Complete the migration (both in Montreal and elsewhere) that moves all Interconnect zones to fully active-active connections. This builds upon our existing standard of "no single failure can affect both EADs in a region" to a stronger commitment of "no single failure can affect any EAD".</li>
|
||
<li>Ensure that clear notifications for isolation outages are sent proactively to customers.</li>
|
||
</ul>
|
||
<h2>Detailed Description of Impact</h2>
|
||
<ul>
|
||
<li>Starting on 14 August, 16:11 US/Pacific, 100% of customers using Cloud Interconnect in Montreal, Canada experienced unavailability, with customers unable to access Google Cloud services via physical connections in that location.</li>
|
||
<li>The interconnect location “yul-zone1-1944” was restored 10 hours, 16 minutes before yul-zone2-1944, allowing customers with redundant connectivity to access services via yul-zone1-1944 until yul-zone2-1944 was restored.</li>
|
||
<li>Other Google Cloud services in northamerica-northeast1 were accessible from interconnects outside of Montreal, Canada.</li>
|
||
</ul>
|
||
<hr>
|
||
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">17 Aug 2023</th> <td class="time">08:36 PDT</td> <td class="description"><p>Google restored full redundancy for both Edge Availability Domains (both primary and redundant paths) on 16 August, at 14:55 US/Pacific</p>
|
||
<p>We will complete a full IR in the following days that will provide additional details.</p>
|
||
<hr>
|
||
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">15 Aug 2023</th> <td class="time">15:42 PDT</td> <td class="description"><h1>Mini Incident Report</h1>
|
||
<p>We apologize for the inconvenience this service outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using <a href="https://cloud.google.com/support">https://cloud.google.com/support</a>.</p>
|
||
<p>(All Times US/Pacific)</p>
|
||
<p><strong>yul-zone1-1944 Incident Start:</strong> 14 August 2023, 16:11</p>
|
||
<p><strong>yul-zone1-1944 Incident End:</strong> 15 August 2023, 01:06</p>
|
||
<p><strong>Duration:</strong> 8 hours, 55 minutes</p>
|
||
<p><strong>yul-zone2-1944 Incident Start:</strong> 14 August 2023, 16:11</p>
|
||
<p><strong>yul-zone2-1944 Incident End:</strong> 15 August 2023, 11:22</p>
|
||
<p><strong>Duration:</strong> 19 hours, 11 minutes</p>
|
||
<p><strong>Affected Services and Features:</strong></p>
|
||
<p>Google Cloud Interconnect.</p>
|
||
<p><strong>Regions/Zones:</strong> Montreal, Canada Interconnect locations.</p>
|
||
<p><strong>Description:</strong></p>
|
||
<p>Google Cloud Interconnect experienced an outage with underground fiber cables in Montreal, Canada. The customer impact during the outage was due to two unrelated issues:</p>
|
||
<p>The <a href="https://cloud.google.com/network-connectivity/docs/interconnect/concepts/terminology#edge-availability-domain">Edge Availability Domains</a> - <a href="https://cloud.google.com/network-connectivity/docs/interconnect/concepts/choosing-colocation-facilities#locations-table">yul-zone1-1944 and yul-zone2-1944</a> were operating without a usable redundant path due to a previous fiber cut that remained unrepaired. The repairs were not completed due to prematurely updating the documentation in our systems to reflect our plans for future migrations.</p>
|
||
<p>The outage occurred when a fire broke out in an underground third-party owned cable chamber, which affected a large leased fiber cable below street level, impacting the availability of the only remaining physical path to these Edge Availability Domains. The fire-affected large fiber cable is leveraged by Google to provide inter-zone connectivity. At the time of the fire this was the only physical path between the affected routers providing interconnect to customers.</p>
|
||
<p>Google engineering teams worked with the third party fiber vendor and restored service for Cloud Interconnect location <a href="https://cloud.google.com/network-connectivity/docs/interconnect/concepts/choosing-colocation-facilities#locations-table">yul-zone1-1944</a> on Tuesday, 2023-08-15 at 01:06 US/Pacific and restored service for <a href="https://cloud.google.com/network-connectivity/docs/interconnect/concepts/choosing-colocation-facilities#locations-table">yul-zone2-1944</a> on Tuesday, 2023-08-15 at 11:22 US/Pacific.</p>
|
||
<p>The fire-damaged fiber could not be repaired in place, so the damaged underground cable chamber had to be bypassed physically. This required splicing over a few hundred fiber links, including the ones leased by Google, which extended the repair timeline.</p>
|
||
<p>Simultaneously, Google attempted to return to service the redundant physical path not affected by the fire, however, this was not immediately successful before splicing efforts completed, resolving the outage. Google is continuing to work with third party vendors to establish all redundant physical paths.</p>
|
||
<p>Google will complete a full IR in the following days that will provide additional details.</p>
|
||
<p><strong>Customer Impact:</strong></p>
|
||
<ul>
|
||
<li>
|
||
<p>Customers using Cloud Interconnect in Montreal, Canada could have experienced availability issues, with customers unable to access Google Cloud services via physical connections in that location. The interconnect location “yul-zone1-1944” was restored 10 hours, 16 minutes before yul-zone2-1944, allowing customers with redundant connectivity to access services via yul-zone1-1944 until yul-zone2-1944 was restored.</p>
|
||
</li>
|
||
<li>
|
||
<p>Google Cloud services in northamerica-northeast1 were accessible from interconnects outside of Montreal, Canada.</p>
|
||
</li>
|
||
</ul>
|
||
<hr>
|
||
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">15 Aug 2023</th> <td class="time">12:32 PDT</td> <td class="description"><p>The issue with Google Cloud Interconnect in Montreal, Canada has been resolved for all affected users as of Tuesday, 2023-08-15 11:22 US/Pacific.</p>
|
||
<p>We will publish an analysis of this incident once we have completed our internal investigation.</p>
|
||
<p>We thank you for your patience while we worked on resolving the issue.</p>
|
||
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__outage" fill="none" aria-label="Outage status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M11.16 5.666 9 7.824 6.843 5.666 5.668 6.841l2.158 2.158-2.158 2.159 1.175 1.175 2.158-2.159 2.159 2.159 1.175-1.175-2.159-2.159 2.159-2.158-1.175-1.175ZM9 .666A8.326 8.326 0 0 0 .668 8.999a8.326 8.326 0 0 0 8.333 8.334 8.326 8.326 0 0 0 8.334-8.334A8.326 8.326 0 0 0 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Z" fill="#D93025" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">15 Aug 2023</th> <td class="time">11:46 PDT</td> <td class="description"><p>Summary: Google Cloud Interconnect in Montreal, Canada is experiencing availability issues.</p>
|
||
<p>Description: Google Cloud Interconnect locations <a href="https://cloud.google.com/network-connectivity/docs/interconnect/concepts/choosing-colocation-facilities#locations-table">" yul-zone1-1944" and " yul-zone2-1944"</a> in Montreal, Canada are now fully restored. Internal monitoring shows traffic on the physical connections.</p>
|
||
<p>Our Engineering team is continuing to closely monitor the traffic.</p>
|
||
<p>We will provide an update by Tuesday, 2023-08-15 12:30 US/Pacific.</p>
|
||
<p>Diagnosis: Customers using Cloud interconnect in Montreal, Canada may experience availability issues, with customers not being able to access GCP services via physical connections in that location.</p>
|
||
<p>Other GCP services in northamerica-northeast1 are not affected. The region should be accessible from interconnects outside of Montreal, Canada.</p>
|
||
<p>Workaround: Customers with redundant interconnects in multiple peering locations can use global dynamic routing to failover their Cloud Interconnect traffic to those locations.</p>
|
||
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__outage" fill="none" aria-label="Outage status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M11.16 5.666 9 7.824 6.843 5.666 5.668 6.841l2.158 2.158-2.158 2.159 1.175 1.175 2.158-2.159 2.159 2.159 1.175-1.175-2.159-2.159 2.159-2.158-1.175-1.175ZM9 .666A8.326 8.326 0 0 0 .668 8.999a8.326 8.326 0 0 0 8.333 8.334 8.326 8.326 0 0 0 8.334-8.334A8.326 8.326 0 0 0 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Z" fill="#D93025" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">15 Aug 2023</th> <td class="time">09:57 PDT</td> <td class="description"><p>Summary: Google Cloud Interconnect in Montreal, Canada is experiencing availability issues.</p>
|
||
<p>Description: We are experiencing an issue with physical connections in Google Cloud Interconnect in Montreal, Canada. Multiple <a href="https://cloud.google.com/network-connectivity/docs/interconnect/concepts/terminology">Edge Availability Domains</a> are affected.</p>
|
||
<p>Service restoration for interconnect location <a href="https://cloud.google.com/network-connectivity/docs/interconnect/concepts/choosing-colocation-facilities#locations-table">yul-zone1-1944</a> was completed on Tuesday, 2023-08-15 01:06 US/Pacific.</p>
|
||
<p>Our engineering team is continuing to work with the hardware vendor for service restoration in <a href="https://cloud.google.com/network-connectivity/docs/interconnect/concepts/choosing-colocation-facilities#locations-table">yul-zone2-1944</a></p>
|
||
<p>We do not have an ETA for complete mitigation at this point.</p>
|
||
<p>We will provide an update by Tuesday, 2023-08-15 12:00 US/Pacific.</p>
|
||
<p>Diagnosis: Customers using Cloud interconnect in Montreal, Canada may experience availability issues, with customers not being able to access GCP services via physical connections in that location.</p>
|
||
<p>Other GCP services in northamerica-northeast1 are not affected. The region should be accessible from interconnects outside of Montreal, Canada.</p>
|
||
<p>Workaround: Customers with redundant interconnects in multiple peering locations can use global dynamic routing to failover their Cloud Interconnect traffic to those locations.</p>
|
||
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__outage" fill="none" aria-label="Outage status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M11.16 5.666 9 7.824 6.843 5.666 5.668 6.841l2.158 2.158-2.158 2.159 1.175 1.175 2.158-2.159 2.159 2.159 1.175-1.175-2.159-2.159 2.159-2.158-1.175-1.175ZM9 .666A8.326 8.326 0 0 0 .668 8.999a8.326 8.326 0 0 0 8.333 8.334 8.326 8.326 0 0 0 8.334-8.334A8.326 8.326 0 0 0 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Z" fill="#D93025" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">14 Aug 2023</th> <td class="time">22:27 PDT</td> <td class="description"><p>Summary: Google Cloud Interconnect in Montreal, Canada is experiencing availability issues.</p>
|
||
<p>Description: We are experiencing an issue with physical connections in Google Cloud Interconnect in Montreal, Canada. Multiple <a href="https://cloud.google.com/network-connectivity/docs/interconnect/concepts/terminology">Edge Availability Domains</a> are affected.</p>
|
||
<p>Mitigation work is underway by our engineering team.</p>
|
||
<p>We will provide an update by Tuesday, 2023-08-15 10:00 US/Pacific.</p>
|
||
<p>Diagnosis: Customers using Cloud interconnect in Montreal, Canada may experience availability issues, with customers not being able to access GCP services via physical connections in that location.</p>
|
||
<p>Other GCP services in northamerica-northeast1 are not affected. The region should be accessible from interconnects outside of Montreal, Canada.</p>
|
||
<p>Workaround: Customers with redundant interconnects in multiple peering locations can use global dynamic routing to failover their Cloud Interconnect traffic to those locations.</p>
|
||
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__outage" fill="none" aria-label="Outage status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M11.16 5.666 9 7.824 6.843 5.666 5.668 6.841l2.158 2.158-2.158 2.159 1.175 1.175 2.158-2.159 2.159 2.159 1.175-1.175-2.159-2.159 2.159-2.158-1.175-1.175ZM9 .666A8.326 8.326 0 0 0 .668 8.999a8.326 8.326 0 0 0 8.333 8.334 8.326 8.326 0 0 0 8.334-8.334A8.326 8.326 0 0 0 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Z" fill="#D93025" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">14 Aug 2023</th> <td class="time">21:29 PDT</td> <td class="description"><p>Summary: Google Cloud Interconnect in Montreal, Canada is experiencing availability issues.</p>
|
||
<p>Description: We are experiencing an issue with physical connections in Google Cloud Interconnect in Montreal, Canada. Multiple <a href="https://cloud.google.com/network-connectivity/docs/interconnect/concepts/terminology">Edge Availability Domains</a> are affected.</p>
|
||
<p>Our engineering team continues to investigate the issue.</p>
|
||
<p>We will provide an update by Monday, 2023-08-14 23:00 US/Pacific.</p>
|
||
<p>Diagnosis: Customers using Cloud interconnect in Montreal, Canada may experience availability issues, with customers not being able to access GCP services via physical connections in that location.</p>
|
||
<p>Other GCP services in northamerica-northeast1 are not affected. The region should be accessible from interconnects outside of Montreal, Canada.</p>
|
||
<p>Workaround: Customers with redundant interconnects in multiple peering locations can use global dynamic routing to failover their Cloud Interconnect traffic to those locations.</p>
|
||
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__outage" fill="none" aria-label="Outage status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M11.16 5.666 9 7.824 6.843 5.666 5.668 6.841l2.158 2.158-2.158 2.159 1.175 1.175 2.158-2.159 2.159 2.159 1.175-1.175-2.159-2.159 2.159-2.158-1.175-1.175ZM9 .666A8.326 8.326 0 0 0 .668 8.999a8.326 8.326 0 0 0 8.333 8.334 8.326 8.326 0 0 0 8.334-8.334A8.326 8.326 0 0 0 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Z" fill="#D93025" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">14 Aug 2023</th> <td class="time">20:33 PDT</td> <td class="description"><p>Summary: Google Cloud Interconnect in Montreal, Canada is experiencing availability issues.</p>
|
||
<p>Description: We are experiencing an issue with physical connections in Google Cloud Interconnect in Montreal, Canada. Multiple <a href="https://cloud.google.com/network-connectivity/docs/interconnect/concepts/terminology">Edge Availability Domains</a> are affected.</p>
|
||
<p>Our engineering team continues to investigate the issue.</p>
|
||
<p>We will provide an update by Monday, 2023-08-14 21:45 US/Pacific.</p>
|
||
<p>Diagnosis: Customers using Cloud interconnect in Montreal, Canada may experience availability issues, with customers not being able to access GCP services via physical connections in that location.</p>
|
||
<p>Other GCP services in northamerica-northeast1 are not affected. The region should be accessible from interconnects outside of Montreal, Canada.</p>
|
||
<p>Workaround: Customers with redundant interconnects in multiple peering locations can use global dynamic routing to failover their Cloud Interconnect traffic to those locations.</p>
|
||
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__outage" fill="none" aria-label="Outage status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M11.16 5.666 9 7.824 6.843 5.666 5.668 6.841l2.158 2.158-2.158 2.159 1.175 1.175 2.158-2.159 2.159 2.159 1.175-1.175-2.159-2.159 2.159-2.158-1.175-1.175ZM9 .666A8.326 8.326 0 0 0 .668 8.999a8.326 8.326 0 0 0 8.333 8.334 8.326 8.326 0 0 0 8.334-8.334A8.326 8.326 0 0 0 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Z" fill="#D93025" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">14 Aug 2023</th> <td class="time">19:04 PDT</td> <td class="description"><p>Summary: Google Cloud Networking is experiencing interconnect issues</p>
|
||
<p>Description: We are experiencing an issue with Google Cloud Networking.</p>
|
||
<p>Our engineering team continues to investigate the issue.</p>
|
||
<p>We will provide an update by Monday, 2023-08-14 21:30 US/Pacific with current details.</p>
|
||
<p>Diagnosis: Some Google Cloud Interconnect customers in the Montréal Canada area connecting to northamerica-northeast1 may observe link down on their attachments.</p>
|
||
<p>Workaround: None at this time.</p>
|
||
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__information" fill="none" aria-label="Informational status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm-.833 4.167v1.666h1.667V4.833H8.168Zm0 3.333v5h1.667v-5H8.168Zm-5.833.833A6.676 6.676 0 0 0 9 15.666a6.676 6.676 0 0 0 6.667-6.667A6.676 6.676 0 0 0 9 2.333a6.676 6.676 0 0 0-6.666 6.666Z" fill="#1967D2" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">14 Aug 2023</th> <td class="time">18:43 PDT</td> <td class="description"><p>Summary: Google Cloud Networking is experiencing interconnect issues</p>
|
||
<p>Description: We are experiencing an issue with Google Cloud Networking.</p>
|
||
<p>Our engineering team continues to investigate the issue.</p>
|
||
<p>We will provide an update by Monday, 2023-08-14 21:30 US/Pacific with current details.</p>
|
||
<p>Diagnosis: Some Google Cloud Interconnect customers in the Montréal Canada area connecting to northamerica-northeast1 may observe link down on their attachments.</p>
|
||
<p>Workaround: None at this time.</p>
|
||
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__information" fill="none" aria-label="Informational status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm-.833 4.167v1.666h1.667V4.833H8.168Zm0 3.333v5h1.667v-5H8.168Zm-5.833.833A6.676 6.676 0 0 0 9 15.666a6.676 6.676 0 0 0 6.667-6.667A6.676 6.676 0 0 0 9 2.333a6.676 6.676 0 0 0-6.666 6.666Z" fill="#1967D2" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">14 Aug 2023</th> <td class="time">18:35 PDT</td> <td class="description"><p>Summary: Google Cloud Networking is experiencing interconnect issues</p>
|
||
<p>Description: We are experiencing an issue with Google Cloud Networking.</p>
|
||
<p>Our engineering team continues to investigate the issue.</p>
|
||
<p>We will provide an update by Monday, 2023-08-14 19:00 US/Pacific with current details.</p>
|
||
<p>Diagnosis: Some Google Cloud Interconnect customers in the Montréal Canada area connecting to northamerica-northeast1 may observe link down on their attachments.</p>
|
||
<p>Workaround: None at this time.</p>
|
||
</td> </tr> </tbody> </table> <psd-table-documentation><script src="https://support.google.com/inapp/api.js" nonce="RKc-31Ij6yXhNLXEN5pJGw"></script><style>ul.hS6GEzRhW2F__table-documentation{color:#767676;display:flex;flex-wrap:wrap;font-size:13px;list-style:none;margin-left:2.5px;margin-right:2.5px;padding:0}ul.hS6GEzRhW2F__table-documentation li{display:inline-block;margin-left:2.5px;margin-right:2.5px}ul.hS6GEzRhW2F__table-documentation .hS6GEzRhW2F__utc-display{flex-basis:20%;flex-grow:1}ul.hS6GEzRhW2F__table-documentation .hS6GEzRhW2F__dot-after::after{content:'\002022';margin-left:5px}.hS6GEzRhW2F__atom-feed-icon{background:url(data:image/gif;base64,R0lGODlhEAAQANUgAOqHCvySDOxZAORxAOBqGdbW1vCtRuKugezFheZlAOiXUdWlZMmNQ754Osh8OOZtJP6bEfuGBdxhAPbizO7YxuesbfHt6tpYAPZ4Av6nGdxqAPtjAOmQMNxNANLS0v///////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACH5BAEAACAALAAAAAAQABAAAAanQJBwSCwOOQONcqkccCxCDsCAqBoyWCzgCdJkFp/wZ3LNagogCYTDYCwm4QpkLvGkIQEAIF+xfCoBEHVpeWFkEU8fHAGDFxEYFBRhBxEVHxQRF3aOjwMNfgoDkhyaIBcYGAOoDmMYlhWlp3ATGAmSCRwfB7EEYgkbkhe5FR12HQKSExsbBBwbGAcKxSAPAgLHy9nLAgR2FgQd4eLj3WggFgUe6uvsaEEAOw==) left no-repeat;background-size:16px 16px;padding-left:19px}</style> <ul class="hS6GEzRhW2F__table-documentation"> <li class="hS6GEzRhW2F__utc-display">All times are US/Pacific</li> </ul> <ul class="hS6GEzRhW2F__table-documentation"> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__documentation"><a href="https://cloud.google.com/support/docs/dashboard">Documentation</a></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__atom-feed"><span class="hS6GEzRhW2F__atom-feed-icon"><a href="https://status.cloud.google.com/en/feed.atom">RSS Feed</a></span></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__json-history"> <a href="https://status.cloud.google.com/incidents.json">JSON History</a> | <a href="https://status.cloud.google.com/incidents.schema.json" aria-label="JSON History Schema">Schema</a></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__json-product-catalog"> <a href="https://status.cloud.google.com/products.json">JSON Product Catalog</a> | <a href="https://status.cloud.google.com/products.schema.json" aria-label="JSON Product Catalog Schema">Schema</a></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__feedback"><a href="#" data-feedback-product-id="104625" onclick="userfeedback.api.startFeedback({'productId': event.target.dataset.feedbackProductId || '5188085'})">Send feedback</a></li> <li class="hS6GEzRhW2F__privacy"><a href="https://policies.google.com/privacy">Privacy policy</a></li> </ul> </psd-table-documentation> </main> </body> </html> |