Files
nexus/sreweekly/articles/326/05-google-incident-report-google-cloud-networking-incident-on-may-20-2022.html
2026-09-12 17:23:01 +08:00

93 lines
26 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<html lang="en"> <head> <meta charset="utf-8"> <title>Incident details | Google Cloud Service Health</title> <link href="https://status.cloud.google.com/images/google-cloud-favicon.png" rel="icon"> <link href="//fonts.googleapis.com/css?family=Open+Sans:300,400,600,700&amp;lang=en" rel="stylesheet" nonce="OQAW2VkoPEkJI9Jl0SqXlA"> <style>table.psd-table{border-collapse:collapse;font-family:"open sans",arial,sans-serif;font-size:13px;width:100%}table.psd-table thead th{color:#767676;font-weight:normal;padding:0 5px 10px 8px;text-align:left;text-transform:uppercase}table.psd-table td,table.psd-table tbody th{background-color:#fcfcfc;border-bottom:1px solid #eee;border-top:1px solid #eee;font-weight:normal;padding:15px 5px 15px 8px;text-align:left}</style> <script nonce="OQAW2VkoPEkJI9Jl0SqXlA">(function(){var trackingId='UA-177437757-6';(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');ga('create',trackingId,{'storage':'none' });ga('set','anonymizeIp',true);ga('set','referrer',document.referrer.split('?')[0]);ga('set','location',window.location.href.split("?")[0]);ga('send','pageview');})();</script> <style>body{font-family:"open sans",arial,sans-serif}main{margin:auto;max-width:978px}p{font-size:13px}h2{color:#444;font-size:16px;font-weight:600}h3{color:#444;font-size:14px;font-weight:600}table.status-updates td.status{padding-right:0}table.status-updates .date{padding-right:20px;white-space:nowrap}table.status-updates td.time{padding-right:20px;white-space:nowrap}.location{padding-right:8px}.location::after{content:','}.locations .location:last-of-type::after{content:''}</style> </head> <body> <psd-dashboard-top-bar class="mPVexkcpuTk__intro"><style>header{min-height:64px;display:flex;align-items:center;margin:auto;max-width:978px;justify-content:space-between}.mPVexkcpuTk__header-border{border-bottom:1px solid #eee}header .mPVexkcpuTk__dashboard-logo{float:left;height:40px}</style> <header class="mPVexkcpuTk__header-border" style="max-width:1200px"> <a class="mPVexkcpuTk__logo-link" href="https://status.cloud.google.com/" aria-label="Google Cloud"> <img class="mPVexkcpuTk__dashboard-logo" src="https://status.cloud.google.com/images/google-cloud-logo.svg" alt="Google Cloud"> </a> <div class="mPVexkcpuTk__upper-right-anchor jv6boZDWPRM__widget-container"><style>.jv6boZDWPRM__widget-container{margin:8px 0}.jv6boZDWPRM__widget-container.jv6boZDWPRM__empty{display:none}.jv6boZDWPRM__widget-container.jv6boZDWPRM__no-margin,.jv6boZDWPRM__widget-container.jv6boZDWPRM__no-widget{margin:0}</style> <div> <div> <div><style>.uoGMeAS7ink__widget-link{-webkit-box-align:center;-webkit-align-items:center;-moz-box-align:center;-ms-flex-align:center;align-items:center;border-radius:4px;display:-webkit-box;display:-webkit-flex;display:-moz-box;display:-ms-flexbox;display:flex;-webkit-box-flex:0;-webkit-flex:none;-moz-box-flex:0;-ms-flex:none;flex:none;-webkit-flex-grow:0;-ms-flex-positive:0;flex-grow:0;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-moz-box-orient:horizontal;-moz-box-direction:normal;-ms-flex-direction:row;flex-direction:row;gap:8px;-webkit-box-ordinal-group:2;-webkit-order:1;-moz-box-ordinal-group:2;-ms-flex-order:1;order:1;padding:8px}.uoGMeAS7ink__visually-hidden{border:0;clip:rect(0,0,0,0);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.uoGMeAS7ink__widget-link-text{-webkit-box-align:center;-webkit-align-items:center;-moz-box-align:center;-ms-flex-align:center;align-items:center;color:#5f6368;display:-webkit-box;display:-webkit-flex;display:-moz-box;display:-ms-flexbox;display:flex;-webkit-box-flex:0;-webkit-flex:none;-moz-box-flex:0;-ms-flex:none;flex:none;-webkit-flex-grow:0;-ms-flex-positive:0;flex-grow:0;font-family:Google Sans,open sans,arial,sans-serif;font-style:normal;font-weight:500;font-size:14px;letter-spacing:.25px;line-height:20px;-webkit-box-ordinal-group:1;-webkit-order:0;-moz-box-ordinal-group:1;-ms-flex-order:0;order:0;text-align:center}</style> <a class="uoGMeAS7ink__widget-link" href="https://status.cloud.google.com/redirect/console" target="_blank"> <p class="uoGMeAS7ink__widget-link-text"> Console </p> <svg class="uoGMeAS7ink__widget-link-icon" aria-hidden="true" fill="#5F6368" height="20" viewBox="0 -960 960 960" width="20" xmlns="http://www.w3.org/2000/svg"> <path d="M216-144q-29.7 0-50.85-21.15Q144-186.3 144-216v-528q0-29.7 21.15-50.85Q186.3-816 216-816h264v72H216v528h528v-264h72v264q0 29.7-21.15 50.85Q773.7-144 744-144H216Zm171-192-51-51 357-357H576v-72h240v240h-72v-117L387-336Z"> </path> </svg> <span class="uoGMeAS7ink__visually-hidden">Opens in a new tab</span> </a> </div> </div> </div> </div> </header> </psd-dashboard-top-bar> <main style="max-width:1200px"> <ul class="vQsaEtosvhM__breadcrumbs"><style>.vQsaEtosvhM__breadcrumbs{font-size:11px;margin:15px 0 20px 0;padding:0}.vQsaEtosvhM__breadcrumbs li{display:inline-block;margin:12px 0 0 0;padding:0}.vQsaEtosvhM__breadcrumbs li a{color:black;text-decoration:none}.vQsaEtosvhM__breadcrumbs li a:after{content:"\00203a";font-size:18px;line-height:13px;padding:0 8px;position:relative;top:1px}.vQsaEtosvhM__breadcrumbs li:last-child a:after{content:""}.vQsaEtosvhM__breadcrumbs li.vQsaEtosvhM__incident-description{max-width:340px;overflow:hidden;text-overflow:ellipsis;vertical-align:bottom;white-space:nowrap}</style> <li> <a href="https://status.cloud.google.com/">Google Cloud Service Health</a></li> <li><a href="https://status.cloud.google.com/summary">Incidents</a></li> <li class="vQsaEtosvhM__incident-description">Global : Cloud Networking faced severe packet loss</li> </ul> <div class="seE0LugIuG5__header-container"><style>h1{color:#444;font-size:18px;font-weight:600}.seE0LugIuG5__dashboard-title{margin:0}.seE0LugIuG5__header-container{-webkit-box-align:center;-webkit-align-items:center;-moz-box-align:center;-ms-flex-align:center;align-items:center;display:-webkit-box;display:-webkit-flex;display:-moz-box;display:-ms-flexbox;display:flex;margin:.67em 0}</style> <h1 class="seE0LugIuG5__dashboard-title dashboard-header">Service Health</h1> <div class="jv6boZDWPRM__widget-container jv6boZDWPRM__empty"> </div> </div> <div class="aNxXV4vcNrg__intro"><style>a{color:#245dc1;text-decoration:none}a:hover{text-decoration:underline}a:visited{color:#7759ae}.aNxXV4vcNrg__intro{color:#767676;font-size:13px}</style> <div> <span> This page provides status information on the services that are part of Google Cloud. Check back here to view the current status of the services listed below. If you are experiencing an issue not listed here, please <a href="https://cloud.google.com/support-hub/">contact Support</a>. Learn more about what's posted on the dashboard in <a href="https://cloud.google.com/support/docs/dashboard">this FAQ</a>. For additional information on these services, please visit <a href="https://cloud.google.com/">https://cloud.google.com/</a>. </span> <p>For incidents related to Google Security Products, visit <a href="https://status.cloud.google.com/security">https://status.cloud.google.com/security</a>. For incidents related to Looker (original), visit <a href="https://status.cloud.google.com/looker">https://status.cloud.google.com/looker</a>.</p>
</div> <div class="jv6boZDWPRM__widget-container jv6boZDWPRM__empty"> </div> </div> <div class="jv6boZDWPRM__widget-container"> <div> <ul class="nqqp3ZlxVUB__service-status-documentation"><style>ul.nqqp3ZlxVUB__service-status-documentation{display:flex;color:#767676;font-size:13px;list-style:none;padding:0}ul.nqqp3ZlxVUB__service-status-documentation .nqqp3ZlxVUB__with-margin{margin:5px}ul .nqqp3ZlxVUB__with-flex{display:inline-flex}.nqqp3ZlxVUB__with-flex span{margin-left:9px}psd-status-icon{display:inline-block}</style> <li class="nqqp3ZlxVUB__with-margin nqqp3ZlxVUB__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon><span>Available</span> </li> <li class="nqqp3ZlxVUB__with-margin nqqp3ZlxVUB__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__information" fill="none" aria-label="Informational status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm-.833 4.167v1.666h1.667V4.833H8.168Zm0 3.333v5h1.667v-5H8.168Zm-5.833.833A6.676 6.676 0 0 0 9 15.666a6.676 6.676 0 0 0 6.667-6.667A6.676 6.676 0 0 0 9 2.333a6.676 6.676 0 0 0-6.666 6.666Z" fill="#1967D2" fill-rule="evenodd"></path></svg> </psd-status-icon><span>Service information</span> </li> <li class="nqqp3ZlxVUB__with-margin nqqp3ZlxVUB__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__disruption" fill="none" aria-label="Disruption status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M8.168 11.5h1.667v1.666H8.168v-1.667Zm0-6.667h1.667v5H8.168v-5ZM8.993.666C4.393.666.668 4.399.668 8.999s3.725 8.334 8.325 8.334c4.608 0 8.342-3.734 8.342-8.334S13.6.666 8.993.666Zm.008 15a6.665 6.665 0 0 1-6.666-6.667A6.665 6.665 0 0 1 9 2.333a6.665 6.665 0 0 1 6.667 6.666A6.665 6.665 0 0 1 9 15.666Z" fill="#E37400" fill-rule="evenodd"></path></svg> </psd-status-icon><span>Service disruption</span> </li> <li class="nqqp3ZlxVUB__with-margin nqqp3ZlxVUB__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__outage" fill="none" aria-label="Outage status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M11.16 5.666 9 7.824 6.843 5.666 5.668 6.841l2.158 2.158-2.158 2.159 1.175 1.175 2.158-2.159 2.159 2.159 1.175-1.175-2.159-2.159 2.159-2.158-1.175-1.175ZM9 .666A8.326 8.326 0 0 0 .668 8.999a8.326 8.326 0 0 0 8.333 8.334 8.326 8.326 0 0 0 8.334-8.334A8.326 8.326 0 0 0 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Z" fill="#D93025" fill-rule="evenodd"></path></svg> </psd-status-icon><span>Service outage</span> </li> </ul> </div> </div> <p class="incident-header"> Incident affecting Google Cloud Networking, Hybrid Connectivity </p> <h2 class="incident-description">Global : Cloud Networking faced severe packet loss</h2> <p> Incident began at <strong class="start-time">2022-05-20 13:47</strong> and ended at <strong class="end-time">2022-05-20 14:07</strong> <span>(all times are <strong class="timezone">US/Pacific</strong>).</span> </p> <h3> Previously affected location(s) </h3> <p class="past-locations locations"> <span class="location">Taiwan (asia-east1)</span><span class="location">Hong Kong (asia-east2)</span><span class="location">Tokyo (asia-northeast1)</span><span class="location">Osaka (asia-northeast2)</span><span class="location">Seoul (asia-northeast3)</span><span class="location">Mumbai (asia-south1)</span><span class="location">Delhi (asia-south2)</span><span class="location">Singapore (asia-southeast1)</span><span class="location">Jakarta (asia-southeast2)</span><span class="location">Sydney (australia-southeast1)</span><span class="location">Melbourne (australia-southeast2)</span><span class="location">Warsaw (europe-central2)</span><span class="location">Finland (europe-north1)</span><span class="location">Belgium (europe-west1)</span><span class="location">London (europe-west2)</span><span class="location">Frankfurt (europe-west3)</span><span class="location">Netherlands (europe-west4)</span><span class="location">Zurich (europe-west6)</span><span class="location">Montréal (northamerica-northeast1)</span><span class="location">Toronto (northamerica-northeast2)</span><span class="location">São Paulo (southamerica-east1)</span><span class="location">Santiago (southamerica-west1)</span><span class="location">Iowa (us-central1)</span><span class="location">South Carolina (us-east1)</span><span class="location">Northern Virginia (us-east4)</span><span class="location">Oregon (us-west1)</span><span class="location">Los Angeles (us-west2)</span><span class="location">Salt Lake City (us-west3)</span><span class="location">Las Vegas (us-west4)</span> </p> <table class="status-updates psd-table"> <thead> <tr> <th class="status" scope="col"></th> <th class="date" scope="col">Date</th> <th class="time" scope="col">Time</th> <th class="description" scope="col">Description</th> </tr> </thead> <tbody> <tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">2 Jun 2022</th> <td class="time">12:11 PDT</td> <td class="description"><p><strong>INCIDENT REPORT</strong></p>
<p><strong>SUMMARY:</strong></p>
<p>On Friday, 20 May 2022 at 13:47 US/Pacific, Google Cloud Networking experienced intermittent packet loss for traffic between multiple cloud regions for a duration of 20 minutes. The issue was identified and mitigated automatically by 14:07 US/Pacific.</p>
<p>We understand this issue has affected our valued customers and users, and we apologize to those who were affected.</p>
<p><strong>ROOT CAUSE:</strong></p>
<p>Google’s production backbone is a global network that enables connectivity for all user-facing traffic via Points of Presence (POPs) or internet exchanges.</p>
<p>A failure of a component on a fiber path from one of the central US gateway campuses in Google’s production backbone led to a decrease in available network bandwidth between the gateway and multiple edge locations, causing packet loss while the backbone automatically moved traffic onto remaining paths.</p>
<p>The network topology in this region is being augmented, and the second-best path has not completed its augmentation. This meant some traffic needed to reroute onto the third-best path, which led to an extended traffic migration period. This disruption was more severe than we had anticipated, and is the subject of remediation actions below.</p>
<p><strong>REMEDIATION AND PREVENTION:</strong></p>
<p>Google’s automated repair mechanisms detected the decrease in available network bandwidth on Friday, 20 May 2022 at 13:47 US/Pacific and automatically routed the traffic through alternate links. The traffic rerouting completed on Friday, 20 May 2022 at 14:06 US/Pacific, mitigating the issue.</p>
<p>While our automated mechanisms worked as intended and recovered the traffic without manual intervention, we understand that the scope of impact caused by this event affected our customers.</p>
<p>We have been working on optimizing our global network to minimize the time spent automatically reconfiguring around failures like this (known as &#34;convergence time&#34;). While we have made progress, efforts to improve are ongoing. We continue to ensure that the current technology is optimally configured to minimize the frequency and severity of these issues.</p>
<p>In this network region, we intend to complete the augmentation by 11 July 2022, which will return the network to the intended topology where any single fiber path failure can be rerouted quickly onto the second-best path.</p>
<p>We will also build automatic analysis to ensure that the network topology during augmentation always supports fast convergence.</p>
<p>Google is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.</p>
<p><strong>DETAILED DESCRIPTION OF IMPACT:</strong></p>
<ul>
<li>
<p>Google Cloud Networking - Affected customers would have observed packet loss for ingress and egress traffic to and from US Central Cloud regions from 13:47 to 14:07 US/Pacific.</p>
</li>
<li>
<p>Cloud VPN - Affected customers would have experienced latency and errors for cross region VPN with global routing from 13:47 to 14:07 US/Pacific. Intra-region traffic for Classic and HA VPN would not have been affected.</p>
</li>
<li>
<p>Cloud Router - Affected customers would have observed delays in global routing propagation from 13:47 to 14:07 US/Pacific.</p>
</li>
<li>
<p>Cloud Interconnect - Affected customers would have experienced latency and errors from 13:47 to 14:07 US/Pacific.</p>
</li>
<li>
<p>Google Cloud Storage - Affected customers would have experienced elevated latency, HTTP 500 errors, or transient API errors from 13:47 to 14:17 US/Pacific.</p>
</li>
<li>
<p>Cloud SQL - Affected customers would have experienced failures for Export, Update, and Delete operations as well as delayed data replication in us-west1 from 13:53 to 14:13 US/Pacific.</p>
</li>
<li>
<p>Messages - Affected customers would have experienced service availability issues from 13:49 to 14:05 US/Pacific.</p>
</li>
</ul>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">23 May 2022</th> <td class="time">14:33 PDT</td> <td class="description"><p><strong>Mini Incident Report</strong></p>
<p>We apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Support by opening a case <a href="https://cloud.google.com/support">https://cloud.google.com/support</a> or help article <a href="https://support.google.com/a/answer/1047213">https://support.google.com/a/answer/1047213</a>.</p>
<p>(All Times US/Pacific)</p>
<p><strong>Incident Start:</strong> 20 May 2022 13:47</p>
<p><strong>Incident End:</strong> 20 May 2022 14:07</p>
<p><strong>Duration:</strong> 20 minutes</p>
<p><strong>Affected Services and Features:</strong></p>
<ul>
<li>Google Cloud Networking</li>
<li>Cloud VPN</li>
<li>Cloud Router</li>
<li>Cloud Interconnect</li>
<li>Google Cloud Storage</li>
<li>Cloud SQL</li>
<li>Messages</li>
</ul>
<p><strong>Regions/Zones:</strong> Multiple regions</p>
<p><strong>Description:</strong></p>
<p>Google Cloud Networking experienced intermittent packet loss for some transit traffic affecting inter-region connectivity for multiple cloud regions, which lasted 20 minutes. From preliminary analysis, the root cause is a hardware issue on an optical (amplification) component affecting Google&#39;s user facing backbone. The issue was identified and mitigated by our automated repair mechanism without manual intervention.</p>
<p><strong>Customer Impact:</strong></p>
<ul>
<li>
<p>Google Cloud Networking - Affected customers would have observed packet loss for Ingress and egress traffic to and from central US cloud regions.</p>
</li>
<li>
<p>Cloud VPN - Affected customers would have experienced latency and errors for cross region VPN with global routing. Intra-region traffic for Classic and HA VPN would not have been affected.</p>
</li>
<li>
<p>Cloud Router - Affected customers would have observed delays in global routing propagation.</p>
</li>
<li>
<p>Cloud Interconnect - Affected customers would have experienced latency and errors.</p>
</li>
<li>
<p>Google Cloud Storage - Affected customers would have experienced elevated latency, HTTP 500 errors, or transient API errors.</p>
</li>
<li>
<p>Cloud SQL - Affected customers would have experienced failures for Export, Update, and Delete operations as well as delayed data replication in us-west1.</p>
</li>
<li>
<p>Messages - Affected customers would have experienced service availability issues.</p>
</li>
</ul>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">20 May 2022</th> <td class="time">14:52 PDT</td> <td class="description"><p>The issue with Google Cloud Networking has been resolved for all affected users as of Friday, 2022-05-20 14:07 US/Pacific.</p>
<p>Affected customers would have experienced high latency, timeouts and errors.</p>
<p>We thank you for your patience while we worked on resolving the issue.</p>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__outage" fill="none" aria-label="Outage status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M11.16 5.666 9 7.824 6.843 5.666 5.668 6.841l2.158 2.158-2.158 2.159 1.175 1.175 2.158-2.159 2.159 2.159 1.175-1.175-2.159-2.159 2.159-2.158-1.175-1.175ZM9 .666A8.326 8.326 0 0 0 .668 8.999a8.326 8.326 0 0 0 8.333 8.334 8.326 8.326 0 0 0 8.334-8.334A8.326 8.326 0 0 0 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Z" fill="#D93025" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">20 May 2022</th> <td class="time">14:44 PDT</td> <td class="description"><p>Summary: Global : Cloud Networking faced severe packet loss</p>
<p>Description: We&#39;ve received a report of an issue with Google Cloud Networking as of Friday, 2022-05-20 13:47 US/Pacific.</p>
<p>We will provide more information by Friday, 2022-05-20 14:45 US/Pacific.</p>
<p>Diagnosis: Customers may have encountered elevated latency errors</p>
<p>Workaround: None at this time.</p>
</td> </tr> </tbody> </table> <psd-table-documentation><script src="https://support.google.com/inapp/api.js" nonce="OQAW2VkoPEkJI9Jl0SqXlA"></script><style>ul.hS6GEzRhW2F__table-documentation{color:#767676;display:flex;flex-wrap:wrap;font-size:13px;list-style:none;margin-left:2.5px;margin-right:2.5px;padding:0}ul.hS6GEzRhW2F__table-documentation li{display:inline-block;margin-left:2.5px;margin-right:2.5px}ul.hS6GEzRhW2F__table-documentation .hS6GEzRhW2F__utc-display{flex-basis:20%;flex-grow:1}ul.hS6GEzRhW2F__table-documentation .hS6GEzRhW2F__dot-after::after{content:'\002022';margin-left:5px}.hS6GEzRhW2F__atom-feed-icon{background:url(data:image/gif;base64,R0lGODlhEAAQANUgAOqHCvySDOxZAORxAOBqGdbW1vCtRuKugezFheZlAOiXUdWlZMmNQ754Osh8OOZtJP6bEfuGBdxhAPbizO7YxuesbfHt6tpYAPZ4Av6nGdxqAPtjAOmQMNxNANLS0v///////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACH5BAEAACAALAAAAAAQABAAAAanQJBwSCwOOQONcqkccCxCDsCAqBoyWCzgCdJkFp/wZ3LNagogCYTDYCwm4QpkLvGkIQEAIF+xfCoBEHVpeWFkEU8fHAGDFxEYFBRhBxEVHxQRF3aOjwMNfgoDkhyaIBcYGAOoDmMYlhWlp3ATGAmSCRwfB7EEYgkbkhe5FR12HQKSExsbBBwbGAcKxSAPAgLHy9nLAgR2FgQd4eLj3WggFgUe6uvsaEEAOw==) left no-repeat;background-size:16px 16px;padding-left:19px}</style> <ul class="hS6GEzRhW2F__table-documentation"> <li class="hS6GEzRhW2F__utc-display">All times are US/Pacific</li> </ul> <ul class="hS6GEzRhW2F__table-documentation"> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__documentation"><a href="https://cloud.google.com/support/docs/dashboard">Documentation</a></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__atom-feed"><span class="hS6GEzRhW2F__atom-feed-icon"><a href="https://status.cloud.google.com/en/feed.atom">RSS Feed</a></span></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__json-history"> <a href="https://status.cloud.google.com/incidents.json">JSON History</a> | <a href="https://status.cloud.google.com/incidents.schema.json" aria-label="JSON History Schema">Schema</a></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__json-product-catalog"> <a href="https://status.cloud.google.com/products.json">JSON Product Catalog</a> | <a href="https://status.cloud.google.com/products.schema.json" aria-label="JSON Product Catalog Schema">Schema</a></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__feedback"><a href="#" data-feedback-product-id="104625" onclick="userfeedback.api.startFeedback({'productId': event.target.dataset.feedbackProductId || '5188085'})">Send feedback</a></li> <li class="hS6GEzRhW2F__privacy"><a href="https://policies.google.com/privacy">Privacy policy</a></li> </ul> </psd-table-documentation> </main> </body> </html>