Files
nexus/sreweekly/articles/488/08-google-cloud-platform-incident-report-july-18-2025.html
2026-09-12 17:23:01 +08:00

69 lines
28 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<html lang="en"> <head> <meta charset="utf-8"> <title>Incident details | Google Cloud Service Health</title> <link href="https://status.cloud.google.com/images/google-cloud-favicon.png" rel="icon"> <link href="//fonts.googleapis.com/css?family=Open+Sans:300,400,600,700&amp;lang=en" rel="stylesheet" nonce="7Ekj-4cnL6uzuzRH8uu-0g"> <style>table.psd-table{border-collapse:collapse;font-family:"open sans",arial,sans-serif;font-size:13px;width:100%}table.psd-table thead th{color:#767676;font-weight:normal;padding:0 5px 10px 8px;text-align:left;text-transform:uppercase}table.psd-table td,table.psd-table tbody th{background-color:#fcfcfc;border-bottom:1px solid #eee;border-top:1px solid #eee;font-weight:normal;padding:15px 5px 15px 8px;text-align:left}</style> <script nonce="7Ekj-4cnL6uzuzRH8uu-0g">(function(){var trackingId='UA-177437757-6';(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');ga('create',trackingId,{'storage':'none' });ga('set','anonymizeIp',true);ga('set','referrer',document.referrer.split('?')[0]);ga('set','location',window.location.href.split("?")[0]);ga('send','pageview');})();</script> <style>body{font-family:"open sans",arial,sans-serif}main{margin:auto;max-width:978px}p{font-size:13px}h2{color:#444;font-size:16px;font-weight:600}h3{color:#444;font-size:14px;font-weight:600}table.status-updates td.status{padding-right:0}table.status-updates .date{padding-right:20px;white-space:nowrap}table.status-updates td.time{padding-right:20px;white-space:nowrap}.location{padding-right:8px}.location::after{content:','}.locations .location:last-of-type::after{content:''}</style> </head> <body> <psd-dashboard-top-bar class="mPVexkcpuTk__intro"><style>header{min-height:64px;display:flex;align-items:center;margin:auto;max-width:978px;justify-content:space-between}.mPVexkcpuTk__header-border{border-bottom:1px solid #eee}header .mPVexkcpuTk__dashboard-logo{float:left;height:40px}</style> <header class="mPVexkcpuTk__header-border" style="max-width:1200px"> <a class="mPVexkcpuTk__logo-link" href="https://status.cloud.google.com/" aria-label="Google Cloud"> <img class="mPVexkcpuTk__dashboard-logo" src="https://status.cloud.google.com/images/google-cloud-logo.svg" alt="Google Cloud"> </a> <div class="mPVexkcpuTk__upper-right-anchor jv6boZDWPRM__widget-container"><style>.jv6boZDWPRM__widget-container{margin:8px 0}.jv6boZDWPRM__widget-container.jv6boZDWPRM__empty{display:none}.jv6boZDWPRM__widget-container.jv6boZDWPRM__no-margin,.jv6boZDWPRM__widget-container.jv6boZDWPRM__no-widget{margin:0}</style> <div> <div> <div><style>.uoGMeAS7ink__widget-link{-webkit-box-align:center;-webkit-align-items:center;-moz-box-align:center;-ms-flex-align:center;align-items:center;border-radius:4px;display:-webkit-box;display:-webkit-flex;display:-moz-box;display:-ms-flexbox;display:flex;-webkit-box-flex:0;-webkit-flex:none;-moz-box-flex:0;-ms-flex:none;flex:none;-webkit-flex-grow:0;-ms-flex-positive:0;flex-grow:0;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-moz-box-orient:horizontal;-moz-box-direction:normal;-ms-flex-direction:row;flex-direction:row;gap:8px;-webkit-box-ordinal-group:2;-webkit-order:1;-moz-box-ordinal-group:2;-ms-flex-order:1;order:1;padding:8px}.uoGMeAS7ink__visually-hidden{border:0;clip:rect(0,0,0,0);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.uoGMeAS7ink__widget-link-text{-webkit-box-align:center;-webkit-align-items:center;-moz-box-align:center;-ms-flex-align:center;align-items:center;color:#5f6368;display:-webkit-box;display:-webkit-flex;display:-moz-box;display:-ms-flexbox;display:flex;-webkit-box-flex:0;-webkit-flex:none;-moz-box-flex:0;-ms-flex:none;flex:none;-webkit-flex-grow:0;-ms-flex-positive:0;flex-grow:0;font-family:Google Sans,open sans,arial,sans-serif;font-style:normal;font-weight:500;font-size:14px;letter-spacing:.25px;line-height:20px;-webkit-box-ordinal-group:1;-webkit-order:0;-moz-box-ordinal-group:1;-ms-flex-order:0;order:0;text-align:center}</style> <a class="uoGMeAS7ink__widget-link" href="https://status.cloud.google.com/redirect/console" target="_blank"> <p class="uoGMeAS7ink__widget-link-text"> Console </p> <svg class="uoGMeAS7ink__widget-link-icon" aria-hidden="true" fill="#5F6368" height="20" viewBox="0 -960 960 960" width="20" xmlns="http://www.w3.org/2000/svg"> <path d="M216-144q-29.7 0-50.85-21.15Q144-186.3 144-216v-528q0-29.7 21.15-50.85Q186.3-816 216-816h264v72H216v528h528v-264h72v264q0 29.7-21.15 50.85Q773.7-144 744-144H216Zm171-192-51-51 357-357H576v-72h240v240h-72v-117L387-336Z"> </path> </svg> <span class="uoGMeAS7ink__visually-hidden">Opens in a new tab</span> </a> </div> </div> </div> </div> </header> </psd-dashboard-top-bar> <main style="max-width:1200px"> <ul class="vQsaEtosvhM__breadcrumbs"><style>.vQsaEtosvhM__breadcrumbs{font-size:11px;margin:15px 0 20px 0;padding:0}.vQsaEtosvhM__breadcrumbs li{display:inline-block;margin:12px 0 0 0;padding:0}.vQsaEtosvhM__breadcrumbs li a{color:black;text-decoration:none}.vQsaEtosvhM__breadcrumbs li a:after{content:"\00203a";font-size:18px;line-height:13px;padding:0 8px;position:relative;top:1px}.vQsaEtosvhM__breadcrumbs li:last-child a:after{content:""}.vQsaEtosvhM__breadcrumbs li.vQsaEtosvhM__incident-description{max-width:340px;overflow:hidden;text-overflow:ellipsis;vertical-align:bottom;white-space:nowrap}</style> <li> <a href="https://status.cloud.google.com/">Google Cloud Service Health</a></li> <li><a href="https://status.cloud.google.com/summary">Incidents</a></li> <li class="vQsaEtosvhM__incident-description">We are investigating elevated error rates with multiple products in us-east1</li> </ul> <div class="seE0LugIuG5__header-container"><style>h1{color:#444;font-size:18px;font-weight:600}.seE0LugIuG5__dashboard-title{margin:0}.seE0LugIuG5__header-container{-webkit-box-align:center;-webkit-align-items:center;-moz-box-align:center;-ms-flex-align:center;align-items:center;display:-webkit-box;display:-webkit-flex;display:-moz-box;display:-ms-flexbox;display:flex;margin:.67em 0}</style> <h1 class="seE0LugIuG5__dashboard-title dashboard-header">Service Health</h1> <div class="jv6boZDWPRM__widget-container jv6boZDWPRM__empty"> </div> </div> <div class="aNxXV4vcNrg__intro"><style>a{color:#245dc1;text-decoration:none}a:hover{text-decoration:underline}a:visited{color:#7759ae}.aNxXV4vcNrg__intro{color:#767676;font-size:13px}</style> <div> <span> This page provides status information on the services that are part of Google Cloud. Check back here to view the current status of the services listed below. If you are experiencing an issue not listed here, please <a href="https://cloud.google.com/support-hub/">contact Support</a>. Learn more about what's posted on the dashboard in <a href="https://cloud.google.com/support/docs/dashboard">this FAQ</a>. For additional information on these services, please visit <a href="https://cloud.google.com/">https://cloud.google.com/</a>. </span> <p>For incidents related to Google Security Products, visit <a href="https://status.cloud.google.com/security">https://status.cloud.google.com/security</a>. For incidents related to Looker (original), visit <a href="https://status.cloud.google.com/looker">https://status.cloud.google.com/looker</a>.</p>
</div> <div class="jv6boZDWPRM__widget-container jv6boZDWPRM__empty"> </div> </div> <div class="jv6boZDWPRM__widget-container"> <div> <ul class="bOgDAiOLw34__service-status-documentation"><style>ul.bOgDAiOLw34__service-status-documentation{display:-webkit-box;display:-webkit-flex;display:-moz-box;display:-ms-flexbox;display:flex;color:#767676;font-size:13px;list-style:none;padding:0}ul.bOgDAiOLw34__service-status-documentation .bOgDAiOLw34__with-margin{margin:5px}ul .bOgDAiOLw34__with-flex{display:-webkit-inline-box;display:-webkit-inline-flex;display:-moz-inline-box;display:-ms-inline-flexbox;display:inline-flex}.bOgDAiOLw34__with-flex span{margin-left:9px}psd-status-icon{display:inline-block}</style> <li class="bOgDAiOLw34__with-margin bOgDAiOLw34__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> <span>Available</span> </li> <li class="bOgDAiOLw34__with-margin bOgDAiOLw34__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__disruption" fill="none" aria-label="Disruption status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M8.168 11.5h1.667v1.666H8.168v-1.667Zm0-6.667h1.667v5H8.168v-5ZM8.993.666C4.393.666.668 4.399.668 8.999s3.725 8.334 8.325 8.334c4.608 0 8.342-3.734 8.342-8.334S13.6.666 8.993.666Zm.008 15a6.665 6.665 0 0 1-6.666-6.667A6.665 6.665 0 0 1 9 2.333a6.665 6.665 0 0 1 6.667 6.666A6.665 6.665 0 0 1 9 15.666Z" fill="#E37400" fill-rule="evenodd"></path></svg> </psd-status-icon> <span>Service disruption</span> </li> <li class="bOgDAiOLw34__with-margin bOgDAiOLw34__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__outage" fill="none" aria-label="Outage status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M11.16 5.666 9 7.824 6.843 5.666 5.668 6.841l2.158 2.158-2.158 2.159 1.175 1.175 2.158-2.159 2.159 2.159 1.175-1.175-2.159-2.159 2.159-2.158-1.175-1.175ZM9 .666A8.326 8.326 0 0 0 .668 8.999a8.326 8.326 0 0 0 8.333 8.334 8.326 8.326 0 0 0 8.334-8.334A8.326 8.326 0 0 0 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Z" fill="#D93025" fill-rule="evenodd"></path></svg> </psd-status-icon> <span>Service outage</span> </li> </ul> </div> </div> <p class="incident-header"> Incident affecting Agent Platform Online Inference, AlloyDB for PostgreSQL, Apigee, Artifact Registry, Certificate Authority Service, Cloud Armor, Cloud Billing, Cloud Build, Cloud External Key Manager, Cloud Firestore, Cloud HSM, Cloud Key Management Service, Cloud Load Balancing, Cloud Memorystore, Cloud Monitoring, Cloud Run, Cloud Spanner, Cloud Storage for Firebase, Cloud Workflows, Database Migration Service, Dataproc Metastore, Dialogflow CX, Eventarc, Google App Engine, Google BigQuery, Google Cloud Bigtable, Google Cloud Console, Google Cloud Dataflow, Google Cloud Dataproc, Google Cloud Pub/Sub, Google Cloud SQL, Google Cloud Storage, Google Cloud Support, Google Cloud Tasks, Google Compute Engine, Google Kubernetes Engine, Hybrid Connectivity, Identity and Access Management, Media CDN, Memorystore for Memcached, Memorystore for Redis, Memorystore for Redis Cluster, Persistent Disk, Private Service Connect, Secret Manager, Service Directory, Virtual Private Cloud (VPC) </p> <h2 class="incident-description">We are investigating elevated error rates with multiple products in us-east1</h2> <p> Incident began at <strong class="start-time">2025-07-18 07:42</strong> and ended at <strong class="end-time">2025-07-18 09:47</strong> <span>(all times are <strong class="timezone">US/Pacific</strong>).</span> </p> <h3> Previously affected location(s) </h3> <p class="past-locations locations"> <span class="location">South Carolina (us-east1)</span> </p> <table class="status-updates psd-table"> <thead> <tr> <th class="status" scope="col"></th> <th class="date" scope="col">Date</th> <th class="time" scope="col">Time</th> <th class="description" scope="col">Description</th> </tr> </thead> <tbody> <tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">22 Jul 2025</th> <td class="time">06:42 PDT</td> <td class="description"><h2># Incident Report</h2>
<h2>## Summary</h2>
<p>On Friday, 18 July 2025 07:50 US/Pacific, several Google Cloud Platform (GCP) and Google Workspace (GWS) products experienced elevated latencies and error rates in the us-east1 region for a duration of up to 1 hour and 57 minutes.</p>
<p><strong>GCP Impact Duration:</strong> 18 July 2025 07:50 - 09:47 US/Pacific : 1 hour 57 minutes<br>
<strong>GWS Impact Duration:</strong> 18 July 2025 07:50 - 08:40 US/Pacific : 50 minutes</p>
<p>We sincerely apologize for this incident, which does not reflect the level of quality and reliability we strive to offer. We are taking immediate steps to improve the platform’s performance and availability.</p>
<h2></h2>
<h2>## Root Cause</h2>
<p>The service interruption was triggered by a procedural error during a planned hardware replacement in our datacenter. An incorrect physical disconnection was made to the active network switch serving our control plane, rather than the redundant unit scheduled for removal. The redundant unit had been properly de-configured as part of the procedure, and the combination of these two events led to partitioning of the network control plane. Our network is designed to withstand this type of control plane failure by failing open, continuing operation.</p>
<p>However, an operational topology change while the network control plane was in a failed open state caused our network fabric&#39;s topology information to become stale. This led to packet loss and service disruption until services were moved away from the fabric and control plane connectivity was restored.</p>
<h2>## Remediation and Prevention</h2>
<p>Google engineers were alerted to the outage by our monitoring system on 18 July 2025 07:06 US/Pacific and immediately started an investigation. The following timeline details the remediation and restoration efforts:</p>
<ul>
<li><strong>07:39 US/Pacific</strong>: The underlying root cause (device disconnect) was identified and onsite technicians were engaged to reconnect the control plane device and restore control plane connectivity. At that moment, network failure open mechanisms worked as expected and no impact was observed.</li>
<li><strong>07:50 US/Pacific</strong>: A topology change led to traffic being routed suboptimally, due to the network being in a fail open state. This caused congestion on the subset of links, packet loss, and latency to customer traffic. Engineers made a decision to move traffic away from the affected fabric, which mitigated the impact for the majority of the services.</li>
<li><strong>08:40 US/Pacific</strong>: Engineers mitigated Workspace impact by shifting traffic away from the affected region.</li>
<li><strong>09:47 US/Pacific</strong>: Onsite technicians reconnected the device, control plane connectivity was fully restored and all services were back to stable state.</li>
</ul>
<p>Google is committed to preventing a repeat of the issue in the future, and is completing the following actions:</p>
<ul>
<li>Pause non-critical workflows until safety controls are implemented (complete).</li>
<li>Strengthen safety controls for hardware upgrade workflows by end of Q3 2025.</li>
<li>Design and implement a mechanism to prevent control plane partitioning in case of dual failure of upstream routers by end of Q4 2025.</li>
</ul>
<h2>## Detailed Description of Impact</h2>
<p>### GCP Impact:</p>
<p>Multiple products in us-east1 were affected by the loss of network connectivity, with the most significant impacts seen in us-east1-b. Other regions were not affected.</p>
<p>The outage caused a range of issues for customers with zonal resources in the region, including packet loss across VPC networks, increased error rates and latency, service unavailable (503) errors, and slow or stuck operations up to loss of networking connectivity. While regional products were briefly impacted, they recovered quickly by failing over to unaffected zones.</p>
<p>A small number (0.1%) of Persistent Disks in us-east1-b were unavailable for the duration of the outage: these disks became available once the outage was mitigated, with no customer data loss.</p>
<p>### GWS Impact:</p>
<p>A small subset of Workspace users, primarily around the Southeast US, experienced varying degrees of unavailability and increased delays across multiple products, including Gmail, Google Meet, Google Drive, Google Chat, Google Calendar, Google Groups, Google Doc/Editors, and Google Voice.</p>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">18 Jul 2025</th> <td class="time">15:08 PDT</td> <td class="description"><h1>Mini Incident Report</h1>
<p>We apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using <a href="https://cloud.google.com/support">https://cloud.google.com/support</a> or to Google Workspace Support using help article <a href="https://support.google.com/a/answer/1047213">https://support.google.com/a/answer/1047213</a>.</p>
<p>(All Times US/Pacific)</p>
<p><strong>GCP Impact start and end time:</strong> 18 July 2025 08:10 - 09:47
<strong>Duration:</strong> 1 hour 37 minutes</p>
<p><strong>GWS Impact start and end time:</strong> 18 July 2025 08:10 - 08:40
<strong>Duration:</strong> 30 minutes</p>
<p><strong>Regions/Zones:</strong> us-east1</p>
<p><strong>Description:</strong>
On Friday, 18 July 2025 08:10 US/Pacific multiple GCP and GWS products experienced elevated latencies and error rates in the us-east1 region for a duration of up to 1 hour and 37 minutes.</p>
<p>Based on the preliminary analysis, the root cause of the issue is a procedural error during a planned hardware maintenance in one of our data centers in the us-east1 region. Our engineering team mitigated the issue by draining traffic away from the clusters and then restoring the affected hardware.</p>
<p>Google will be completing a full incident report in the following days that will provide a full root cause and preventive actions.</p>
<p><strong>Customer Impact:</strong>
The affected GCP and GWS products experienced elevated latencies and errors rates in the us-east1 region.</p>
<p><strong>Affected Products:</strong></p>
<p><strong>GCP :</strong>
AlloyDB for PostgreSQL, Apigee, Artifact Registry, Cloud Armor, Cloud Billing, Cloud Build, Cloud External Key Manager, Cloud Filestore, Cloud HSM, Cloud Key Management Service, Cloud Load Balancing, Cloud Monitoring, Cloud Run, Cloud Spanner, Cloud Storage for Firebase, Cloud Workflows, Database Migration Service, Dialogflow CX, Dialogflow ES, Google BigQuery, Google Cloud Dataflow, Google Cloud Dataproc, Google Cloud Storage, Google Cloud Support, Google Cloud Tasks, Google Compute Engine, Hybrid Connectivity, Media CDN, Network Telemetry, Private Service Connect, Secret Manager, Service Directory, Vertex AI Online Prediction, Virtual Private Cloud (VPC)</p>
<p><strong>Workspace :</strong>
Gmail, Google Meet, Google Drive, Google Chat, Google Calendar, Google Groups, Google Doc/Editors, Google Voice</p>
<p><strong>Google SecOps:</strong>
Google SecOps SOAR &amp; Google SecOps</p>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">18 Jul 2025</th> <td class="time">11:03 PDT</td> <td class="description"><p>The issue has been resolved for all affected products as of 2025-07-18 09:47 US/Pacific.</p>
<p>From preliminary analysis, during a routine maintenance of our network in us-east1-b, we experienced elevated packet loss, causing service disruption in the zone.</p>
<p>We will publish a full Incident Report with root cause once we have completed our internal investigations.</p>
<p>We thank you for your patience while we worked on resolving the issue.</p>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__disruption" fill="none" aria-label="Disruption status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M8.168 11.5h1.667v1.666H8.168v-1.667Zm0-6.667h1.667v5H8.168v-5ZM8.993.666C4.393.666.668 4.399.668 8.999s3.725 8.334 8.325 8.334c4.608 0 8.342-3.734 8.342-8.334S13.6.666 8.993.666Zm.008 15a6.665 6.665 0 0 1-6.666-6.667A6.665 6.665 0 0 1 9 2.333a6.665 6.665 0 0 1 6.667 6.666A6.665 6.665 0 0 1 9 15.666Z" fill="#E37400" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">18 Jul 2025</th> <td class="time">10:32 PDT</td> <td class="description"><p>Our engineers have successfully recovered the network control plane in the affected us-east1 zones.</p>
<p>We&#39;re seeing multiple services reporting full recovery, and product engineers continue to validate the remaining services.</p>
<p>We&#39;ll provide another update with more details by 11:00 AM US/Pacific, July 18, 2025.</p>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__disruption" fill="none" aria-label="Disruption status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M8.168 11.5h1.667v1.666H8.168v-1.667Zm0-6.667h1.667v5H8.168v-5ZM8.993.666C4.393.666.668 4.399.668 8.999s3.725 8.334 8.325 8.334c4.608 0 8.342-3.734 8.342-8.334S13.6.666 8.993.666Zm.008 15a6.665 6.665 0 0 1-6.666-6.667A6.665 6.665 0 0 1 9 2.333a6.665 6.665 0 0 1 6.667 6.666A6.665 6.665 0 0 1 9 15.666Z" fill="#E37400" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">18 Jul 2025</th> <td class="time">09:58 PDT</td> <td class="description"><p>Our engineers have successfully recovered the network control plane in the affected us-east1 zones. We&#39;re seeing multiple services reporting full recovery, and product engineers are now validating the remaining services.</p>
<p>We&#39;ll provide another update with more details by 10:30 AM US/Pacific, July 18, 2025.</p>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__disruption" fill="none" aria-label="Disruption status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M8.168 11.5h1.667v1.666H8.168v-1.667Zm0-6.667h1.667v5H8.168v-5ZM8.993.666C4.393.666.668 4.399.668 8.999s3.725 8.334 8.325 8.334c4.608 0 8.342-3.734 8.342-8.334S13.6.666 8.993.666Zm.008 15a6.665 6.665 0 0 1-6.666-6.667A6.665 6.665 0 0 1 9 2.333a6.665 6.665 0 0 1 6.667 6.666A6.665 6.665 0 0 1 9 15.666Z" fill="#E37400" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">18 Jul 2025</th> <td class="time">09:29 PDT</td> <td class="description"><p>Our engineers have confirmed that us-east1-b is partially affected. All other zones in us-east1 are currently operating normally.</p>
<p>Our engineers have recovered the failed hardware and are currently recovering the network control plane in the affected zones.</p>
<p>We&#39;ll provide another update by 10:00 AM US/Pacific, July 18, 2025.</p>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__disruption" fill="none" aria-label="Disruption status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M8.168 11.5h1.667v1.666H8.168v-1.667Zm0-6.667h1.667v5H8.168v-5ZM8.993.666C4.393.666.668 4.399.668 8.999s3.725 8.334 8.325 8.334c4.608 0 8.342-3.734 8.342-8.334S13.6.666 8.993.666Zm.008 15a6.665 6.665 0 0 1-6.666-6.667A6.665 6.665 0 0 1 9 2.333a6.665 6.665 0 0 1 6.667 6.666A6.665 6.665 0 0 1 9 15.666Z" fill="#E37400" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">18 Jul 2025</th> <td class="time">08:54 PDT</td> <td class="description"><p>We&#39;re currently experiencing elevated latency and error rates for several Cloud services in the us-east1 region, beginning at 7:06 AM PDT today, July 18, 2025. Our initial investigation points to a hardware infrastructure failure as the likely cause.</p>
<p>We apologize for any disruption this may be causing. We&#39;ll provide an update with more details by 9:15 AM PDT today.</p>
</td> </tr> </tbody> </table> <psd-table-documentation><script src="https://support.google.com/inapp/api.js" nonce="7Ekj-4cnL6uzuzRH8uu-0g"></script><style>ul.hS6GEzRhW2F__table-documentation{color:#767676;display:flex;flex-wrap:wrap;font-size:13px;list-style:none;margin-left:2.5px;margin-right:2.5px;padding:0}ul.hS6GEzRhW2F__table-documentation li{display:inline-block;margin-left:2.5px;margin-right:2.5px}ul.hS6GEzRhW2F__table-documentation .hS6GEzRhW2F__utc-display{flex-basis:20%;flex-grow:1}ul.hS6GEzRhW2F__table-documentation .hS6GEzRhW2F__dot-after::after{content:'\002022';margin-left:5px}.hS6GEzRhW2F__atom-feed-icon{background:url(data:image/gif;base64,R0lGODlhEAAQANUgAOqHCvySDOxZAORxAOBqGdbW1vCtRuKugezFheZlAOiXUdWlZMmNQ754Osh8OOZtJP6bEfuGBdxhAPbizO7YxuesbfHt6tpYAPZ4Av6nGdxqAPtjAOmQMNxNANLS0v///////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACH5BAEAACAALAAAAAAQABAAAAanQJBwSCwOOQONcqkccCxCDsCAqBoyWCzgCdJkFp/wZ3LNagogCYTDYCwm4QpkLvGkIQEAIF+xfCoBEHVpeWFkEU8fHAGDFxEYFBRhBxEVHxQRF3aOjwMNfgoDkhyaIBcYGAOoDmMYlhWlp3ATGAmSCRwfB7EEYgkbkhe5FR12HQKSExsbBBwbGAcKxSAPAgLHy9nLAgR2FgQd4eLj3WggFgUe6uvsaEEAOw==) left no-repeat;background-size:16px 16px;padding-left:19px}</style> <ul class="hS6GEzRhW2F__table-documentation"> <li class="hS6GEzRhW2F__utc-display">All times are US/Pacific</li> </ul> <ul class="hS6GEzRhW2F__table-documentation"> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__documentation"><a href="https://cloud.google.com/support/docs/dashboard">Documentation</a></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__atom-feed"><span class="hS6GEzRhW2F__atom-feed-icon"><a href="https://status.cloud.google.com/en/feed.atom">RSS Feed</a></span></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__json-history"> <a href="https://status.cloud.google.com/incidents.json">JSON History</a> | <a href="https://status.cloud.google.com/incidents.schema.json" aria-label="JSON History Schema">Schema</a></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__json-product-catalog"> <a href="https://status.cloud.google.com/products.json">JSON Product Catalog</a> | <a href="https://status.cloud.google.com/products.schema.json" aria-label="JSON Product Catalog Schema">Schema</a></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__feedback"><a href="#" data-feedback-product-id="104625" onclick="userfeedback.api.startFeedback({'productId': event.target.dataset.feedbackProductId || '5188085'})">Send feedback</a></li> <li class="hS6GEzRhW2F__privacy"><a href="https://policies.google.com/privacy">Privacy policy</a></li> </ul> </psd-table-documentation> </main> </body> </html>