Files
nexus/sreweekly/articles/442/08-google-cloud-incident-report-september-7-incident-in-asia-northeast1.html
2026-09-12 17:23:01 +08:00

96 lines
26 KiB
HTML

<html lang="en"> <head> <meta charset="utf-8"> <title>Incident details | Google Cloud Service Health</title> <link href="https://status.cloud.google.com/images/google-cloud-favicon.png" rel="icon"> <link href="//fonts.googleapis.com/css?family=Open+Sans:300,400,600,700&amp;lang=en" rel="stylesheet" nonce="aNuRXrlzI3u8pr9mubbnqA"> <style>table.psd-table{border-collapse:collapse;font-family:"open sans",arial,sans-serif;font-size:13px;width:100%}table.psd-table thead th{color:#767676;font-weight:normal;padding:0 5px 10px 8px;text-align:left;text-transform:uppercase}table.psd-table td,table.psd-table tbody th{background-color:#fcfcfc;border-bottom:1px solid #eee;border-top:1px solid #eee;font-weight:normal;padding:15px 5px 15px 8px;text-align:left}</style> <script nonce="aNuRXrlzI3u8pr9mubbnqA">(function(){var trackingId='UA-177437757-6';(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');ga('create',trackingId,{'storage':'none' });ga('set','anonymizeIp',true);ga('set','referrer',document.referrer.split('?')[0]);ga('set','location',window.location.href.split("?")[0]);ga('send','pageview');})();</script> <style>body{font-family:"open sans",arial,sans-serif}main{margin:auto;max-width:978px}p{font-size:13px}h2{color:#444;font-size:16px;font-weight:600}h3{color:#444;font-size:14px;font-weight:600}table.status-updates td.status{padding-right:0}table.status-updates .date{padding-right:20px;white-space:nowrap}table.status-updates td.time{padding-right:20px;white-space:nowrap}.location{padding-right:8px}.location::after{content:','}.locations .location:last-of-type::after{content:''}</style> </head> <body> <psd-dashboard-top-bar class="mPVexkcpuTk__intro"><style>header{min-height:64px;display:flex;align-items:center;margin:auto;max-width:978px;justify-content:space-between}.mPVexkcpuTk__header-border{border-bottom:1px solid #eee}header .mPVexkcpuTk__dashboard-logo{float:left;height:40px}</style> <header class="mPVexkcpuTk__header-border" style="max-width:1200px"> <a class="mPVexkcpuTk__logo-link" href="https://status.cloud.google.com/" aria-label="Google Cloud"> <img class="mPVexkcpuTk__dashboard-logo" src="https://status.cloud.google.com/images/google-cloud-logo.svg" alt="Google Cloud"> </a> <div class="mPVexkcpuTk__upper-right-anchor jv6boZDWPRM__widget-container"><style>.jv6boZDWPRM__widget-container{margin:8px 0}.jv6boZDWPRM__widget-container.jv6boZDWPRM__empty{display:none}.jv6boZDWPRM__widget-container.jv6boZDWPRM__no-margin,.jv6boZDWPRM__widget-container.jv6boZDWPRM__no-widget{margin:0}</style> <div> <div> <div><style>.uoGMeAS7ink__widget-link{-webkit-box-align:center;-webkit-align-items:center;-moz-box-align:center;-ms-flex-align:center;align-items:center;border-radius:4px;display:-webkit-box;display:-webkit-flex;display:-moz-box;display:-ms-flexbox;display:flex;-webkit-box-flex:0;-webkit-flex:none;-moz-box-flex:0;-ms-flex:none;flex:none;-webkit-flex-grow:0;-ms-flex-positive:0;flex-grow:0;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-moz-box-orient:horizontal;-moz-box-direction:normal;-ms-flex-direction:row;flex-direction:row;gap:8px;-webkit-box-ordinal-group:2;-webkit-order:1;-moz-box-ordinal-group:2;-ms-flex-order:1;order:1;padding:8px}.uoGMeAS7ink__visually-hidden{border:0;clip:rect(0,0,0,0);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.uoGMeAS7ink__widget-link-text{-webkit-box-align:center;-webkit-align-items:center;-moz-box-align:center;-ms-flex-align:center;align-items:center;color:#5f6368;display:-webkit-box;display:-webkit-flex;display:-moz-box;display:-ms-flexbox;display:flex;-webkit-box-flex:0;-webkit-flex:none;-moz-box-flex:0;-ms-flex:none;flex:none;-webkit-flex-grow:0;-ms-flex-positive:0;flex-grow:0;font-family:Google Sans,open sans,arial,sans-serif;font-style:normal;font-weight:500;font-size:14px;letter-spacing:.25px;line-height:20px;-webkit-box-ordinal-group:1;-webkit-order:0;-moz-box-ordinal-group:1;-ms-flex-order:0;order:0;text-align:center}</style> <a class="uoGMeAS7ink__widget-link" href="https://status.cloud.google.com/redirect/console" target="_blank"> <p class="uoGMeAS7ink__widget-link-text"> Console </p> <svg class="uoGMeAS7ink__widget-link-icon" aria-hidden="true" fill="#5F6368" height="20" viewBox="0 -960 960 960" width="20" xmlns="http://www.w3.org/2000/svg"> <path d="M216-144q-29.7 0-50.85-21.15Q144-186.3 144-216v-528q0-29.7 21.15-50.85Q186.3-816 216-816h264v72H216v528h528v-264h72v264q0 29.7-21.15 50.85Q773.7-144 744-144H216Zm171-192-51-51 357-357H576v-72h240v240h-72v-117L387-336Z"> </path> </svg> <span class="uoGMeAS7ink__visually-hidden">Opens in a new tab</span> </a> </div> </div> </div> </div> </header> </psd-dashboard-top-bar> <main style="max-width:1200px"> <ul class="vQsaEtosvhM__breadcrumbs"><style>.vQsaEtosvhM__breadcrumbs{font-size:11px;margin:15px 0 20px 0;padding:0}.vQsaEtosvhM__breadcrumbs li{display:inline-block;margin:12px 0 0 0;padding:0}.vQsaEtosvhM__breadcrumbs li a{color:black;text-decoration:none}.vQsaEtosvhM__breadcrumbs li a:after{content:"\00203a";font-size:18px;line-height:13px;padding:0 8px;position:relative;top:1px}.vQsaEtosvhM__breadcrumbs li:last-child a:after{content:""}.vQsaEtosvhM__breadcrumbs li.vQsaEtosvhM__incident-description{max-width:340px;overflow:hidden;text-overflow:ellipsis;vertical-align:bottom;white-space:nowrap}</style> <li> <a href="https://status.cloud.google.com/">Google Cloud Service Health</a></li> <li><a href="https://status.cloud.google.com/summary">Incidents</a></li> <li class="vQsaEtosvhM__incident-description">Google Compute Engine (GCE) VM instance creation/ deletion and all related operations for other products were failing in the asia-northeast1 region.</li> </ul> <div class="seE0LugIuG5__header-container"><style>h1{color:#444;font-size:18px;font-weight:600}.seE0LugIuG5__dashboard-title{margin:0}.seE0LugIuG5__header-container{-webkit-box-align:center;-webkit-align-items:center;-moz-box-align:center;-ms-flex-align:center;align-items:center;display:-webkit-box;display:-webkit-flex;display:-moz-box;display:-ms-flexbox;display:flex;margin:.67em 0}</style> <h1 class="seE0LugIuG5__dashboard-title dashboard-header">Service Health</h1> <div class="jv6boZDWPRM__widget-container jv6boZDWPRM__empty"> </div> </div> <div class="aNxXV4vcNrg__intro"><style>a{color:#245dc1;text-decoration:none}a:hover{text-decoration:underline}a:visited{color:#7759ae}.aNxXV4vcNrg__intro{color:#767676;font-size:13px}</style> <div> <span> This page provides status information on the services that are part of Google Cloud. Check back here to view the current status of the services listed below. If you are experiencing an issue not listed here, please <a href="https://cloud.google.com/support-hub/">contact Support</a>. Learn more about what's posted on the dashboard in <a href="https://cloud.google.com/support/docs/dashboard">this FAQ</a>. For additional information on these services, please visit <a href="https://cloud.google.com/">https://cloud.google.com/</a>. </span> <p>For incidents related to Google Security Products, visit <a href="https://status.cloud.google.com/security">https://status.cloud.google.com/security</a>. For incidents related to Looker (original), visit <a href="https://status.cloud.google.com/looker">https://status.cloud.google.com/looker</a>.</p>
</div> <div class="jv6boZDWPRM__widget-container jv6boZDWPRM__empty"> </div> </div> <div class="jv6boZDWPRM__widget-container"> <div> <ul class="nqqp3ZlxVUB__service-status-documentation"><style>ul.nqqp3ZlxVUB__service-status-documentation{display:flex;color:#767676;font-size:13px;list-style:none;padding:0}ul.nqqp3ZlxVUB__service-status-documentation .nqqp3ZlxVUB__with-margin{margin:5px}ul .nqqp3ZlxVUB__with-flex{display:inline-flex}.nqqp3ZlxVUB__with-flex span{margin-left:9px}psd-status-icon{display:inline-block}</style> <li class="nqqp3ZlxVUB__with-margin nqqp3ZlxVUB__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon><span>Available</span> </li> <li class="nqqp3ZlxVUB__with-margin nqqp3ZlxVUB__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__information" fill="none" aria-label="Informational status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm-.833 4.167v1.666h1.667V4.833H8.168Zm0 3.333v5h1.667v-5H8.168Zm-5.833.833A6.676 6.676 0 0 0 9 15.666a6.676 6.676 0 0 0 6.667-6.667A6.676 6.676 0 0 0 9 2.333a6.676 6.676 0 0 0-6.666 6.666Z" fill="#1967D2" fill-rule="evenodd"></path></svg> </psd-status-icon><span>Service information</span> </li> <li class="nqqp3ZlxVUB__with-margin nqqp3ZlxVUB__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__disruption" fill="none" aria-label="Disruption status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M8.168 11.5h1.667v1.666H8.168v-1.667Zm0-6.667h1.667v5H8.168v-5ZM8.993.666C4.393.666.668 4.399.668 8.999s3.725 8.334 8.325 8.334c4.608 0 8.342-3.734 8.342-8.334S13.6.666 8.993.666Zm.008 15a6.665 6.665 0 0 1-6.666-6.667A6.665 6.665 0 0 1 9 2.333a6.665 6.665 0 0 1 6.667 6.666A6.665 6.665 0 0 1 9 15.666Z" fill="#E37400" fill-rule="evenodd"></path></svg> </psd-status-icon><span>Service disruption</span> </li> <li class="nqqp3ZlxVUB__with-margin nqqp3ZlxVUB__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__outage" fill="none" aria-label="Outage status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M11.16 5.666 9 7.824 6.843 5.666 5.668 6.841l2.158 2.158-2.158 2.159 1.175 1.175 2.158-2.159 2.159 2.159 1.175-1.175-2.159-2.159 2.159-2.158-1.175-1.175ZM9 .666A8.326 8.326 0 0 0 .668 8.999a8.326 8.326 0 0 0 8.333 8.334 8.326 8.326 0 0 0 8.334-8.334A8.326 8.326 0 0 0 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Z" fill="#D93025" fill-rule="evenodd"></path></svg> </psd-status-icon><span>Service outage</span> </li> </ul> </div> </div> <p class="incident-header"> Incident affecting Cloud Build, Cloud Developer Tools, Google Cloud Dataflow, Google Cloud Deploy, Google Cloud SQL, Google Compute Engine, Google Kubernetes Engine </p> <h2 class="incident-description">Google Compute Engine (GCE) VM instance creation/ deletion and all related operations for other products were failing in the asia-northeast1 region.</h2> <p> Incident began at <strong class="start-time">2024-09-07 04:20</strong> and ended at <strong class="end-time">2024-09-07 06:10</strong> <span>(all times are <strong class="timezone">US/Pacific</strong>).</span> </p> <h3> Previously affected location(s) </h3> <p class="past-locations locations"> <span class="location">Tokyo (asia-northeast1)</span> </p> <table class="status-updates psd-table"> <thead> <tr> <th class="status" scope="col"></th> <th class="date" scope="col">Date</th> <th class="time" scope="col">Time</th> <th class="description" scope="col">Description</th> </tr> </thead> <tbody> <tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">13 Sep 2024</th> <td class="time">06:44 PDT</td> <td class="description"><h1>Incident Report</h1>
<h2>Summary</h2>
<p>On 7 September 2024 starting at 04:20 US/Pacific, several Google Cloud products experienced a service degradation of varying impact or were unavailable in asia-northeast1 region for a period of 1 hour 50 minutes. The list of impacted products and services is detailed below.</p>
<p>To our Google Cloud customers whose businesses were impacted during this outage, we sincerely apologize. This is not the level of quality and reliability we strive to offer you.</p>
<h2>Root Cause</h2>
<p>Most Google Cloud products and services use a regional metadata store to support their internal operations. The metadata store supports critical functions such as servicing customer requests, load balancing, admin operations, and retrieving/storing metadata including server location information.</p>
<p>Google Compute Engine (GCE) internal DNS depends on the regional metadata store for storing instance metadata. A routine update of the metadata store to a new software version had a change which resulted in poor handling of a rare resource contention corner case, which caused the writes from GCE internal DNS to a zonal replica of the metadata store to fail.</p>
<p>During such zonal issues, we have automated failover mechanisms to use the healthy replicas from other zones. But during this disruption, a secondary issue caused automated failover to not work, rendering the entire metadata storage unavailable despite two other healthy zones being available.</p>
<p>This resulted in disruptions to all GCE instance operations in the asia-northeast1 region. Actions such as creating, deleting, starting, and stopping instances or consuming reservations were affected. This, in turn, affected operations of other services dependent on GCE instances, including GKE, Cloud Build, Cloud Dataflow, Cloud Deploy and Cloud SQL.</p>
<h2>Remediation and Prevention</h2>
<p>Google engineers were alerted to the issue by internal monitoring on 7 September 2024 at 04:26 US/Pacific and immediately started an investigation. The issue was fully mitigated at 06:10 US/Pacific after failover of the metadata storage operations to the healthy zones was manually initiated by our engineering team.</p>
<p>Google is committed to preventing a repeat of this issue in the future and is completing the following actions:</p>
<ul>
<li>Improve the automated failover logic used in the metadata storage infrastructure.</li>
<li>Improve the testing of how the metadata storage service handles resource contention corner cases.</li>
<li>Improve internal processes and documentations to enable faster response and mitigation time for this type of issue.</li>
</ul>
<h2>Detailed Description of Impact</h2>
<ul>
<li>
<p><strong>Google Compute Engine</strong>
Google Compute Engine requests and operations like creating, deleting, starting or modifying VMs were not executed.</p>
</li>
<li>
<p><strong>Cloud Build</strong>
Cloud Build deployments via the Cloud Build API or via Cloud Build Integrations were not being executed.</p>
</li>
<li>
<p><strong>Cloud Dataflow</strong>
Google Cloud Dataflow jobs failed at start up, existing job autoscale functionality was disrupted.</p>
</li>
<li>
<p><strong>Cloud Deploy</strong>
Google Cloud Deploy actions (render, deploy, verify, pre- and post-deploy hooks) were unresponsive and unable to complete their tasks.</p>
</li>
<li>
<p><strong>Google Cloud SQL</strong>
Google Cloud SQL, dependent on Google Compute Engine, experienced operational issues during this incident. No database creation or deletion operations were possible during the incident.</p>
</li>
<li>
<p><strong>Google Kubernetes Engine</strong>
Google Kubernetes Engine experienced issues with operations that depend on GCE. Specifically, cluster and node pool creation, node autoscaling, cluster and node upgrades, and automatic repairs were unsuccessful in the affected region and zones.</p>
</li>
</ul>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">9 Sep 2024</th> <td class="time">06:35 PDT</td> <td class="description"><h1>Mini Incident Report</h1>
<p>We apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below.</p>
<p>Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues.</p>
<p>If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using <a href="https://cloud.google.com/support">https://cloud.google.com/support</a>.</p>
<p>(All Times US/Pacific)</p>
<p><strong>Incident Start:</strong> 07 September, 2024 04:20</p>
<p><strong>Incident End:</strong> 07 September, 2024 06:10</p>
<p><strong>Duration:</strong> 01 hours, 50 minutes</p>
<p><strong>Affected Services and Features:</strong></p>
<ul>
<li>Cloud Build</li>
<li>Google Cloud Dataflow</li>
<li>Google Cloud Deploy</li>
<li>Google Cloud SQL</li>
<li>Google Compute Engine</li>
<li>Google Kubernetes Engine</li>
</ul>
<p><strong>Regions/Zones:</strong> asia-northeast1</p>
<p><strong>Description:</strong></p>
<p>Several Google Cloud products experienced a service degradation of varying impact, or were unavailable for a duration of 01 hours, 50 minutes in asia-northeast1 region.</p>
<p>Google Engineers have identified the cause to be a change rollout to an internal component. This change was subsequently rolled back which mitigated all known impacts.</p>
<p>Google will complete a full Incident Report (IR) in the following days that will provide a full root cause.</p>
<p><strong>Customer Impact:</strong></p>
<p>Through the incident duration, the impacted Google Cloud services experienced different kinds of service degradations as detailed below.</p>
<ul>
<li>Google Compute Engine requests and operations like creating, deleting, starting or modifying VMs were not executed.</li>
<li>Cloud Build deployed via the Cloud Build API or via CloudBuild Integrations were not being executed.</li>
<li>Google Cloud Dataflow jobs failed at start up, existing job autoscale functionality was disrupted.</li>
<li>Google Cloud Deploy actions (render, deploy, verify, pre- and post-deploy hooks) were unresponsive and unable to complete their tasks.</li>
<li>Google Cloud SQL, dependent on Google Compute Engine, which experienced operational issues during this incident. No databases were able to be created or deleted during the incident.</li>
</ul>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">7 Sep 2024</th> <td class="time">07:15 PDT</td> <td class="description"><p>The issue with Cloud Build, Google Cloud Dataflow, Google Cloud Deploy, Google Cloud SQL, Google Compute Engine, Google Kubernetes Engine has been resolved for all affected customers as of Saturday, 2024-09-07 06:10 US/Pacific.</p>
<p>We will publish an analysis of this incident once we have completed our internal investigation.</p>
<p>We thank you for your patience while we worked on resolving the issue.</p>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__outage" fill="none" aria-label="Outage status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M11.16 5.666 9 7.824 6.843 5.666 5.668 6.841l2.158 2.158-2.158 2.159 1.175 1.175 2.158-2.159 2.159 2.159 1.175-1.175-2.159-2.159 2.159-2.158-1.175-1.175ZM9 .666A8.326 8.326 0 0 0 .668 8.999a8.326 8.326 0 0 0 8.333 8.334 8.326 8.326 0 0 0 8.334-8.334A8.326 8.326 0 0 0 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Z" fill="#D93025" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">7 Sep 2024</th> <td class="time">07:13 PDT</td> <td class="description"><p>Summary: Google Compute Engine (GCE) VM instance creation/ deletion and all related operations for other products were failing in the asia-northeast1 region.</p>
<p>Description: Engineering team has rolled out the fix which mitigated the impact on Saturday, 2024-09-07 06:10 US/Pacific.</p>
<p>We will provide more information by Saturday, 2024-09-07 07:30 US/Pacific.</p>
<p>Diagnosis: GCE Customers impacted by this issue were experiencing &#34;Internal error. Please try again or contact Google Support. (Code: &#39;-1343002181035865699&#39;)&#34; while attempting instance creation.</p>
<p>Cloud Build customers were observing their builds not being executed.</p>
<p>Google Cloud Dataflow customers were unable to create jobs or scale existing jobs.</p>
<p>Workaround: Customers may retry their operation in case they experience failures.</p>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__outage" fill="none" aria-label="Outage status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M11.16 5.666 9 7.824 6.843 5.666 5.668 6.841l2.158 2.158-2.158 2.159 1.175 1.175 2.158-2.159 2.159 2.159 1.175-1.175-2.159-2.159 2.159-2.158-1.175-1.175ZM9 .666A8.326 8.326 0 0 0 .668 8.999a8.326 8.326 0 0 0 8.333 8.334 8.326 8.326 0 0 0 8.334-8.334A8.326 8.326 0 0 0 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Z" fill="#D93025" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">7 Sep 2024</th> <td class="time">06:11 PDT</td> <td class="description"><p>Summary: Google Compute Engine (GCE) VM instance creation/ deletion operations, Google Cloud Dataflow and Cloud Build are failing in the asia-northeast1 region.</p>
<p>Description: Mitigation work is currently underway by our engineering teams.</p>
<p>We do not have an ETA for mitigation at this point.</p>
<p>We will provide more information by Saturday, 2024-09-07 07:30 US/Pacific.</p>
<p>Diagnosis: GCE Customers impacted by this issue may experience &#34;Internal error. Please try again or contact Google Support. (Code: &#39;-1343002181035865699&#39;)&#34; while attempting instance creation.</p>
<p>Cloud Build customers may observe their builds not being executed.</p>
<p>Google Cloud Dataflow customers are unable to create jobs or scale existing jobs.</p>
<p>Workaround: Customers may attempt their GCVE, Cloud Build and Cloud Dataflow operations in other region.</p>
</td> </tr> </tbody> </table> <psd-table-documentation><script src="https://support.google.com/inapp/api.js" nonce="aNuRXrlzI3u8pr9mubbnqA"></script><style>ul.hS6GEzRhW2F__table-documentation{color:#767676;display:flex;flex-wrap:wrap;font-size:13px;list-style:none;margin-left:2.5px;margin-right:2.5px;padding:0}ul.hS6GEzRhW2F__table-documentation li{display:inline-block;margin-left:2.5px;margin-right:2.5px}ul.hS6GEzRhW2F__table-documentation .hS6GEzRhW2F__utc-display{flex-basis:20%;flex-grow:1}ul.hS6GEzRhW2F__table-documentation .hS6GEzRhW2F__dot-after::after{content:'\002022';margin-left:5px}.hS6GEzRhW2F__atom-feed-icon{background:url(data:image/gif;base64,R0lGODlhEAAQANUgAOqHCvySDOxZAORxAOBqGdbW1vCtRuKugezFheZlAOiXUdWlZMmNQ754Osh8OOZtJP6bEfuGBdxhAPbizO7YxuesbfHt6tpYAPZ4Av6nGdxqAPtjAOmQMNxNANLS0v///////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACH5BAEAACAALAAAAAAQABAAAAanQJBwSCwOOQONcqkccCxCDsCAqBoyWCzgCdJkFp/wZ3LNagogCYTDYCwm4QpkLvGkIQEAIF+xfCoBEHVpeWFkEU8fHAGDFxEYFBRhBxEVHxQRF3aOjwMNfgoDkhyaIBcYGAOoDmMYlhWlp3ATGAmSCRwfB7EEYgkbkhe5FR12HQKSExsbBBwbGAcKxSAPAgLHy9nLAgR2FgQd4eLj3WggFgUe6uvsaEEAOw==) left no-repeat;background-size:16px 16px;padding-left:19px}</style> <ul class="hS6GEzRhW2F__table-documentation"> <li class="hS6GEzRhW2F__utc-display">All times are US/Pacific</li> </ul> <ul class="hS6GEzRhW2F__table-documentation"> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__documentation"><a href="https://cloud.google.com/support/docs/dashboard">Documentation</a></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__atom-feed"><span class="hS6GEzRhW2F__atom-feed-icon"><a href="https://status.cloud.google.com/en/feed.atom">RSS Feed</a></span></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__json-history"> <a href="https://status.cloud.google.com/incidents.json">JSON History</a> | <a href="https://status.cloud.google.com/incidents.schema.json" aria-label="JSON History Schema">Schema</a></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__json-product-catalog"> <a href="https://status.cloud.google.com/products.json">JSON Product Catalog</a> | <a href="https://status.cloud.google.com/products.schema.json" aria-label="JSON Product Catalog Schema">Schema</a></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__feedback"><a href="#" data-feedback-product-id="104625" onclick="userfeedback.api.startFeedback({'productId': event.target.dataset.feedbackProductId || '5188085'})">Send feedback</a></li> <li class="hS6GEzRhW2F__privacy"><a href="https://policies.google.com/privacy">Privacy policy</a></li> </ul> </psd-table-documentation> </main> </body> </html>