Files
nexus/sreweekly/articles/456/07-google-cloud-incident-report-bigquery-incident-on-december-4-2024.html
2026-09-12 17:23:01 +08:00

70 lines
29 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<html lang="en"> <head> <meta charset="utf-8"> <title>Incident details | Google Cloud Service Health</title> <link href="https://status.cloud.google.com/images/google-cloud-favicon.png" rel="icon"> <link href="//fonts.googleapis.com/css?family=Open+Sans:300,400,600,700&amp;lang=en" rel="stylesheet" nonce="6udoI7VDq3ImaEUTbqrUYA"> <style>table.psd-table{border-collapse:collapse;font-family:"open sans",arial,sans-serif;font-size:13px;width:100%}table.psd-table thead th{color:#767676;font-weight:normal;padding:0 5px 10px 8px;text-align:left;text-transform:uppercase}table.psd-table td,table.psd-table tbody th{background-color:#fcfcfc;border-bottom:1px solid #eee;border-top:1px solid #eee;font-weight:normal;padding:15px 5px 15px 8px;text-align:left}</style> <script nonce="6udoI7VDq3ImaEUTbqrUYA">(function(){var trackingId='UA-177437757-6';(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');ga('create',trackingId,{'storage':'none' });ga('set','anonymizeIp',true);ga('set','referrer',document.referrer.split('?')[0]);ga('set','location',window.location.href.split("?")[0]);ga('send','pageview');})();</script> <style>body{font-family:"open sans",arial,sans-serif}main{margin:auto;max-width:978px}p{font-size:13px}h2{color:#444;font-size:16px;font-weight:600}h3{color:#444;font-size:14px;font-weight:600}table.status-updates td.status{padding-right:0}table.status-updates .date{padding-right:20px;white-space:nowrap}table.status-updates td.time{padding-right:20px;white-space:nowrap}.location{padding-right:8px}.location::after{content:','}.locations .location:last-of-type::after{content:''}</style> </head> <body> <psd-dashboard-top-bar class="mPVexkcpuTk__intro"><style>header{min-height:64px;display:flex;align-items:center;margin:auto;max-width:978px;justify-content:space-between}.mPVexkcpuTk__header-border{border-bottom:1px solid #eee}header .mPVexkcpuTk__dashboard-logo{float:left;height:40px}</style> <header class="mPVexkcpuTk__header-border" style="max-width:1200px"> <a class="mPVexkcpuTk__logo-link" href="https://status.cloud.google.com/" aria-label="Google Cloud"> <img class="mPVexkcpuTk__dashboard-logo" src="https://status.cloud.google.com/images/google-cloud-logo.svg" alt="Google Cloud"> </a> <div class="mPVexkcpuTk__upper-right-anchor jv6boZDWPRM__widget-container"><style>.jv6boZDWPRM__widget-container{margin:8px 0}.jv6boZDWPRM__widget-container.jv6boZDWPRM__empty{display:none}.jv6boZDWPRM__widget-container.jv6boZDWPRM__no-margin,.jv6boZDWPRM__widget-container.jv6boZDWPRM__no-widget{margin:0}</style> <div> <div> <div><style>.uoGMeAS7ink__widget-link{-webkit-box-align:center;-webkit-align-items:center;-moz-box-align:center;-ms-flex-align:center;align-items:center;border-radius:4px;display:-webkit-box;display:-webkit-flex;display:-moz-box;display:-ms-flexbox;display:flex;-webkit-box-flex:0;-webkit-flex:none;-moz-box-flex:0;-ms-flex:none;flex:none;-webkit-flex-grow:0;-ms-flex-positive:0;flex-grow:0;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-webkit-flex-direction:row;-moz-box-orient:horizontal;-moz-box-direction:normal;-ms-flex-direction:row;flex-direction:row;gap:8px;-webkit-box-ordinal-group:2;-webkit-order:1;-moz-box-ordinal-group:2;-ms-flex-order:1;order:1;padding:8px}.uoGMeAS7ink__visually-hidden{border:0;clip:rect(0,0,0,0);height:1px;margin:-1px;overflow:hidden;padding:0;position:absolute;width:1px}.uoGMeAS7ink__widget-link-text{-webkit-box-align:center;-webkit-align-items:center;-moz-box-align:center;-ms-flex-align:center;align-items:center;color:#5f6368;display:-webkit-box;display:-webkit-flex;display:-moz-box;display:-ms-flexbox;display:flex;-webkit-box-flex:0;-webkit-flex:none;-moz-box-flex:0;-ms-flex:none;flex:none;-webkit-flex-grow:0;-ms-flex-positive:0;flex-grow:0;font-family:Google Sans,open sans,arial,sans-serif;font-style:normal;font-weight:500;font-size:14px;letter-spacing:.25px;line-height:20px;-webkit-box-ordinal-group:1;-webkit-order:0;-moz-box-ordinal-group:1;-ms-flex-order:0;order:0;text-align:center}</style> <a class="uoGMeAS7ink__widget-link" href="https://status.cloud.google.com/redirect/console" target="_blank"> <p class="uoGMeAS7ink__widget-link-text"> Console </p> <svg class="uoGMeAS7ink__widget-link-icon" aria-hidden="true" fill="#5F6368" height="20" viewBox="0 -960 960 960" width="20" xmlns="http://www.w3.org/2000/svg"> <path d="M216-144q-29.7 0-50.85-21.15Q144-186.3 144-216v-528q0-29.7 21.15-50.85Q186.3-816 216-816h264v72H216v528h528v-264h72v264q0 29.7-21.15 50.85Q773.7-144 744-144H216Zm171-192-51-51 357-357H576v-72h240v240h-72v-117L387-336Z"> </path> </svg> <span class="uoGMeAS7ink__visually-hidden">Opens in a new tab</span> </a> </div> </div> </div> </div> </header> </psd-dashboard-top-bar> <main style="max-width:1200px"> <ul class="vQsaEtosvhM__breadcrumbs"><style>.vQsaEtosvhM__breadcrumbs{font-size:11px;margin:15px 0 20px 0;padding:0}.vQsaEtosvhM__breadcrumbs li{display:inline-block;margin:12px 0 0 0;padding:0}.vQsaEtosvhM__breadcrumbs li a{color:black;text-decoration:none}.vQsaEtosvhM__breadcrumbs li a:after{content:"\00203a";font-size:18px;line-height:13px;padding:0 8px;position:relative;top:1px}.vQsaEtosvhM__breadcrumbs li:last-child a:after{content:""}.vQsaEtosvhM__breadcrumbs li.vQsaEtosvhM__incident-description{max-width:340px;overflow:hidden;text-overflow:ellipsis;vertical-align:bottom;white-space:nowrap}</style> <li> <a href="https://status.cloud.google.com/">Google Cloud Service Health</a></li> <li><a href="https://status.cloud.google.com/summary">Incidents</a></li> <li class="vQsaEtosvhM__incident-description">We experienced elevated errors with BigQuery accessing Google Drive.
</li> </ul> <div class="seE0LugIuG5__header-container"><style>h1{color:#444;font-size:18px;font-weight:600}.seE0LugIuG5__dashboard-title{margin:0}.seE0LugIuG5__header-container{-webkit-box-align:center;-webkit-align-items:center;-moz-box-align:center;-ms-flex-align:center;align-items:center;display:-webkit-box;display:-webkit-flex;display:-moz-box;display:-ms-flexbox;display:flex;margin:.67em 0}</style> <h1 class="seE0LugIuG5__dashboard-title dashboard-header">Service Health</h1> <div class="jv6boZDWPRM__widget-container jv6boZDWPRM__empty"> </div> </div> <div class="aNxXV4vcNrg__intro"><style>a{color:#245dc1;text-decoration:none}a:hover{text-decoration:underline}a:visited{color:#7759ae}.aNxXV4vcNrg__intro{color:#767676;font-size:13px}</style> <div> <span> This page provides status information on the services that are part of Google Cloud. Check back here to view the current status of the services listed below. If you are experiencing an issue not listed here, please <a href="https://cloud.google.com/support-hub/">contact Support</a>. Learn more about what's posted on the dashboard in <a href="https://cloud.google.com/support/docs/dashboard">this FAQ</a>. For additional information on these services, please visit <a href="https://cloud.google.com/">https://cloud.google.com/</a>. </span> <p>For incidents related to Google Security Products, visit <a href="https://status.cloud.google.com/security">https://status.cloud.google.com/security</a>. For incidents related to Looker (original), visit <a href="https://status.cloud.google.com/looker">https://status.cloud.google.com/looker</a>.</p>
</div> <div class="jv6boZDWPRM__widget-container jv6boZDWPRM__empty"> </div> </div> <div class="jv6boZDWPRM__widget-container"> <div> <ul class="nqqp3ZlxVUB__service-status-documentation"><style>ul.nqqp3ZlxVUB__service-status-documentation{display:flex;color:#767676;font-size:13px;list-style:none;padding:0}ul.nqqp3ZlxVUB__service-status-documentation .nqqp3ZlxVUB__with-margin{margin:5px}ul .nqqp3ZlxVUB__with-flex{display:inline-flex}.nqqp3ZlxVUB__with-flex span{margin-left:9px}psd-status-icon{display:inline-block}</style> <li class="nqqp3ZlxVUB__with-margin nqqp3ZlxVUB__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon><span>Available</span> </li> <li class="nqqp3ZlxVUB__with-margin nqqp3ZlxVUB__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__information" fill="none" aria-label="Informational status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm-.833 4.167v1.666h1.667V4.833H8.168Zm0 3.333v5h1.667v-5H8.168Zm-5.833.833A6.676 6.676 0 0 0 9 15.666a6.676 6.676 0 0 0 6.667-6.667A6.676 6.676 0 0 0 9 2.333a6.676 6.676 0 0 0-6.666 6.666Z" fill="#1967D2" fill-rule="evenodd"></path></svg> </psd-status-icon><span>Service information</span> </li> <li class="nqqp3ZlxVUB__with-margin nqqp3ZlxVUB__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__disruption" fill="none" aria-label="Disruption status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M8.168 11.5h1.667v1.666H8.168v-1.667Zm0-6.667h1.667v5H8.168v-5ZM8.993.666C4.393.666.668 4.399.668 8.999s3.725 8.334 8.325 8.334c4.608 0 8.342-3.734 8.342-8.334S13.6.666 8.993.666Zm.008 15a6.665 6.665 0 0 1-6.666-6.667A6.665 6.665 0 0 1 9 2.333a6.665 6.665 0 0 1 6.667 6.666A6.665 6.665 0 0 1 9 15.666Z" fill="#E37400" fill-rule="evenodd"></path></svg> </psd-status-icon><span>Service disruption</span> </li> <li class="nqqp3ZlxVUB__with-margin nqqp3ZlxVUB__with-flex"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__outage" fill="none" aria-label="Outage status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M11.16 5.666 9 7.824 6.843 5.666 5.668 6.841l2.158 2.158-2.158 2.159 1.175 1.175 2.158-2.159 2.159 2.159 1.175-1.175-2.159-2.159 2.159-2.158-1.175-1.175ZM9 .666A8.326 8.326 0 0 0 .668 8.999a8.326 8.326 0 0 0 8.333 8.334 8.326 8.326 0 0 0 8.334-8.334A8.326 8.326 0 0 0 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Z" fill="#D93025" fill-rule="evenodd"></path></svg> </psd-status-icon><span>Service outage</span> </li> </ul> </div> </div> <p class="incident-header"> Incident affecting Google BigQuery </p> <h2 class="incident-description">We experienced elevated errors with BigQuery accessing Google Drive.
</h2> <p> Incident began at <strong class="start-time">2024-12-04 14:30</strong> and ended at <strong class="end-time">2024-12-04 18:28</strong> <span>(all times are <strong class="timezone">US/Pacific</strong>).</span> </p> <h3> Previously affected location(s) </h3> <p class="past-locations locations"> <span class="location">Johannesburg (africa-south1)</span><span class="location">Taiwan (asia-east1)</span><span class="location">Hong Kong (asia-east2)</span><span class="location">Tokyo (asia-northeast1)</span><span class="location">Osaka (asia-northeast2)</span><span class="location">Seoul (asia-northeast3)</span><span class="location">Mumbai (asia-south1)</span><span class="location">Delhi (asia-south2)</span><span class="location">Singapore (asia-southeast1)</span><span class="location">Jakarta (asia-southeast2)</span><span class="location">Sydney (australia-southeast1)</span><span class="location">Melbourne (australia-southeast2)</span><span class="location">Warsaw (europe-central2)</span><span class="location">Finland (europe-north1)</span><span class="location">Madrid (europe-southwest1)</span><span class="location">Berlin (europe-west10)</span><span class="location">Turin (europe-west12)</span><span class="location">London (europe-west2)</span><span class="location">Frankfurt (europe-west3)</span><span class="location">Netherlands (europe-west4)</span><span class="location">Zurich (europe-west6)</span><span class="location">Milan (europe-west8)</span><span class="location">Paris (europe-west9)</span><span class="location">Doha (me-central1)</span><span class="location">Dammam (me-central2)</span><span class="location">Tel Aviv (me-west1)</span><span class="location">Montréal (northamerica-northeast1)</span><span class="location">Toronto (northamerica-northeast2)</span><span class="location">São Paulo (southamerica-east1)</span><span class="location">Santiago (southamerica-west1)</span><span class="location">Iowa (us-central1)</span><span class="location">South Carolina (us-east1)</span><span class="location">Northern Virginia (us-east4)</span><span class="location">Columbus (us-east5)</span><span class="location">Dallas (us-south1)</span><span class="location">Oregon (us-west1)</span><span class="location">Los Angeles (us-west2)</span><span class="location">Salt Lake City (us-west3)</span><span class="location">Las Vegas (us-west4)</span> </p> <table class="status-updates psd-table"> <thead> <tr> <th class="status" scope="col"></th> <th class="date" scope="col">Date</th> <th class="time" scope="col">Time</th> <th class="description" scope="col">Description</th> </tr> </thead> <tbody> <tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">10 Dec 2024</th> <td class="time">11:21 PST</td> <td class="description"><h1>Incident Report</h1>
<h2>Summary</h2>
<p>Starting on 4 December 2024 at 14:30 US/Pacific, Google BigQuery experienced elevated invalid value and internal system errors globally for traffic related to BigQuery and Google Drive integration for 3 hours and 25 minutes. The incident affected users and tasks attempting to export data to Google Drive, resulting in failed export jobs.</p>
<p>The impacted users would have encountered “API key not valid” and “Failed to read the spreadsheet” errors for export jobs when accessing Google Drive. This resulted in service unavailability or failing jobs for the duration of this disruption for Google BigQuery.</p>
<p>To our BigQuery customers whose business analytics were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.</p>
<h2>Root Cause</h2>
<p>This disruption of data export functionality was triggered by the deletion of an internal project containing essential API keys. This deletion was an unintended consequence of several contributing factors:</p>
<ul>
<li>An internally used API key was flagged for a Google policy non-compliance and deemed no longer in use which led to the deleting of the API key.</li>
<li>Unclear Internal Google Project Ownership: The project ownership was not clearly recorded, and thus incorrectly associated with a deprecated service.</li>
<li>Outdated Information: The combination of the perceived lack of recent activity and the incorrect ownership led to the project being mistakenly classified as abandoned and deprecated.</li>
</ul>
<h2>Remediation and Prevention</h2>
<p>Google engineers were alerted to the service degrading via a support case on 4 December 2024, at approximately 14:21 US/Pacific when users began experiencing failures in data export operations. Google Engineers were alerted to the service disruption through internal monitoring systems and user reports. Upon investigation, the deletion of the project was identified as the root cause.</p>
<p>To mitigate the impact, the project was restored at approximately 15:45 US/Pacific. This action successfully recovered the API keys and over time restored the data export functionality for all affected users. The final error related to this incident was observed at approximately 17:55 US/Pacific, indicating full service recovery.</p>
<p>Google is committed to continually improving our technology and operations to prevent service disruptions. We apologize for any inconvenience this incident may have caused and appreciate your understanding.</p>
<p>Google is committed to preventing a repeat of this issue in the future and is completing the following actions.</p>
<ul>
<li>
<p><strong>Remove dependency on API keys for BigQuery integrations with other Google services:</strong> This will eliminate the entire failure mode.</p>
</li>
<li>
<p><strong>Implement accidental deletion protection for critical internal resources:</strong> Use mechanisms like project liens to ensure that a critical resource cannot be deleted accidentally.</p>
</li>
<li>
<p><strong>Enhance Project Metadata:</strong> We are implementing a process for regular review and validation of project ownership and metadata. This will ensure that critical information about project usage and status is accurate and up-to-date, reducing the risk of incorrect assumptions about project status.</p>
</li>
<li>
<p><strong>Strengthen Internal Processes and Access Controls:</strong> We are strengthening our processes for deprecating and deleting projects, including mandatory reviews, impact assessments, and stakeholder approvals. This will prevent accidental deletion of critical projects and ensure that all potential impacts are thoroughly evaluated before any action is taken. We are also strengthening access controls for project deletion, ensuring that only authorized personnel with appropriate approvals can perform this action. This will add an additional layer of protection against unintended project deletion.</p>
</li>
</ul>
<h2>Detailed Description of Impact</h2>
<p>Starting on 4 December 2024, Google BigQuery experienced elevated error rates for data export operations to Google Drive globally. Between approximately 14:21 and 18:04 US/Pacific, users attempting to export data from BigQuery to Google Drive encountered failures, resulting in service disruption for this specific functionality.</p>
<p>The incident affected all regions and impacted users encountered errors such as &#34;API key not valid,&#34; &#34;Failed to read the spreadsheet,&#34; or &#34;[Error: 80324028]&#34;. Internal error messages further specified the issue as &#34;Dremel returned third-party error from GDRIVE: FAILED_PRECONDITION: Encountered an error while creating temporary directory&#34; with an underlying status of &#34;Http(400) Bad Request, API key not valid. Please pass a valid API key.&#34;</p>
<h3>Google BigQuery</h3>
<p>This disruption specifically impacted users and automated tasks relying on the BigQuery to Google Drive export functionality. Export jobs initiated during this period failed to complete, preventing data transfer and potentially impacting downstream processes and workflows dependent on this data.</p>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">5 Dec 2024</th> <td class="time">11:56 PST</td> <td class="description"><h1>Mini Incident Report</h1>
<p>We apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using <a href="https://cloud.google.com/support">https://cloud.google.com/support</a> or to Google Workspace Support using help article <a href="https://support.google.com/a/answer/1047213">https://support.google.com/a/answer/1047213</a>.</p>
<p>(All Times US/Pacific)</p>
<p><strong>Incident Start:</strong> 4 December, 2024 14:30</p>
<p><strong>Incident End:</strong> 4 December, 2024 17:55</p>
<p><strong>Duration:</strong> 3 hours, 25 minutes</p>
<p><strong>Affected Services and Features:</strong></p>
<p>Google BigQuery</p>
<p><strong>Regions/Zones:</strong> Global</p>
<p><strong>Description:</strong></p>
<p>Google BigQuery experienced elevated invalid value and internal system errors globally for traffic related to BigQuery and Google Drive integration for 3 hours and 25 minutes. From preliminary analysis, the root cause has been identified as a misconfiguration in BigQuery&#39;s Google Drive integration. Google will complete a full IR in the following days that will provide a full root cause.</p>
<p><strong>Customer Impact:</strong></p>
<p>The impacted users would have encountered “API key not valid” and “Failed to read the spreadsheet” errors for export jobs when accessing Google Drive. This resulted in service unavailability or failing jobs for the duration of this disruption for Google BigQuery.</p>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__available" aria-label="Available status" fill="none" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M9.001.666A8.336 8.336 0 0 0 .668 8.999c0 4.6 3.733 8.334 8.333 8.334s8.334-3.734 8.334-8.334S13.6.666 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Zm-1.666-4.833L5.168 8.666 4.001 9.833l3.334 3.333L14 6.499l-1.166-1.166-5.5 5.5Z" fill="#1E8E3E" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">4 Dec 2024</th> <td class="time">18:28 PST</td> <td class="description"><p>The issue with Google BigQuery accessing Google Drive has been resolved for all affected projects as of Wednesday, 2024-12-04 18:10 US/Pacific.</p>
<p>We will publish an analysis of this incident once we have completed our internal investigation.</p>
<p>We thank you for your patience while we worked on resolving the issue.</p>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__outage" fill="none" aria-label="Outage status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M11.16 5.666 9 7.824 6.843 5.666 5.668 6.841l2.158 2.158-2.158 2.159 1.175 1.175 2.158-2.159 2.159 2.159 1.175-1.175-2.159-2.159 2.159-2.158-1.175-1.175ZM9 .666A8.326 8.326 0 0 0 .668 8.999a8.326 8.326 0 0 0 8.333 8.334 8.326 8.326 0 0 0 8.334-8.334A8.326 8.326 0 0 0 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Z" fill="#D93025" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">4 Dec 2024</th> <td class="time">17:40 PST</td> <td class="description"><p>Summary: We are working on mitigating elevated errors with BigQuery accessing Google Drive.</p>
<p>Description: We are working on a mitigation for the issue that is causing elevated error rates for query and export jobs that access Google Drive starting Wednesday 2024-12-04 14:30 US/Pacific. The root cause of the issue is understood, and our engineering team is working on mitigating the issue. As of 17:30 US/Pacific, the majority of the customers are no longer impacted.</p>
<p>We will provide more information by Wednesday, 2024-12-04 18:30 US/Pacific.</p>
<p>We apologize to all those affected by the issue.</p>
<p>Diagnosis: Impacted customers would encounter elevated errors or invalid values with “Error: 80324028” while accessing the Google Drive and these jobs would fail.</p>
<p>Workaround: There is no workaround at this time.</p>
</td> </tr><tr> <td class="status"> <psd-status-icon class="psd__status-icon"> <svg class="psd__status-icon psd__outage" fill="none" aria-label="Outage status" height="18" width="18" xmlns="http://www.w3.org/2000/svg"><path clip-rule="evenodd" d="M11.16 5.666 9 7.824 6.843 5.666 5.668 6.841l2.158 2.158-2.158 2.159 1.175 1.175 2.158-2.159 2.159 2.159 1.175-1.175-2.159-2.159 2.159-2.158-1.175-1.175ZM9 .666A8.326 8.326 0 0 0 .668 8.999a8.326 8.326 0 0 0 8.333 8.334 8.326 8.326 0 0 0 8.334-8.334A8.326 8.326 0 0 0 9 .666Zm0 15a6.676 6.676 0 0 1-6.666-6.667A6.676 6.676 0 0 1 9 2.333a6.676 6.676 0 0 1 6.667 6.666A6.676 6.676 0 0 1 9 15.666Z" fill="#D93025" fill-rule="evenodd"></path></svg> </psd-status-icon> </td> <th class="date" scope="row">4 Dec 2024</th> <td class="time">16:40 PST</td> <td class="description"><p>Summary: We are experiencing elevated errors with BigQuery.</p>
<p>Description: As of Wednesday 2024-12-04 14:30 US/Pacific, we are experiencing elevated errors with BigQuery related to query and export jobs that access Google Drive.</p>
<p>Mitigation work is currently underway by our engineering team</p>
<p>The mitigation is expected to complete by Wednesday, 2024-12-04 17:30 US/Pacific.</p>
<p>We will provide the next update by Wednesday, 2024-12-04 18:00 US/Pacific.</p>
<p>Diagnosis: Impacted customers would encounter elevated errors or invalid values with “Error: 80324028” while accessing the Google Drive and these jobs would fail.</p>
<p>Workaround: There is no workaround at this time.</p>
</td> </tr> </tbody> </table> <psd-table-documentation><script src="https://support.google.com/inapp/api.js" nonce="6udoI7VDq3ImaEUTbqrUYA"></script><style>ul.hS6GEzRhW2F__table-documentation{color:#767676;display:flex;flex-wrap:wrap;font-size:13px;list-style:none;margin-left:2.5px;margin-right:2.5px;padding:0}ul.hS6GEzRhW2F__table-documentation li{display:inline-block;margin-left:2.5px;margin-right:2.5px}ul.hS6GEzRhW2F__table-documentation .hS6GEzRhW2F__utc-display{flex-basis:20%;flex-grow:1}ul.hS6GEzRhW2F__table-documentation .hS6GEzRhW2F__dot-after::after{content:'\002022';margin-left:5px}.hS6GEzRhW2F__atom-feed-icon{background:url(data:image/gif;base64,R0lGODlhEAAQANUgAOqHCvySDOxZAORxAOBqGdbW1vCtRuKugezFheZlAOiXUdWlZMmNQ754Osh8OOZtJP6bEfuGBdxhAPbizO7YxuesbfHt6tpYAPZ4Av6nGdxqAPtjAOmQMNxNANLS0v///////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACH5BAEAACAALAAAAAAQABAAAAanQJBwSCwOOQONcqkccCxCDsCAqBoyWCzgCdJkFp/wZ3LNagogCYTDYCwm4QpkLvGkIQEAIF+xfCoBEHVpeWFkEU8fHAGDFxEYFBRhBxEVHxQRF3aOjwMNfgoDkhyaIBcYGAOoDmMYlhWlp3ATGAmSCRwfB7EEYgkbkhe5FR12HQKSExsbBBwbGAcKxSAPAgLHy9nLAgR2FgQd4eLj3WggFgUe6uvsaEEAOw==) left no-repeat;background-size:16px 16px;padding-left:19px}</style> <ul class="hS6GEzRhW2F__table-documentation"> <li class="hS6GEzRhW2F__utc-display">All times are US/Pacific</li> </ul> <ul class="hS6GEzRhW2F__table-documentation"> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__documentation"><a href="https://cloud.google.com/support/docs/dashboard">Documentation</a></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__atom-feed"><span class="hS6GEzRhW2F__atom-feed-icon"><a href="https://status.cloud.google.com/en/feed.atom">RSS Feed</a></span></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__json-history"> <a href="https://status.cloud.google.com/incidents.json">JSON History</a> | <a href="https://status.cloud.google.com/incidents.schema.json" aria-label="JSON History Schema">Schema</a></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__json-product-catalog"> <a href="https://status.cloud.google.com/products.json">JSON Product Catalog</a> | <a href="https://status.cloud.google.com/products.schema.json" aria-label="JSON Product Catalog Schema">Schema</a></li> <li class="hS6GEzRhW2F__dot-after hS6GEzRhW2F__feedback"><a href="#" data-feedback-product-id="104625" onclick="userfeedback.api.startFeedback({'productId': event.target.dataset.feedbackProductId || '5188085'})">Send feedback</a></li> <li class="hS6GEzRhW2F__privacy"><a href="https://policies.google.com/privacy">Privacy policy</a></li> </ul> </psd-table-documentation> </main> </body> </html>