Files
nexus/sreweekly/articles/524/04-automating-gcp-quota-monitoring-across-multiple-projects.html
2026-09-12 17:23:01 +08:00

1462 lines
58 KiB
HTML

<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="description" content="How auto-discovering quota alerts were built for an entire GCP organization after production incidents caused by invisible quota limits.
">
<link rel="prev" href="../../../../../flutter/2025/12/22/a-way-of-managing-flutter-dependencies/">
<link rel="next" href="../../../../../software-engineering/2026/08/04/measuring-data-consistency-in-eventually-consistent-systems/">
<link rel="icon" href="../../../../../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.7">
<title>Automating GCP quota monitoring across multiple projects - Tech @ Coop Norge SA</title>
<link rel="stylesheet" href="../../../../../../assets/stylesheets/main.ec1eaa64.min.css">
<link rel="stylesheet" href="../../../../../../assets/stylesheets/palette.ab4e12ef.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
<link rel="stylesheet" href="../../../../../../stylesheets/extra.css">
<script>__md_scope=new URL("../../../../../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
</head>
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="custom" data-md-color-accent="indigo">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#automating-gcp-quota-monitoring-across-multiple-projects" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header md-header--shadow" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href="../../../../../.." title="Tech @ Coop Norge SA" class="md-header__button md-logo" aria-label="Tech @ Coop Norge SA" data-md-component="logo">
<img src="../../../../../../images/coop-logo.svg" alt="logo">
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
Tech @ Coop Norge SA
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Automating GCP quota monitoring across multiple projects
</span>
</div>
</div>
</div>
<div class="md-header__source">
<a href="https://github.com/coopnorge" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
</div>
<div class="md-source__repository">
Open Source
</div>
</a>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" hidden>
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href="../../../../../.." title="Tech @ Coop Norge SA" class="md-nav__button md-logo" aria-label="Tech @ Coop Norge SA" data-md-component="logo">
<img src="../../../../../../images/coop-logo.svg" alt="logo">
</a>
Tech @ Coop Norge SA
</label>
<div class="md-nav__source">
<a href="https://github.com/coopnorge" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
</div>
<div class="md-source__repository">
Open Source
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--active">
<a href="../../../../../.." class="md-nav__link">
<span class="md-ellipsis">
Home
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" >
<label class="md-nav__link" for="__nav_2" id="__nav_2_label" tabindex="0">
<span class="md-ellipsis">
Archive
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_2">
<span class="md-nav__icon md-icon"></span>
Archive
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../../../../archive/2026/" class="md-nav__link">
<span class="md-ellipsis">
2026
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../../../../archive/2025/" class="md-nav__link">
<span class="md-ellipsis">
2025
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../../../../archive/2024/" class="md-nav__link">
<span class="md-ellipsis">
2024
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
<span class="md-ellipsis">
Categories
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3">
<span class="md-nav__icon md-icon"></span>
Categories
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../../../../../category/code-architecture/" class="md-nav__link">
<span class="md-ellipsis">
Code Architecture
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../../../../category/flutter/" class="md-nav__link">
<span class="md-ellipsis">
Flutter
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../../../../category/mobile/" class="md-nav__link">
<span class="md-ellipsis">
Mobile
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../../../../category/platform-engineering/" class="md-nav__link">
<span class="md-ellipsis">
Platform Engineering
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../../../../../category/software-engineering/" class="md-nav__link">
<span class="md-ellipsis">
Software Engineering
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#the-incidents" class="md-nav__link">
<span class="md-ellipsis">
The incidents
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#why-native-solutions-fall-short" class="md-nav__link">
<span class="md-ellipsis">
Why native solutions fall short
</span>
</a>
<nav class="md-nav" aria-label="Why native solutions fall short">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#consumer-quotas" class="md-nav__link">
<span class="md-ellipsis">
Consumer quotas
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#resource-specific-quotas" class="md-nav__link">
<span class="md-ellipsis">
Resource-specific quotas
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#the-solution" class="md-nav__link">
<span class="md-ellipsis">
The solution
</span>
</a>
<nav class="md-nav" aria-label="The solution">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#architecture" class="md-nav__link">
<span class="md-ellipsis">
Architecture
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#consumer-quota-alerts" class="md-nav__link">
<span class="md-ellipsis">
Consumer quota alerts
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#resource-specific-quota-alerts" class="md-nav__link">
<span class="md-ellipsis">
Resource-specific quota alerts
</span>
</a>
<nav class="md-nav" aria-label="Resource-specific quota alerts">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#step-1-fetch-metric-descriptors" class="md-nav__link">
<span class="md-ellipsis">
Step 1: fetch metric descriptors
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#step-2-filter-to-resource-specific-quota-metrics" class="md-nav__link">
<span class="md-ellipsis">
Step 2: filter to resource-specific quota metrics
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#step-3-resolve-the-correct-on-labels" class="md-nav__link">
<span class="md-ellipsis">
Step 3: resolve the correct on() labels
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#step-4-convert-metric-names-and-generate-promql" class="md-nav__link">
<span class="md-ellipsis">
Step 4: convert metric names and generate PromQL
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#step-5-group-by-service" class="md-nav__link">
<span class="md-ellipsis">
Step 5: group by service
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#the-terraform" class="md-nav__link">
<span class="md-ellipsis">
The Terraform
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#technical-challenges" class="md-nav__link">
<span class="md-ellipsis">
Technical challenges
</span>
</a>
<nav class="md-nav" aria-label="Technical challenges">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#sparse-sampling-and-alert-flapping" class="md-nav__link">
<span class="md-ellipsis">
Sparse sampling and alert flapping
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#the-resource_container-gotcha" class="md-nav__link">
<span class="md-ellipsis">
The resource_container gotcha
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#label-mismatches-between-usage-and-limit" class="md-nav__link">
<span class="md-ellipsis">
Label mismatches between usage and limit
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#read-only-api-quota-noise" class="md-nav__link">
<span class="md-ellipsis">
Read-only API quota noise
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#the-workflow-alert-to-resolution" class="md-nav__link">
<span class="md-ellipsis">
The workflow: alert to resolution
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#apis-to-enable" class="md-nav__link">
<span class="md-ellipsis">
APIs to enable
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#links-and-resources" class="md-nav__link">
<span class="md-ellipsis">
Links and resources
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content md-content--post" data-md-component="content">
<div class="md-sidebar md-sidebar--post" data-md-component="sidebar" data-md-type="navigation">
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner md-post">
<nav class="md-nav md-nav--primary">
<div class="md-post__back">
<div class="md-nav__title md-nav__container">
<a href="../../../../../.." class="md-nav__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
<span class="md-ellipsis">
Back to index
</span>
</a>
</div>
</div>
<div class="md-post__authors md-typeset">
<div class="md-profile md-post__profile">
<span class="md-author md-author--long">
<img src="https://avatars.githubusercontent.com/u/26136082" alt="Aksel Allas">
</span>
<span class="md-profile__description">
<strong>
<a href="https://github.com/AkselAllas">Aksel Allas</a>
</strong>
<br>
Staff Platform Engineer
</span>
</div>
</div>
<ul class="md-post__meta md-nav__list">
<li class="md-nav__item md-nav__item--section">
<div class="md-post__title">
<span class="md-ellipsis">
Metadata
</span>
</div>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<div class="md-nav__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 19H5V8h14m-3-7v2H8V1H6v2H5c-1.11 0-2 .89-2 2v14a2 2 0 0 0 2 2h14a2 2 0 0 0 2-2V5a2 2 0 0 0-2-2h-1V1m-1 11h-5v5h5z"/></svg>
<time datetime="2026-06-25 00:00:00+00:00" class="md-ellipsis">June 25, 2026</time>
</div>
</li>
<li class="md-nav__item">
<div class="md-nav__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9 3v15h3V3zm3 2 4 13 3-1-4-13zM5 5v13h3V5zM3 19v2h18v-2z"/></svg>
<span class="md-ellipsis">
in
<a href="../../../../../../category/platform-engineering/">Platform Engineering</a></span>
</div>
</li>
<li class="md-nav__item">
<div class="md-nav__link">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 20a8 8 0 0 0 8-8 8 8 0 0 0-8-8 8 8 0 0 0-8 8 8 8 0 0 0 8 8m0-18a10 10 0 0 1 10 10 10 10 0 0 1-10 10C6.47 22 2 17.5 2 12A10 10 0 0 1 12 2m.5 5v5.25l4.5 2.67-.75 1.23L11 13V7z"/></svg>
<span class="md-ellipsis">
8 min read
</span>
</div>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<article class="md-content__inner md-typeset">
<nav class="md-tags" >
<span class="md-tag">GCP</span>
<span class="md-tag">Monitoring</span>
<span class="md-tag">Terraform</span>
</nav>
<h1 id="automating-gcp-quota-monitoring-across-multiple-projects">Automating GCP quota monitoring across multiple projects<a class="headerlink" href="#automating-gcp-quota-monitoring-across-multiple-projects" title="Permanent link">&para;</a></h1>
<p>Every GCP resource and API have quotas. In a big organization,
you can start having production incidents due to hitting quotas
you didn't know about in projects you have never touched before.</p>
<!-- more -->
<h2 id="the-incidents">The incidents<a class="headerlink" href="#the-incidents" title="Permanent link">&para;</a></h2>
<p>In May 2026, a VPC dynamic route quota was silently exceeded.
GCP dropped BGP-learned (Border Gateway Protocol) routes
without any alert. Traffic to on-prem destinations fell back to
the default internet gateway and was black-holed
(dropped without notification).</p>
<p>A couple production services were down before the issue was
traced to <code>routeStatus: DROPPED</code> in the Cloud Router output.</p>
<p>The fix was a quota increase and a BGP re-sync, but finding the
root cause took a long time because nothing indicated that
a quota had been hit.</p>
<p>A separate incident involved GKE Persistent Disk storage:
usage grew from 900 GB to the 1000 GB limit during a GKE
version update without anyone noticing until provisioning
workloads started failing.</p>
<p>Both incidents had the same root cause: <strong>zero visibility into
GCP quotas</strong> across GCP projects.</p>
<h2 id="why-native-solutions-fall-short">Why native solutions fall short<a class="headerlink" href="#why-native-solutions-fall-short" title="Permanent link">&para;</a></h2>
<p>GCP does have quota monitoring built into
GCP Cloud Monitoring.
<a href="https://cloud.google.com/monitoring/alerts/using-quota-metrics">Here's the best doc on monitoring&nbsp;⧉</a>.
You can create alert policies across projects that fire when
a quota approaches its limit. So why wasn't it being used?</p>
<p>The problem is that GCP has <strong>two completely separate quota
metric systems</strong>, and neither supports a simple "alert on
everything" approach.</p>
<h3 id="consumer-quotas">Consumer quotas<a class="headerlink" href="#consumer-quotas" title="Permanent link">&para;</a></h3>
<p>The first system uses <code>serviceruntime.googleapis.com/quota/*</code>
metrics with a generic <code>consumer_quota</code> resource type. These
cover API-level quotas: request rates, allocation limits,
storage quotas, and similar. A single <code>quota_metric</code> label
identifies which specific quota the time series belongs to.</p>
<p>The good news: you can write a <a href="https://prometheus.io/docs/prometheus/latest/querying/basics/">PromQL&nbsp;⧉</a>
(Prometheus Query Language) query that matches
<strong>all</strong> consumer quotas without specifying individual services
or quota names. PromQL is a language for selecting and
aggregating time series metrics. GCP Cloud Monitoring supports
it as an alternative to its native query language, and it's
what powers the alert conditions described below.</p>
<h3 id="resource-specific-quotas">Resource-specific quotas<a class="headerlink" href="#resource-specific-quotas" title="Permanent link">&para;</a></h3>
<p>The second system uses service-specific metrics like
<code>compute.googleapis.com/quota/dynamic_routes_per_region_per_peering_group/usage</code>.
Each service defines its own monitored resource type and label
set. These cover infrastructure-level quotas: VPC routes,
instances per network, GKE nodes per cluster.</p>
<p>These aren't covered by consumer quota queries. Each metric
has its own path and its own set of labels for the <code>on()</code>
clause in PromQL. There are currently over 370 such metrics
across 28 services.</p>
<p>The VPC routing incident? That was a resource-specific quota.
The native consumer quota alerts would never have caught it.</p>
<h2 id="the-solution">The solution<a class="headerlink" href="#the-solution" title="Permanent link">&para;</a></h2>
<p>The requirements:</p>
<ol>
<li>Covers both quota systems</li>
<li>Automatically picks up new quotas and services</li>
<li>Works across all 300 monitored projects from a single place</li>
<li>Doesn't require manual configuration per quota</li>
</ol>
<h3 id="architecture">Architecture<a class="headerlink" href="#architecture" title="Permanent link">&para;</a></h3>
<pre class="mermaid"><code>graph LR
subgraph "Scoping Project"
MS[Metrics Scope] --&gt; AP[Alert Policies]
AP --&gt; NC[Slack Channel]
end
subgraph "Monitored Projects"
P1[project-1] --&gt; MS
P2[project-2] --&gt; MS
P3[project-N] --&gt; MS
end
subgraph "Auto-Discovery via github repo for managing Scoping Project"
TF[Terraform] --&gt;|data external| SH[get_quota_metrics.sh]
SH --&gt;|Cloud Monitoring API| PY[build_quota_promql.py]
PY --&gt;|per-service PromQL| TF
end
TF --&gt; AP</code></pre>
<p>All alerts run in a single <strong>scoping project</strong>. Every monitored
project gets added to its
<a href="https://cloud.google.com/monitoring/settings/multiple-projects">metrics scope&nbsp;⧉</a>,
so one set of alert policies covers every project. PromQL
queries group by <code>project_id</code>, <code>quota_metric</code>, <code>location</code>, and
<code>service</code>, so each unique combination fires as a separate
incident. You know exactly which quota in which project and
region is at risk.</p>
<h3 id="consumer-quota-alerts">Consumer quota alerts<a class="headerlink" href="#consumer-quota-alerts" title="Permanent link">&para;</a></h3>
<p>For consumer quotas, there are three alert policies. No service
filter, no quota name filter. They match all consumer quotas
automatically.</p>
<p><strong>Allocation usage &gt; 80%:</strong> resource limits like disk, CPU,
IP addresses:</p>
<div class="highlight"><span class="filename">PromQL</span><pre><span></span><code><span class="o">(</span>
<span class="w"> </span><span class="k">max</span><span class="w"> </span><span class="k">by</span><span class="w"> </span><span class="o">(</span><span class="nv">project_id</span><span class="p">,</span><span class="w"> </span><span class="nv">quota_metric</span><span class="p">,</span><span class="w"> </span><span class="nv">location</span><span class="p">,</span><span class="w"> </span><span class="nv">service</span><span class="o">)</span><span class="w"> </span><span class="o">(</span>
<span class="w"> </span><span class="nv">last_over_time</span><span class="o">(</span>
<span class="w"> </span><span class="nv">serviceruntime_googleapis_com</span><span class="err">:</span><span class="nv">quota_allocation_usage</span><span class="p">{</span>
<span class="w"> </span><span class="nl">monitored_resource</span><span class="o">=</span><span class="p">&quot;</span><span class="s">consumer_quota</span><span class="p">&quot;</span>
<span class="w"> </span><span class="p">}[</span><span class="s">6h</span><span class="p">]</span>
<span class="w"> </span><span class="o">)</span>
<span class="w"> </span><span class="o">)</span>
<span class="w"> </span><span class="o">/</span>
<span class="w"> </span><span class="k">min</span><span class="w"> </span><span class="k">by</span><span class="w"> </span><span class="o">(</span><span class="nv">project_id</span><span class="p">,</span><span class="w"> </span><span class="nv">quota_metric</span><span class="p">,</span><span class="w"> </span><span class="nv">location</span><span class="p">,</span><span class="w"> </span><span class="nv">service</span><span class="o">)</span><span class="w"> </span><span class="o">(</span>
<span class="w"> </span><span class="nv">last_over_time</span><span class="o">(</span>
<span class="w"> </span><span class="nv">serviceruntime_googleapis_com</span><span class="err">:</span><span class="nv">quota_limit</span><span class="p">{</span>
<span class="w"> </span><span class="nl">monitored_resource</span><span class="o">=</span><span class="p">&quot;</span><span class="s">consumer_quota</span><span class="p">&quot;</span>
<span class="w"> </span><span class="p">}[</span><span class="s">6h</span><span class="p">]</span>
<span class="w"> </span><span class="o">)</span>
<span class="w"> </span><span class="o">)</span>
<span class="o">)</span><span class="w"> </span><span class="o">&gt;</span><span class="w"> </span><span class="mf">0.8</span>
</code></pre></div>
<p><strong>Rate usage &gt; 80%:</strong> API request rates, with read-only APIs
excluded to reduce noise. Hitting a read rate limit causes
retries, not outages:</p>
<div class="highlight"><span class="filename">PromQL</span><pre><span></span><code><span class="o">(</span>
<span class="w"> </span><span class="k">sum</span><span class="w"> </span><span class="k">by</span><span class="w"> </span><span class="o">(</span><span class="nv">project_id</span><span class="p">,</span><span class="w"> </span><span class="nv">quota_metric</span><span class="p">,</span><span class="w"> </span><span class="nv">location</span><span class="p">,</span><span class="w"> </span><span class="nv">service</span><span class="o">)</span><span class="w"> </span><span class="o">(</span>
<span class="w"> </span><span class="kr">increase</span><span class="o">(</span>
<span class="w"> </span><span class="nv">serviceruntime_googleapis_com</span><span class="err">:</span><span class="nv">quota_rate_net_usage</span><span class="p">{</span>
<span class="w"> </span><span class="nl">monitored_resource</span><span class="o">=</span><span class="p">&quot;</span><span class="s">consumer_quota</span><span class="p">&quot;,</span>
<span class="w"> </span><span class="nl">quota_metric</span><span class="o">!~</span><span class="p">&quot;</span><span class="s">.*/get_.*|.*/list_.*|.*read_requests.*|.*/read$|.*/fetch_.*|.*search_requests.*</span><span class="p">&quot;</span>
<span class="w"> </span><span class="p">}[</span><span class="s">1m</span><span class="p">]</span>
<span class="w"> </span><span class="o">)</span>
<span class="w"> </span><span class="o">)</span>
<span class="w"> </span><span class="o">/</span>
<span class="w"> </span><span class="k">max</span><span class="w"> </span><span class="k">by</span><span class="w"> </span><span class="o">(</span><span class="nv">project_id</span><span class="p">,</span><span class="w"> </span><span class="nv">quota_metric</span><span class="p">,</span><span class="w"> </span><span class="nv">location</span><span class="p">,</span><span class="w"> </span><span class="nv">service</span><span class="o">)</span><span class="w"> </span><span class="o">(</span>
<span class="w"> </span><span class="nv">last_over_time</span><span class="o">(</span>
<span class="w"> </span><span class="nv">serviceruntime_googleapis_com</span><span class="err">:</span><span class="nv">quota_limit</span><span class="p">{</span>
<span class="w"> </span><span class="nl">monitored_resource</span><span class="o">=</span><span class="p">&quot;</span><span class="s">consumer_quota</span><span class="p">&quot;,</span>
<span class="w"> </span><span class="nl">quota_metric</span><span class="o">!~</span><span class="p">&quot;</span><span class="s">.*/get_.*|.*/list_.*|.*read_requests.*|.*/read$|.*/fetch_.*|.*search_requests.*</span><span class="p">&quot;</span>
<span class="w"> </span><span class="p">}[</span><span class="s">6h</span><span class="p">]</span>
<span class="w"> </span><span class="o">)</span>
<span class="w"> </span><span class="o">)</span>
<span class="o">)</span><span class="w"> </span><span class="o">&gt;</span><span class="w"> </span><span class="mf">0.8</span>
</code></pre></div>
<p><strong>Quota exceeded:</strong> safety net for anything that slips past the
80% warning, with the same read-only exclusion:</p>
<div class="highlight"><span class="filename">PromQL</span><pre><span></span><code><span class="k">max</span><span class="w"> </span><span class="k">by</span><span class="w"> </span><span class="o">(</span><span class="nv">project_id</span><span class="p">,</span><span class="w"> </span><span class="nv">quota_metric</span><span class="p">,</span><span class="w"> </span><span class="nv">location</span><span class="p">,</span><span class="w"> </span><span class="nv">service</span><span class="o">)</span><span class="w"> </span><span class="o">(</span>
<span class="w"> </span><span class="nv">last_over_time</span><span class="o">(</span>
<span class="w"> </span><span class="nv">serviceruntime_googleapis_com</span><span class="err">:</span><span class="nv">quota_exceeded</span><span class="p">{</span>
<span class="w"> </span><span class="nl">monitored_resource</span><span class="o">=</span><span class="p">&quot;</span><span class="s">consumer_quota</span><span class="p">&quot;,</span>
<span class="w"> </span><span class="nl">quota_metric</span><span class="o">!~</span><span class="p">&quot;</span><span class="s">.*/get_.*|.*/list_.*|.*read_requests.*|.*/read$|.*/fetch_.*|.*search_requests.*</span><span class="p">&quot;</span>
<span class="w"> </span><span class="p">}[</span><span class="s">6h</span><span class="p">]</span>
<span class="w"> </span><span class="o">)</span>
<span class="o">)</span><span class="w"> </span><span class="o">&gt;</span><span class="w"> </span><span class="mi">0</span>
</code></pre></div>
<p>When someone enables a new GCP API or Google adds a new quota,
these queries pick it up with zero configuration changes.</p>
<h3 id="resource-specific-quota-alerts">Resource-specific quota alerts<a class="headerlink" href="#resource-specific-quota-alerts" title="Permanent link">&para;</a></h3>
<p>Resource-specific quotas can't be covered by a single query.
Each metric has different labels. A VPC network quota has
<code>network_id</code>, a GKE quota has <code>cluster_name</code>, an AI Platform
quota has <code>base_model</code>. The PromQL <code>on()</code> clause must match per
metric.</p>
<p>Instead of maintaining a static list, a discovery script runs
as a Terraform
<a href="https://registry.terraform.io/providers/hashicorp/external/latest/docs/data-sources/external"><code>data "external"</code>&nbsp;⧉</a>
source. Here's how it works in detail.</p>
<h4 id="step-1-fetch-metric-descriptors">Step 1: fetch metric descriptors<a class="headerlink" href="#step-1-fetch-metric-descriptors" title="Permanent link">&para;</a></h4>
<p>A bash wrapper calls the GCP Cloud Monitoring API to get every
metric descriptor in the scoping project. This includes metrics
from all projects in the metrics scope:</p>
<div class="highlight"><span class="filename">Bash</span><pre><span></span><code>curl<span class="w"> </span>-s<span class="w"> </span>-H<span class="w"> </span><span class="s2">&quot;Authorization: Bearer </span><span class="si">${</span><span class="nv">TOKEN</span><span class="si">}</span><span class="s2">&quot;</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span><span class="s2">&quot;</span><span class="si">${</span><span class="nv">BASE_URL</span><span class="si">}</span><span class="s2">/metricDescriptors&quot;</span><span class="w"> </span>&gt;<span class="w"> </span><span class="s2">&quot;</span><span class="si">${</span><span class="nv">METRICS_FILE</span><span class="si">}</span><span class="s2">&quot;</span>
curl<span class="w"> </span>-s<span class="w"> </span>-H<span class="w"> </span><span class="s2">&quot;Authorization: Bearer </span><span class="si">${</span><span class="nv">TOKEN</span><span class="si">}</span><span class="s2">&quot;</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span><span class="s2">&quot;</span><span class="si">${</span><span class="nv">BASE_URL</span><span class="si">}</span><span class="s2">/monitoredResourceDescriptors&quot;</span><span class="w"> </span><span class="se">\</span>
<span class="w"> </span>&gt;<span class="w"> </span><span class="s2">&quot;</span><span class="si">${</span><span class="nv">RESOURCES_FILE</span><span class="si">}</span><span class="s2">&quot;</span>
</code></pre></div>
<p>The metric descriptors tell you what quota metrics exist (for
example,
<code>compute.googleapis.com/quota/dynamic_routes_per_region_per_peering_group/usage</code>)
and what <strong>metric labels</strong> each one has (for example,
<code>limit_name</code>).</p>
<p>The resource descriptors tell you what <strong>resource labels</strong> each
monitored resource type has. For example,
<code>compute.googleapis.com/VpcNetwork</code> has <code>resource_container</code>,
<code>location</code>, and <code>network_id</code>.</p>
<h4 id="step-2-filter-to-resource-specific-quota-metrics">Step 2: filter to resource-specific quota metrics<a class="headerlink" href="#step-2-filter-to-resource-specific-quota-metrics" title="Permanent link">&para;</a></h4>
<p>A Python script processes the JSON. It finds all metrics
matching the pattern
<code>&lt;service&gt;.googleapis.com/quota/&lt;name&gt;/usage</code> and <code>*/limit</code>,
excluding <code>serviceruntime</code> (those are consumer quotas handled
separately) and <code>*_internal</code> metrics (they have descriptors but
get rejected by the alerting API):</p>
<h4 id="step-3-resolve-the-correct-on-labels">Step 3: resolve the correct <code>on()</code> labels<a class="headerlink" href="#step-3-resolve-the-correct-on-labels" title="Permanent link">&para;</a></h4>
<p>This is the tricky part. For the PromQL division
<code>usage / limit</code> to work, the <code>on()</code> clause must list every
label shared between the two sides. These labels come from two
sources: the resource type and the metric itself.</p>
<p>One gotcha: the resource descriptor calls the project label
<code>resource_container</code>, but in actual PromQL queries it appears
as <code>project_id</code>. This was discovered by querying the raw time
series API and comparing:</p>
<p>For quotas where usage has extra labels that limit doesn't
(mainly AI Platform metrics with a <code>method</code> label),
<code>group_left()</code> allows the many-to-one join.</p>
<h4 id="step-4-convert-metric-names-and-generate-promql">Step 4: convert metric names and generate PromQL<a class="headerlink" href="#step-4-convert-metric-names-and-generate-promql" title="Permanent link">&para;</a></h4>
<p><a href="https://docs.cloud.google.com/monitoring/promql#transforming-names">GCP Cloud Monitoring PromQL&nbsp;⧉</a> uses a different
naming convention than the API. The first <code>/</code> becomes <code>:</code>, and
all other special characters become <code>_</code>:</p>
<p>Each quota becomes one PromQL clause:</p>
<div class="highlight"><span class="filename">Python</span><pre><span></span><code><span class="n">clause</span> <span class="o">=</span> <span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;last_over_time(</span><span class="si">{</span><span class="n">usage_name</span><span class="si">}</span><span class="s2">[</span><span class="si">{</span><span class="n">lookback</span><span class="si">}</span><span class="s2">])&quot;</span>
<span class="sa">f</span><span class="s2">&quot; / on(</span><span class="si">{</span><span class="n">on_labels</span><span class="si">}</span><span class="s2">) group_left() &quot;</span>
<span class="sa">f</span><span class="s2">&quot;last_over_time(</span><span class="si">{</span><span class="n">limit_name</span><span class="si">}</span><span class="s2">[</span><span class="si">{</span><span class="n">lookback</span><span class="si">}</span><span class="s2">])&quot;</span>
<span class="sa">f</span><span class="s2">&quot; &gt; </span><span class="si">{</span><span class="n">threshold</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="p">)</span>
</code></pre></div>
<h4 id="step-5-group-by-service">Step 5: group by service<a class="headerlink" href="#step-5-group-by-service" title="Permanent link">&para;</a></h4>
<p>Clauses are grouped by service name extracted from the metric
path and joined with <code>or</code>. The script outputs a flat JSON
object with keys as service names, and values as complete
PromQL queries:</p>
<div class="highlight"><span class="filename">JSON</span><pre><span></span><code><span class="p">{</span>
<span class="w"> </span><span class="nt">&quot;compute&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;last_over_time(...) / on(...) ... &gt; 0.8\nor\nlast_over_time(...) ...&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;container&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;...&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nt">&quot;storage&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;...&quot;</span>
<span class="p">}</span>
</code></pre></div>
<p>Terraform's <code>for_each</code> iterates over this map, creating one
alert policy per service. Currently that's 28 services covering
370+ quota metrics. When Google adds a new service with
resource-specific quotas, the next <code>terraform apply</code> creates a
new alert policy automatically.</p>
<p>A generated query for compute quotas looks like this (one
clause per quota, joined with <code>or</code>):</p>
<div class="highlight"><span class="filename">PromQL</span><pre><span></span><code><span class="nv">last_over_time</span><span class="o">(</span><span class="nv">compute_googleapis_com</span><span class="err">:</span><span class="nv">quota_dynamic_routes_per_region_per_peering_group_usage</span><span class="p">[</span><span class="s">6h</span><span class="p">]</span><span class="o">)</span>
<span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="k">on</span><span class="o">(</span><span class="nv">limit_name</span><span class="p">,</span><span class="w"> </span><span class="nv">location</span><span class="p">,</span><span class="w"> </span><span class="nv">network_id</span><span class="p">,</span><span class="w"> </span><span class="nv">project_id</span><span class="o">)</span><span class="w"> </span><span class="k">group_left</span><span class="o">()</span>
<span class="w"> </span><span class="nv">last_over_time</span><span class="o">(</span><span class="nv">compute_googleapis_com</span><span class="err">:</span><span class="nv">quota_dynamic_routes_per_region_per_peering_group_limit</span><span class="p">[</span><span class="s">6h</span><span class="p">]</span><span class="o">)</span>
<span class="w"> </span><span class="o">&gt;</span><span class="w"> </span><span class="mf">0.8</span>
<span class="ow">or</span>
<span class="nv">last_over_time</span><span class="o">(</span><span class="nv">compute_googleapis_com</span><span class="err">:</span><span class="nv">quota_instances_per_vpc_network_usage</span><span class="p">[</span><span class="s">6h</span><span class="p">]</span><span class="o">)</span>
<span class="w"> </span><span class="o">/</span><span class="w"> </span><span class="k">on</span><span class="o">(</span><span class="nv">limit_name</span><span class="p">,</span><span class="w"> </span><span class="nv">location</span><span class="p">,</span><span class="w"> </span><span class="nv">network_id</span><span class="p">,</span><span class="w"> </span><span class="nv">project_id</span><span class="o">)</span><span class="w"> </span><span class="k">group_left</span><span class="o">()</span>
<span class="w"> </span><span class="nv">last_over_time</span><span class="o">(</span><span class="nv">compute_googleapis_com</span><span class="err">:</span><span class="nv">quota_instances_per_vpc_network_limit</span><span class="p">[</span><span class="s">6h</span><span class="p">]</span><span class="o">)</span>
<span class="w"> </span><span class="o">&gt;</span><span class="w"> </span><span class="mf">0.8</span>
</code></pre></div>
<p>When a new service adds resource-specific quota metrics, the
next <code>terraform apply</code> creates a new alert policy for that
service automatically.</p>
<h3 id="the-terraform">The Terraform<a class="headerlink" href="#the-terraform" title="Permanent link">&para;</a></h3>
<p>The Terraform configuration ties it all together. The
<a href="https://developer.hashicorp.com/terraform/language/meta-arguments/for_each"><code>for_each</code>&nbsp;⧉</a>
over the discovery script output creates one alert policy per
service:</p>
<div class="highlight"><span class="filename">Terraform</span><pre><span></span><code><span class="kr">data</span><span class="w"> </span><span class="nc">&quot;external&quot;</span><span class="w"> </span><span class="nv">&quot;quota_metrics&quot;</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="na">program</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">[</span>
<span class="w"> </span><span class="s2">&quot;bash&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="s2">&quot;${path.module}/scripts/get_quota_metrics.sh&quot;</span><span class="p">,</span>
<span class="w"> </span><span class="nv">local.quota_monitoring_project_id</span><span class="p">,</span>
<span class="w"> </span><span class="nf">tostring</span><span class="p">(</span><span class="nv">local.quota_alert_threshold</span><span class="p">),</span>
<span class="w"> </span><span class="nv">local.quota_alert_lookback</span><span class="p">,</span>
<span class="w"> </span><span class="p">]</span>
<span class="w"> </span><span class="nb">query</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="na">exclusions</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nf">jsonencode</span><span class="p">(</span><span class="nv">local.quota_alert_exclusions</span><span class="p">)</span>
<span class="w"> </span><span class="p">}</span>
<span class="p">}</span>
<span class="kr">resource</span><span class="w"> </span><span class="nc">&quot;google_monitoring_alert_policy&quot;</span><span class="w"> </span><span class="nv">&quot;quota_resource_specific&quot;</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="na">for_each</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nv">data.external.quota_metrics.result</span>
<span class="w"> </span><span class="na">project</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nv">local.quota_monitoring_project_id</span>
<span class="w"> </span><span class="na">display_name</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;Quota &gt; 80% - ${each.key} resource quotas&quot;</span>
<span class="w"> </span><span class="nb">conditions</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="na">display_name</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;${each.key} resource quota &gt; 80%&quot;</span>
<span class="w"> </span><span class="nb">condition_prometheus_query_language</span><span class="w"> </span><span class="p">{</span>
<span class="w"> </span><span class="na">query</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nv">each.value</span>
<span class="w"> </span><span class="na">duration</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;0s&quot;</span>
<span class="w"> </span><span class="na">evaluation_interval</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;30s&quot;</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="p">}</span>
<span class="w"> </span><span class="na">notification_channels</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="nv">local.quota_alert_notification_channels</span>
<span class="p">}</span>
</code></pre></div>
<h2 id="technical-challenges">Technical challenges<a class="headerlink" href="#technical-challenges" title="Permanent link">&para;</a></h2>
<h3 id="sparse-sampling-and-alert-flapping">Sparse sampling and alert flapping<a class="headerlink" href="#sparse-sampling-and-alert-flapping" title="Permanent link">&para;</a></h3>
<p>Quota metrics are sampled infrequently. Data points arrive
every 5 to 15 minutes with gaps. Alerts would fire when a data
point showed usage exceeding 80%, then immediately resolve when
the next evaluation found no data, then fire again when the
next data point arrived.</p>
<p>PromQL alert conditions don't support
<code>evaluation_missing_data = "EVALUATION_MISSING_DATA_ACTIVE"</code>
(that's only available for <code>condition_threshold</code>). The fix was
wrapping every metric selector in <code>last_over_time(...[6h])</code>,
which returns the most recent data point within the look-back
window. No more flapping.</p>
<h3 id="the-resource_container-gotcha">The <code>resource_container</code> gotcha<a class="headerlink" href="#the-resource_container-gotcha" title="Permanent link">&para;</a></h3>
<p>The GCP Cloud Monitoring API's resource descriptors list a
label called <code>resource_container</code>, but in actual PromQL
queries, that label appears as <code>project_id</code>. This was
discovered by querying the raw time series API and comparing
label names. The script maps <code>resource_container</code> to
<code>project_id</code> automatically.</p>
<h3 id="label-mismatches-between-usage-and-limit">Label mismatches between usage and limit<a class="headerlink" href="#label-mismatches-between-usage-and-limit" title="Permanent link">&para;</a></h3>
<p>For some quotas (mainly AI Platform), the <code>/usage</code> metric has
an extra <code>method</code> label that the <code>/limit</code> metric doesn't have.
A naive division fails because PromQL can't match series with
different label sets. Using <code>group_left()</code> handles the
many-to-one join.</p>
<h3 id="read-only-api-quota-noise">Read-only API quota noise<a class="headerlink" href="#read-only-api-quota-noise" title="Permanent link">&para;</a></h3>
<p>Rate quota alerts were extremely noisy. Quotas like
<code>read_requests</code>, <code>list_requests</code>, and <code>search_requests</code> would
fire constantly. Hitting a read rate limit causes retries, not
outages. It's low-risk noise that drowns out real issues.</p>
<p>A regular expression filter on the <code>quota_metric</code> label
excludes read-only patterns:</p>
<div class="highlight"><span class="filename">PromQL</span><pre><span></span><code><span class="nv">quota_metric</span><span class="err">!~</span><span class="p">&quot;</span><span class="s">.*/get_.*|.*/list_.*|.*read_requests.*|.*/read$|.*/fetch_.*|.*search_requests.*</span><span class="p">&quot;</span>
</code></pre></div>
<h2 id="the-workflow-alert-to-resolution">The workflow: alert to resolution<a class="headerlink" href="#the-workflow-alert-to-resolution" title="Permanent link">&para;</a></h2>
<p>When a quota alert fires, here's the investigation path:</p>
<p><strong>1. Alert arrives in Slack</strong> with the project ID, quota name,
service, and current ratio.</p>
<p><strong>2. Check the Quotas page</strong> in the GCP Console for the
affected project. The
<a href="https://console.cloud.google.com/iam-admin/quotas">Quotas &amp; System Limits&nbsp;⧉</a>
page shows current usage alongside limits.</p>
<p><strong>3. Check API usage and error rates</strong> to understand what's
driving the consumption. The API dashboard shows request counts,
error rates, and latency per method:</p>
<p><img alt="GCP API Methods dashboard showing DNS API request counts and error rates" src="../../../../../../images/gcp-quota-monitoring/api-error-rates.png" /></p>
<p><strong>4. Increase the quota</strong> if the usage is legitimate. Some
quotas can be increased through self-service.</p>
<p>Some quotas are marked <code>is_fixed</code> and require a support ticket
to increase. The VPC dynamic routes quota that caused the first
incident was one of these.</p>
<h2 id="apis-to-enable">APIs to enable<a class="headerlink" href="#apis-to-enable" title="Permanent link">&para;</a></h2>
<p>Three APIs need to be enabled on each monitored project for
quota metrics to flow correctly:</p>
<table>
<thead>
<tr>
<th>API</th>
<th>Why</th>
</tr>
</thead>
<tbody>
<tr>
<td><code>cloudquotas.googleapis.com</code></td>
<td>Accurate quota data. Not on by default.</td>
</tr>
<tr>
<td><code>storage-component.googleapis.com</code></td>
<td>Google Cloud Storage quota visibility</td>
</tr>
<tr>
<td><code>storage.googleapis.com</code></td>
<td>Google Cloud Storage quota visibility</td>
</tr>
</tbody>
</table>
<p>These get enabled through Terraform on all monitored projects
once in the beginning, and were added to the new project
Terraform module so future projects get them automatically.</p>
<h2 id="links-and-resources">Links and resources<a class="headerlink" href="#links-and-resources" title="Permanent link">&para;</a></h2>
<ul>
<li><a href="https://cloud.google.com/monitoring/alerts/using-quota-metrics">Cloud Monitoring: Using quota metrics&nbsp;⧉</a>
Google's documentation on quota alerting</li>
<li><a href="https://cloud.google.com/monitoring/settings/multiple-projects">Metrics scopes overview&nbsp;⧉</a>
Multi-project monitoring (375 project default limit)</li>
<li><a href="https://docs.cloud.google.com/monitoring/promql#transforming-names">PromQL metric name transformation&nbsp;⧉</a>
How GCP metric names map to PromQL names</li>
</ul>
<aside class="md-source-file">
<span class="md-source-file__fact">
<span class="md-icon" title="Contributors">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 2A10 10 0 0 0 2 12c0 4.42 2.87 8.17 6.84 9.5.5.08.66-.23.66-.5v-1.69c-2.77.6-3.36-1.34-3.36-1.34-.46-1.16-1.11-1.47-1.11-1.47-.91-.62.07-.6.07-.6 1 .07 1.53 1.03 1.53 1.03.87 1.52 2.34 1.07 2.91.83.09-.65.35-1.09.63-1.34-2.22-.25-4.55-1.11-4.55-4.92 0-1.11.38-2 1.03-2.71-.1-.25-.45-1.29.1-2.64 0 0 .84-.27 2.75 1.02.79-.22 1.65-.33 2.5-.33s1.71.11 2.5.33c1.91-1.29 2.75-1.02 2.75-1.02.55 1.35.2 2.39.1 2.64.65.71 1.03 1.6 1.03 2.71 0 3.82-2.34 4.66-4.57 4.91.36.31.69.92.69 1.85V21c0 .27.16.59.67.5C19.14 20.16 22 16.42 22 12A10 10 0 0 0 12 2"/></svg>
</span>
<span>GitHub</span>
<nav>
<a href="https://github.com/AkselAllas" class="md-author" title="@AkselAllas">
<img src="https://avatars.githubusercontent.com/u/26136082?v=4&size=72" alt="AkselAllas">
</a>
<a href="https://github.com/pravindahal" class="md-author" title="@pravindahal">
<img src="https://avatars.githubusercontent.com/u/894731?u=04797a957b10f54b8ee71dc4e0a6fb97e3b6a153&v=4&size=72" alt="Pravin Dahal">
</a>
<a href="https://github.com/nhhagen" class="md-author" title="@nhhagen">
<img src="https://avatars.githubusercontent.com/u/1181536?u=69e016bd41118554d5bd638d129e6f69238c344d&v=4&size=72" alt="Niels Henrik Hagen">
</a>
</nav>
</span>
</aside>
</article>
</div>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"annotate": null, "base": "../../../../../..", "features": [], "search": "../../../../../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="../../../../../../assets/javascripts/bundle.d7400e89.min.js"></script>
</body>
</html>