308 lines
18 KiB
HTML
308 lines
18 KiB
HTML
<!doctype html>
|
|
<html lang="en">
|
|
<head>
|
|
<title>PostgreSQL: pg_upgrade can result in early wraparound on databases with high
|
|
transaction load</title>
|
|
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
|
<meta http-equiv="Content-Type" content="text/xhtml; charset=utf-8" />
|
|
<meta property="og:url" content="https://www.postgresql.org/message-id/CALSof1GM6i21BLr8PsFiRYazakojvesdc%2B_MiR-L_V5NNkRuWg%40mail.gmail.com" />
|
|
<meta property="og:type" content="article" />
|
|
<meta property="article:author" content="Jason Harvey" />
|
|
<meta property="article:published_time" content="2019-05-20T10:10:17" />
|
|
<meta property="og:image" content="https://www.postgresql.org/media-archives/img/about/press/elephant.png" />
|
|
<meta property="og:title" content="pg_upgrade can result in early wraparound on databases with high
|
|
transaction load" />
|
|
<meta property="og:description" content="Hello, This week I upgraded one of my large(2.8TB), high-volume databases from 9 to 11. The upgrade itself went fine. …" />
|
|
<meta property="og:site_name" content="PostgreSQL Mailing List Archives" />
|
|
|
|
<meta name="theme-color" content="#336791"/>
|
|
<meta name="copyright" content="The PostgreSQL Global Development Group" />
|
|
<link href="/media-archives/css/fontawesome.css?51485f82" rel="stylesheet">
|
|
<link rel="stylesheet" href="/media-archives/css/bootstrap.min.css?51485f82">
|
|
<link rel="shortcut icon" href="/favicon.ico" />
|
|
|
|
<link rel="stylesheet" type="text/css" href="/dyncss/base.css?51485f82">
|
|
|
|
</head>
|
|
<body>
|
|
<div class="container-fluid">
|
|
<div class="row justify-content-md-center">
|
|
<div class="col">
|
|
<!-- Header -->
|
|
<nav class="navbar navbar-expand-lg navbar-light bg-light">
|
|
<a class="navbar-brand p-0" href="/">
|
|
<img class="logo" src="/media-archives/img/about/press/elephant.png" alt="PostgreSQL Elephant Logo">
|
|
</a>
|
|
<button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#pgNavbar" aria-controls="pgNavbar" aria-expanded="false" aria-label="Toggle navigation">
|
|
<span class="navbar-toggler-icon"></span>
|
|
</button>
|
|
<div class="collapse navbar-collapse" id="pgNavbar">
|
|
<ul class="navbar-nav mr-auto">
|
|
<li class="nav-item p-2"><a href="/" title="Home">Home</a></li>
|
|
<li class="nav-item p-2"><a href="/about/" title="About">About</a></li>
|
|
<li class="nav-item p-2"><a href="/download/" title="Download">Download</a></li>
|
|
<li class="nav-item p-2"><a href="/docs/" title="Documentation">Documentation</a></li>
|
|
<li class="nav-item p-2"><a href="/community/" title="Community">Community</a></li>
|
|
<li class="nav-item p-2"><a href="/developer/" title="Developers">Developers</a></li>
|
|
<li class="nav-item p-2"><a href="/support/" title="Support">Support</a></li>
|
|
<li class="nav-item p-2"><a href="/about/donate/" title="Donate">Donate</a></li>
|
|
<li class="nav-item p-2"><a href="/account/" title="Your account">Your account</a></li>
|
|
</ul>
|
|
<form role="search" method="get" action="https://www.postgresql.org/search/">
|
|
<div class="input-group">
|
|
<input id="q" name="q" type="text" size="20" maxlength="255" accesskey="s" class="form-control" placeholder="Search for...">
|
|
<span class="input-group-btn">
|
|
<button class="btn btn-default" type="submit"><i class="fas fa-search"></i></button>
|
|
</span>
|
|
</div><!-- /input-group -->
|
|
</form>
|
|
</div>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="container-fluid margin archive">
|
|
<div class="row">
|
|
<div class="col-lg-2">
|
|
<div id="pgSideWrap">
|
|
<div id="pgSideNav">
|
|
<h2>Quick Links</h2>
|
|
<ul>
|
|
<li><a href="/community/">Community</a></li>
|
|
<li><a href="/community/contributors/">Contributors</a></li>
|
|
<li><a href="/list/">Mailing Lists</a></li>
|
|
<li><a href="/community/irc/">IRC</a></li>
|
|
|
|
<li><a href="/community/user-groups/">Local User Groups</a></li>
|
|
<li><a href="/about/events/">Events</a></li>
|
|
<li class="last-child"><a href="/community/international/">International Sites</a></li>
|
|
</ul>
|
|
</div>
|
|
</div> <!-- pgSideWrap -->
|
|
</div>
|
|
<div class="col-lg-10">
|
|
<div id="pgContentWrap">
|
|
|
|
|
|
<h1 class="subject">pg_upgrade can result in early wraparound on databases with high
|
|
transaction load</h1>
|
|
|
|
|
|
|
|
<table class="table-sm table-responsive message-header" style="border-bottom: 0.05em solid #E7EAE8; margin-bottom: 1em;">
|
|
<tr>
|
|
<th scope="row" style="width: 10%">From:</th>
|
|
<td>Jason Harvey <jason(at)reddit(dot)com></td>
|
|
</tr>
|
|
<tr>
|
|
<th scope="row">To:</th>
|
|
<td>pgsql-general(at)postgresql(dot)org</td>
|
|
</tr>
|
|
|
|
<tr>
|
|
<th scope="row">Subject:</th>
|
|
<td>pg_upgrade can result in early wraparound on databases with high
|
|
transaction load</td>
|
|
</tr>
|
|
<tr>
|
|
<th scope="row">Date:</th>
|
|
<td>2019-05-20 10:10:17</td>
|
|
</tr>
|
|
<tr>
|
|
<th scope="row">Message-ID:</th>
|
|
<td><a href="/message-id/CALSof1GM6i21BLr8PsFiRYazakojvesdc%2B_MiR-L_V5NNkRuWg%40mail.gmail.com">CALSof1GM6i21BLr8PsFiRYazakojvesdc+_MiR-L_V5NNkRuWg@mail.gmail.com</a></td>
|
|
</tr>
|
|
<tr>
|
|
<th class="align-middle" scope="row">Views:</th>
|
|
<td>
|
|
<a href="/message-id/flat/CALSof1GM6i21BLr8PsFiRYazakojvesdc%2B_MiR-L_V5NNkRuWg%40mail.gmail.com">Whole Thread</a> |
|
|
<a href="#" data-ref="/message-id/raw/CALSof1GM6i21BLr8PsFiRYazakojvesdc%2B_MiR-L_V5NNkRuWg%40mail.gmail.com" class="post-link">Raw Message</a> |
|
|
<a href="#" data-ref="/message-id/mbox/CALSof1GM6i21BLr8PsFiRYazakojvesdc%2B_MiR-L_V5NNkRuWg%40mail.gmail.com" class="post-link">Download mbox</a>
|
|
| <a href="#" data-ref="/message-id/resend/CALSof1GM6i21BLr8PsFiRYazakojvesdc%2B_MiR-L_V5NNkRuWg%40mail.gmail.com" class="post-link">Resend email</a>
|
|
</td>
|
|
</tr>
|
|
|
|
<tr>
|
|
<th class="align-middle" scope="row">Thread:</th>
|
|
<td>
|
|
<select id="thread_select" class="custom-select">
|
|
<option value="CALSof1GM6i21BLr8PsFiRYazakojvesdc%2B_MiR-L_V5NNkRuWg%40mail.gmail.com" selected="selected">2019-05-20 10:10:17 from Jason Harvey <jason(at)reddit(dot)com></option>
|
|
<option value="CAH2-Wz%3D9Cj8H0%3DGAiHducPETWf6ZXEzjk_pFkjPWnWH6hptpLA%40mail.gmail.com"> 2019-05-21 22:23:00 from Peter Geoghegan <pg(at)bowt(dot)ie></option>
|
|
<option value="20190615183759.GB239428%40rfd.leadboat.com"> 2019-06-15 18:37:59 from Noah Misch <noah(at)leadboat(dot)com></option>
|
|
<option value="20210423234256.hwopuftipdmp3okf%40alap3.anarazel.de"> 2021-04-23 23:42:56 from Andres Freund <andres(at)anarazel(dot)de></option>
|
|
<option value="20210424002827.GO7256%40telsasoft.com"> 2021-04-24 00:28:27 from Justin Pryzby <pryzby(at)telsasoft(dot)com></option>
|
|
<option value="20210424010005.fttcfskp3unr4jiu%40alap3.anarazel.de"> 2021-04-24 01:00:05 from Andres Freund <andres(at)anarazel(dot)de></option>
|
|
<option value="fe006d56-85f1-5f1e-98e7-05b53dff4f51%40amazon.com"> 2021-05-04 08:17:49 from "Drouvot, Bertrand" <bdrouvot(at)amazon(dot)com> 📎</option>
|
|
<option value="78bf65ea-517a-51ed-947b-9728a6ace513%40amazon.com"> 2021-05-18 11:26:38 from "Drouvot, Bertrand" <bdrouvot(at)amazon(dot)com> 📎</option>
|
|
<option value="20210727023904.GA19774%40momjian.us"> 2021-07-27 02:39:04 from Bruce Momjian <bruce(at)momjian(dot)us></option>
|
|
<option value="3019dc16-f4c7-bfa0-fe15-eac36b4dd451%40amazon.com"> 2021-07-27 07:25:22 from "Drouvot, Bertrand" <bdrouvot(at)amazon(dot)com></option>
|
|
<option value="20210727125002.GD19774%40momjian.us"> 2021-07-27 12:50:02 from Bruce Momjian <bruce(at)momjian(dot)us></option>
|
|
<option value="1973418.1657040382%40sss.pgh.pa.us"> 2022-07-05 16:59:42 from Tom Lane <tgl(at)sss(dot)pgh(dot)pa(dot)us></option>
|
|
<option value="220cb128-4f52-237b-66d8-8dbcd75ead53%40dunslane.net"> 2022-07-05 18:52:57 from Andrew Dunstan <andrew(at)dunslane(dot)net></option>
|
|
<option value="1997756.1657048658%40sss.pgh.pa.us"> 2022-07-05 19:17:38 from Tom Lane <tgl(at)sss(dot)pgh(dot)pa(dot)us></option>
|
|
<option value="67a252e2-c847-f1f0-106e-ea4f7ffc543a%40dunslane.net"> 2022-07-05 21:25:43 from Andrew Dunstan <andrew(at)dunslane(dot)net></option>
|
|
<option value="2021227.1657058707%40sss.pgh.pa.us"> 2022-07-05 22:05:07 from Tom Lane <tgl(at)sss(dot)pgh(dot)pa(dot)us></option>
|
|
<option value="CAH2-Wz%3D9CDyXZYh9jL9tM9QqeB5da9DDNUQp%3DYEpwKp%3Dz5r-xQ%40mail.gmail.com"> 2022-07-05 19:41:00 from Peter Geoghegan <pg(at)bowt(dot)ie></option>
|
|
<option value="CAH2-Wzm56aDZwhk99ka%2BTWMDDSv%3DKyEFt8uby9iww51w1_1Lpg%40mail.gmail.com"> 2022-07-05 19:50:12 from Peter Geoghegan <pg(at)bowt(dot)ie></option>
|
|
<option value="ECFFD994-D2A9-4E74-A607-01AF4C252340%40yesql.se"> 2024-05-16 06:11:37 from Daniel Gustafsson <daniel(at)yesql(dot)se> 📎</option>
|
|
<option value="2122064.1715881657%40sss.pgh.pa.us"> 2024-05-16 17:47:37 from Tom Lane <tgl(at)sss(dot)pgh(dot)pa(dot)us></option>
|
|
<option value="6D72E937-528A-4744-B01A-1158ABEFE3A5%40yesql.se"> 2024-05-17 12:35:42 from Daniel Gustafsson <daniel(at)yesql(dot)se></option>
|
|
|
|
</select>
|
|
</td>
|
|
</tr>
|
|
|
|
|
|
<tr>
|
|
<th scope="row">Lists:</th>
|
|
<td>
|
|
|
|
<span class="listname"><a href="/list/pgsql-general/since/201905200000">pgsql-general</a></span>
|
|
|
|
<span class="listname"><a href="/list/pgsql-hackers/since/201905200000">pgsql-hackers</a></span>
|
|
|
|
</td>
|
|
</tr>
|
|
|
|
</table>
|
|
|
|
<div class="message-content"><p>Hello,</p>
|
|
|
|
<p>This week I upgraded one of my large(2.8TB), high-volume databases from 9<br>to 11. The upgrade itself went fine. About two days later, we unexpectedly<br>hit transaction ID wraparound. What was perplexing about this was that the<br>age of our oldest `datfrozenxid` was only 1.2 billion - far away from where<br>I'd expect a wraparound. Curiously, the wraparound error referred to a<br>mysterious database of `OID 0`:</p>
|
|
|
|
<p>UPDATE ERROR: database is not accepting commands to avoid wraparound data<br>loss in database with OID 0</p>
|
|
|
|
<p>We were able to recover after a few hours by greatly speeding up our vacuum<br>on our largest table.</p>
|
|
|
|
<p>In a followup investigation I uncovered the reason we hit the wraparound so<br>early, and also the cause of the mysterious OID 0 message. When pg_upgrade<br>executes, it calls pg_resetwal to set the next transaction ID. Within<br>pg_resetwal is the following code:<br><a href="https://github.com/postgres/postgres/blob/6cd404b344f7e27f4d64555bb133f18a758fe851/src/bin/pg_resetwal/pg_resetwal.c#L440-L450" rel="nofollow">https://github.com/postgres/postgres/blob/6cd404b344f7e27f4d64555bb133f18a758fe851/src/bin/pg_resetwal/pg_resetwal.c#L440-L450</a></p>
|
|
|
|
<p>This sets the controldata to have a fake database (OID 0) on the brink of<br>transaction wraparound. Specifically, after pg_upgrade is ran, wraparound<br>will occur within around 140 million transactions (provided the autovacuum<br>doesn't finish first). I confirmed by analyzing our controldata before and<br>after the upgrade that this was the cause of our early wraparound.</p>
|
|
|
|
<p>Given the size and heavy volume of our database, we tend to complete a<br>vacuum in the time it takes around 250 million transactions to execute.<br>With our tunings this tends to be rather safe and we stay well away from<br>the wraparound point under normal circumstances.</p>
|
|
|
|
<p>Unfortunately we had no obvious way of knowing that the upgrade would place<br>our database upon the brink of wraparound. In fact, since this info is only<br>persisted in the controldata, the only way to discover this state to my<br>knowledge would be to inspect the controldata itself. Other standard means<br>of monitoring for wraparound risk involve watching `pg_database` or<br>`pg_class`, which in this case tells us nothing helpful since the fake<br>database present in the controldata is not represented in those stats.</p>
|
|
|
|
<p>I'd like to suggest that either the pg_upgrade->pg_resetwal behaviour be<br>adjusted, or the pg_upgrade documentation highlight this potential<br>scenario. I'm happy to contribute code and/or documentation pull requests<br>to accomplish this.</p>
|
|
|
|
<p>Thank you,<br>Jason Harvey<br><a href="http://reddit.com" rel="nofollow">reddit.com</a><br></p></div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<h3 class="messages" style="margin: 0.5em 0;">Responses</h3>
|
|
<ul>
|
|
|
|
<li class="message-responses">
|
|
<a href="/message-id/CAH2-Wz%3D9Cj8H0%3DGAiHducPETWf6ZXEzjk_pFkjPWnWH6hptpLA%40mail.gmail.com">Re: pg_upgrade can result in early wraparound on databases with high
|
|
transaction load</a> at 2019-05-21 22:23:00 from Peter Geoghegan
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
<h3 class="messages" style="margin: 0.5em 0;">Browse pgsql-general by date</h3>
|
|
<table class="table table-sm table-responsive message-nav">
|
|
<thead>
|
|
<tr>
|
|
<th scope="col"> </th>
|
|
<th scope="col">From</th>
|
|
<th scope="col">Date</th>
|
|
<th scope="col">Subject</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
|
|
<tr>
|
|
<th scope="row"><a href="/message-id/CA%2BbJJbxrcfDPaBA6v7SYFjthLBKcDk0FTA68cPJ%3DUoLUkDiQYw%40mail.gmail.com">Next Message</a></th>
|
|
<td>Francisco Olarte</td>
|
|
<td>2019-05-20 10:18:12</td>
|
|
<td><a href="/message-id/CA%2BbJJbxrcfDPaBA6v7SYFjthLBKcDk0FTA68cPJ%3DUoLUkDiQYw%40mail.gmail.com">Re: Bug in documentation (trim(...))?</a></td>
|
|
</tr>
|
|
|
|
|
|
<tr>
|
|
<th scope="row"><a href="/message-id/CAGZ55DSQL7K3g0zBJ4-8hJDeQW4HKxGwyprcw2T3dWJaDfrZvg%40mail.gmail.com">Previous Message</a></th>
|
|
<td>Johann Spies</td>
|
|
<td>2019-05-20 10:09:32</td>
|
|
<td><a href="/message-id/CAGZ55DSQL7K3g0zBJ4-8hJDeQW4HKxGwyprcw2T3dWJaDfrZvg%40mail.gmail.com">Bug in documentation (trim(...))?</a></td>
|
|
</tr>
|
|
|
|
</tbody>
|
|
</table>
|
|
|
|
<h3 class="messages" style="margin: 0.5em 0;">Browse pgsql-hackers by date</h3>
|
|
<table class="table table-sm table-responsive message-nav">
|
|
<thead>
|
|
<tr>
|
|
<th scope="col"> </th>
|
|
<th scope="col">From</th>
|
|
<th scope="col">Date</th>
|
|
<th scope="col">Subject</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
|
|
<tr>
|
|
<th scope="row"><a href="/message-id/CAJs94Ean4tMn_AJ5Qvo6yPDTs46zdguyEFbgzAD_%2BwGzHvcwyA%40mail.gmail.com">Next Message</a></th>
|
|
<td>Matwey V. Kornilov</td>
|
|
<td>2019-05-20 11:32:39</td>
|
|
<td><a href="/message-id/CAJs94Ean4tMn_AJ5Qvo6yPDTs46zdguyEFbgzAD_%2BwGzHvcwyA%40mail.gmail.com">[PATCH v2] Introduce spgist quadtree @<(point,circle) operator</a></td>
|
|
</tr>
|
|
|
|
|
|
<tr>
|
|
<th scope="row"><a href="/message-id/20190520.165905.261954393.horiguchi.kyotaro%40lab.ntt.co.jp">Previous Message</a></th>
|
|
<td>Kyotaro HORIGUCHI</td>
|
|
<td>2019-05-20 07:59:05</td>
|
|
<td><a href="/message-id/20190520.165905.261954393.horiguchi.kyotaro%40lab.ntt.co.jp">Re: Statistical aggregate functions are not working with PARTIAL
|
|
aggregation</a></td>
|
|
</tr>
|
|
|
|
</tbody>
|
|
</table>
|
|
|
|
|
|
|
|
|
|
|
|
<form id="mail_other_options_form" method="post" action="/"></form>
|
|
|
|
</div> <!-- pgContentWrap -->
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Footer -->
|
|
<footer id="footer">
|
|
<div class="container">
|
|
<div class="row">
|
|
<div class="col-md-12">
|
|
<ul>
|
|
<li><a href="https://twitter.com/postgresql"><img src="/media-archives/img/atpostgresql.png" alt="@postgresql"></a></li>
|
|
<li><a href="https://git.postgresql.org/gitweb/?p=postgresql.git"><img src="/media-archives/img/git.png" alt="Git"></a></li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<!-- Copyright -->
|
|
<div class="container">
|
|
<a href="/about/privacypolicy">Privacy Policy</a> |
|
|
<a href="/about/policies/coc/">Code of Conduct</a> |
|
|
<a href="/about/">About PostgreSQL</a> |
|
|
<a href="/about/contact/">Contact</a><br/>
|
|
<p>Copyright © 1996-2026 The PostgreSQL Global Development Group</p>
|
|
</div>
|
|
</footer>
|
|
<script src="/media-archives/js/jquery-3.2.1.slim.min.js?51485f82"></script>
|
|
<script src="/media-archives/js/popper.min.js?51485f82"></script>
|
|
<script src="/media-archives/js/bootstrap.min.js?51485f82"></script>
|
|
<script src="/media-archives/js/main.js?51485f82"></script>
|
|
</body>
|
|
</html>
|