Files
nexus/sreweekly/articles/66/03-how-we-implement-disaster-recovery-and-high-availability-with-postgres.html
2026-09-12 17:23:01 +08:00

18 lines
46 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html> <html class='no-js' lang='en' xml:lang='en'> <head> <meta charset="utf-8"/> <meta name="viewport" content="width=device-width,initial-scale=1,shrink-to-fit=no"/> <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/> <title>How we implement Disaster Recovery and High Availability with Postgres on Citus Cloud - Citus Data</title> <meta name="description" content="In the database world, two methods are used to protect data. High Availability (HA) replicates the latest database version virtually instantly. Disaster Recovery (DR) offers continuous protection by saving every database change, allowing restoration to any point in time. How are DR &amp; HA implemented in Citus Cloud?"/> <meta itemprop="name" content="How we implement Disaster Recovery and High Availability with Postgres on Citus Cloud"/> <meta itemprop="description" content="In the database world, two methods are used to protect data. High Availability (HA) replicates the latest database version virtually instantly. Disaster Recovery (DR) offers continuous protection by saving every database change, allowing restoration to any point in time. How are DR &amp; HA implemented in Citus Cloud?"/> <meta itemprop="image" content="https://cdn.citusdata.com/images/social/fractal-1200x630.jpg"/> <meta name="twitter:card" content="summary_large_image"/> <meta name="twitter:site" content="Citus Data"/> <meta name="twitter:title" content="How we implement Disaster Recovery and High Availability with Postgres on Citus Cloud"/> <meta name="twitter:description" content="In the database world, two methods are used to protect data. High Availability (HA) replicates the latest database version virtually instantly. Disaster Recovery (DR) offers continuous protection by saving every database change, allowing restoration to any point in time. How are DR &amp; HA implemented in Citus Cloud?"/> <meta name="twitter:image" content="https://cdn.citusdata.com/images/social/fractal-1200x630.jpg"/> <meta property="og:type" content="website"/> <meta property="og:title" content="How we implement Disaster Recovery and High Availability with Postgres on Citus Cloud"/> <meta property="og:image" content="https://cdn.citusdata.com/images/social/fractal-1200x630.jpg"/> <meta property="og:description" content="In the database world, two methods are used to protect data. High Availability (HA) replicates the latest database version virtually instantly. Disaster Recovery (DR) offers continuous protection by saving every database change, allowing restoration to any point in time. How are DR &amp; HA implemented in Citus Cloud?"/> <meta property="og:site_name" content="Citus Data"/> <meta property="og:locale" content="en_US"/> <meta content='https://cdn.citusdata.com/images/social/fractal-1200x630.jpg' name='thumbnail'> <meta content='max-image-preview:large' name='robots'> <link href='../../../../../apple-touch-icon.png' rel='apple-touch-icon' sizes='180x180'> <link href='../../../../../favicon-32x32-d90d6b33.png' rel='icon' sizes='32x32' type='image/png'> <link href='../../../../../favicon-16x16-893340cc.png' rel='icon' sizes='16x16' type='image/png'> <link href='/site.webmanifest' rel='manifest'> <link color='#239f49' href='../../../../../safari-pinned-tab-a9f2b371.svg' rel='mask-icon'> <meta content='#239f49' name='msapplication-TileColor'> <meta content='#239f49' name='theme-color'> <script>
document.documentElement.classList.remove("no-js");
</script> <link href='https://fonts.googleapis.com' rel='preconnect'> <link crossorigin='' href='https://fonts.gstatic.com' rel='preconnect'> <link href='https://fonts.googleapis.com/css2?family=Lato:wght@400;700&amp;display=swap' rel='stylesheet'> <link href='https://fonts.googleapis.com/css2?family=Lato:ital,wght@0,300;0,900;1,300;1,400;1,700;1,900&amp;display=swap' rel='stylesheet'> <link href="../../../../../assets/stylesheets/global-d87f7546.css" rel="stylesheet" loading="lazy"/> <link href="../../../../../assets/stylesheets/blog-7cd9044c.css" rel="stylesheet" loading="lazy"/> </head> <body class='blog blog_2017 blog_2017_03 blog_2017_03_23 blog_2017_03_23_a-look-into-disaster-recovery-and-high-availability-and-how-they-work-with-postgres-on-citus-cloud blog_2017_03_23_a-look-into-disaster-recovery-and-high-availability-and-how-they-work-with-postgres-on-citus-cloud_index'> <noscript> <iframe aria-hidden='true' height='0' role='none' src='https://www.googletagmanager.com/ns.html?id=GTM-N69TBZ9' style='display:none;visibility:hidden' width='0'></iframe> </noscript> <div id='cookie-banner'></div> <div aria-label='notification bar' id='topbar' role='complementary'> <div class='page-content-wrapper'> <div class='promo-text'> <p>&#128250;&nbsp;All 44 POSETTE: An Event for Postgres 2026 talks are <a href='https://aka.ms/posette-playlist' target='_blank' rel='noopener'>now available on YouTube</a>.</p> <button aria-label='Close notification bar button' id='hideTop' onclick='javascript:;'> <i aria-hidden='true' class='icon'> <svg alt='close icon' style='transform: scale(1.4)' width='13'> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-times-circle'></use> </svg> </i> </button> </div> </div> </div> <div class='new' id='site-nav'> <a href='javascript:void(0)' id='skipToMain' role='button' tabindex='0'>Skip navigation</a> <div class='mobile-nav-flag'></div> <div id='holder'> <div class='page-content-wrapper'> <nav role='navigation'> <div id='logo'> <a aria-label='link to homepage' class='logo' href='/' name='header-logo-link'> <svg alt="Citus logo" class="citus-logo-svg" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 145 32.86" alt="Citus logo" width="145" height="33"><path d="M103.01 3.33h3.45v19.06h-3.45v-1.45a6.78 6.78 0 01-2 1.38 5.71 5.71 0 01-2.2.42 6.09 6.09 0 01-4.61-2.06 7.2 7.2 0 01-1.95-5.14 7.38 7.38 0 011.89-5.22 6 6 0 014.57-2 5.87 5.87 0 012.32.47 6.47 6.47 0 012 1.4zm-3.63 8.14a3.51 3.51 0 00-2.66 1.13 4.51 4.51 0 000 5.83 3.48 3.48 0 002.65 1.15 3.56 3.56 0 002.7-1.13 4.14 4.14 0 001.07-3 4 4 0 00-1.07-2.91 3.62 3.62 0 00-2.69-1.07zm19.18-2.83h3.45v13.75h-3.45v-1.45a6.78 6.78 0 01-2 1.38 5.71 5.71 0 01-2.2.42 6.09 6.09 0 01-4.61-2.06 7.2 7.2 0 01-2-5.14 7.38 7.38 0 011.89-5.22 6 6 0 014.57-2 5.87 5.87 0 012.32.47 6.47 6.47 0 012 1.4zm-3.63 2.83a3.51 3.51 0 00-2.66 1.13 4.51 4.51 0 000 5.83 3.48 3.48 0 002.65 1.15 3.56 3.56 0 002.7-1.13 4.14 4.14 0 001.07-3 4 4 0 00-1.07-2.91 3.62 3.62 0 00-2.69-1.07zm10.26-7.9h3.45v5.07h2.05v3h-2.05v10.78h-3.45V11.61h-1.77v-3h1.77zm16.36 5.07H145v13.75h-3.45v-1.45a6.62 6.62 0 01-2 1.38 5.72 5.72 0 01-2.21.42 6.09 6.09 0 01-4.6-2.06 7.2 7.2 0 01-2-5.14 7.42 7.42 0 011.88-5.22 6 6 0 014.57-2 5.88 5.88 0 012.33.47 6.32 6.32 0 012 1.4zm-3.62 2.83a3.49 3.49 0 00-2.66 1.13 4 4 0 00-1.06 2.9 4.09 4.09 0 001.08 2.93 3.48 3.48 0 002.65 1.15 3.56 3.56 0 002.7-1.13 4.14 4.14 0 001.07-3 4 4 0 00-1.07-2.91 3.62 3.62 0 00-2.71-1.07zM23.8 13.18c.49-2.48-6.59-3.52-10.65 1.48-4.57 5.62 4.62 11.47 15 3.61 6.67-5 8.78-11.95 4.86-15.5C25.49-4.08 7.1 2.34.98 16.04 7.13 5.67 22.56.77 28.71 7.1c2.57 2.64 1.14 7.95-3.58 10.22-7.63 3.54-6.87-4.28-1.33-4.14z" fill="#09436a"/><path d="M53.37 11.42l-2.87 1.62a4.25 4.25 0 00-3.45-1.51 4.35 4.35 0 00-3.14 1.16 4 4 0 00-1.19 3 3.78 3.78 0 001.15 2.86 4.19 4.19 0 003 1.11 4.37 4.37 0 003.6-1.58l2.72 1.87a8.39 8.39 0 01-11.92.73 7.21 7.21 0 01-1.06-8.71A7.19 7.19 0 0143 9.32a8.3 8.3 0 014-1 8.19 8.19 0 013.68.82 7 7 0 012.69 2.28zm-.76-5.06V3.57h7.19v2.79zm1.87 2.28h3.45v13.75h-3.45zm7-5.07h3.45v5.07h2.05v3h-2.05v10.78H61.5V11.61h-1.77v-3h1.77zm6.86 5.07h3.49v6.62a9.07 9.07 0 00.27 2.69 2.32 2.32 0 00.85 1.17 2.8 2.8 0 002.92 0 2.41 2.41 0 00.89-1.2 9.58 9.58 0 00.21-2.55V8.64h3.45v5.82a14.54 14.54 0 01-.56 4.93 5.45 5.45 0 01-2.05 2.49 6.33 6.33 0 01-3.44.86 6.05 6.05 0 01-3.66-1 5.26 5.26 0 01-2-2.82 17 17 0 01-.41-4.55zm22.5 1.92l-2.13 2.14a3.62 3.62 0 00-2.37-1.29 1.53 1.53 0 00-.91.24.77.77 0 00-.32.62.8.8 0 00.2.51 4.23 4.23 0 001 .64l1.27.63a7.42 7.42 0 012.74 2 4.17 4.17 0 01-.6 5.46 5.14 5.14 0 01-3.61 1.22 5.73 5.73 0 01-4.8-2.35l2.12-2.31a5 5 0 001.42 1.14 3.2 3.2 0 001.45.44 1.72 1.72 0 001.1-.33 1 1 0 00.42-.76c0-.53-.5-1.05-1.51-1.55l-1.16-.58c-2.22-1.12-3.34-2.53-3.34-4.21a3.65 3.65 0 011.29-2.78 4.58 4.58 0 013.22-1.16 5.62 5.62 0 012.52.59 5.51 5.51 0 012.02 1.69zM23.37 7.75c-12.63-3.8-29.76 11.72-21 21.92 5.85 6.79 26.17 3.5 32.73-17-7.77 14.16-22.37 16.22-27.9 12.17C.73 20.04 8.84 6.67 23.39 7.75z" fill="#1e873d"/></svg> </a> </div> <div class='nav-wrapper'> <div class='tagline'> <a data-github='citusdata/citus' href='https://github.com/citusdata/citus' rel='noopener' target='_blank'> <i aria-hidden='true' class='icon icon-lg'> <svg alt='GitHub icon'> <title>GitHub icon</title> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-github'></use> </svg> </i> <span class='sr-only'>go to Citus GitHub repo with</span> &nbsp;<strong data-stars title="Citus GitHub stargazers">10,100</strong> <span class='sr-only'>stargazers</span> </a> </div> <div class='expander' id='nav-menu'> <button aria-controls='menu1' aria-expanded='false' aria-label='mobile menu toggle' class='toggle expander-trigger' id='drop'> <span id='nav-icon'> <span></span> <span></span> <span></span> <span></span> </span> </button> <ul aria-labelledby='drop' class='menu expander-content expander-hidden' id='menu1'> <li class='button desktop-hidden'> <a class='btn btn-block nav-btn' href='/download/'>Download</a> </li> <li class='first'> <a class='nav-link' href='/overview/'>Overview</a> </li> <li class='dropdown'> <button aria-controls='menu4' aria-expanded='false' class='toggle' id='drop-3'>Use Cases</button> <ul aria-hidden='true' aria-labelledby='drop-3' id='menu4'> <li> <a class='nav-link' href='/use-cases'>Overview</a> </li> <li> <a class='nav-link' href='/use-cases/multi-tenant-apps'>Multi-Tenant Applications</a> </li> <li> <a class='nav-link' href='/use-cases/real-time-analytics'>Real-Time Analytics</a> </li> </ul> </li> <li class='dropdown'> <button aria-controls='menu5' aria-expanded='false' class='toggle' id='drop-4'>Resources</button> <ul aria-hidden='true' aria-labelledby='drop-4' id='menu5'> <li> <a class='nav-link' href='/getting-started/'>Getting Started</a> </li> <li> <a class='nav-link' href='https://learn.microsoft.com/postgresql/citus/' rel='noopener' target='_blank'>Docs</a> </li> <li> <a class='nav-link' href='/join-newsletter'>Newsletters</a> </li> <li> <a class='nav-link' href='/faq'>FAQ</a> </li> <li> <a aria-label='Talking Postgres Podcast' class='nav-link' href='https://talkingpostgres.com/' rel='noopener' target='_blank'>Podcast</a> </li> </ul> </li> <li class='dropdown'> <button aria-controls='menu3' aria-expanded='false' class='toggle' id='drop-2'>About</button> <ul aria-hidden='true' aria-labelledby='drop-2' id='menu3'> <li> <a aria-label='contact us' class='nav-link' href='/about/contact_us/'><span aria-hidden="true">Contact Us</span></a> </li> <li> <a class='nav-link' href='/about/our-story/'>Our Story</a> </li> <li> <a class='nav-link' href='/events'>Events</a> </li> <li> <a class='nav-link' href='/jobs/'>Careers</a> </li> </ul> </li> <li> <a class='nav-link' href='/updates/'>Updates</a> </li> <li> <a class='nav-link' href='/blog/'>Blog</a> </li> <li> <a class='nav-link' href='https://learn.microsoft.com/postgresql/citus/' rel='noopener' target='_blank'>Docs</a> </li> <li class='github-stars'> <a class='nav-link' data-github='citusdata/citus' href='https://github.com/citusdata/citus' rel='noopener' target='_blank'> <i aria-hidden='true' class='icon icon-lg'> <svg alt='GitHub icon'> <title>GitHub icon</title> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-github'></use> </svg> </i> <span class='sr-only'>go to Citus GitHub repo with</span> &nbsp;<strong data-stars title="Citus GitHub stargazers">10,100</strong> <span class='sr-only'>stargazers</span> </a> </li> <li> <a class='btn btn-ghost' data-kill='' href='javascript:void(0)' id='closeMobileMenu' role='button' tabindex='0'>Close Menu</a> </li> <li class='button mobile-hidden'> <a class='btn btn-block nav-btn' href='/download/'>Download</a> </li> </ul> </div> </div> </nav> </div> </div> </div> <div id='mainContent'> <div aria-hidden='true' class='g-recaptcha' id='recaptchaBox'></div> </div> <link as='image' href='https://cdn.citusdata.com/images/social/fractal-1200x630.webp' rel='preload' type='image/webp'> <style>#article-header.blog-header{background-image:url("https://cdn.citusdata.com/images/social/fractal-1200x630.webp")}.no-webp #article-header.blog-header{background-image:url("https://cdn.citusdata.com/images/social/fractal-1200x630.jpg")}.no-js #article-header.blog-header{background-image:url("https://cdn.citusdata.com/images/social/fractal-1200x630.jpg")}</style> <header class='blog-header new pg-section' id='article-header' style=''> <div class='blog-title text-center text-white' style=''> <div class='page-content-wrapper'> <h1 class='h2'> <div aria-hidden='true' class='deco-lines'> <div class='left-lines'> <div class='thick-line'></div> <div class='thin-line'></div> </div> <div class='right-lines'> <div class='thin-line'></div> <div class='thick-line'></div> <div class='thin-line'></div> </div> </div> <small class='long-title'> How we implement Disaster Recovery and High Availability with Postgres on Citus Cloud </small> </h1> <p> <small> <span class='font-regular'>Written by</span> <a class='underline text-white' href='/blog/authors/daniel-farina/'><strong>Daniel Farina</strong></a> <br> March 23, 2017 </small> </p> </div> </div> </header> <main class='blog-article-content new' data-sticky-container=''> <div aria-hidden='true' class='green-arrow'></div> <div class='page-content-wrapper wide'> <aside aria-label='left sidebar' class='sidebar'> <div class='sticky-cta' data-margin-top='90' data-sticky-class='is-stuck' data-sticky-for='1200' data-sticky=''> <div class='share-blog sidebar-box'> <div aria-hidden='true' class='arrow-border'></div> <div class='social-media-icons new'> <h6><small>SHARE THIS POST</small></h6> <p> <span class='link-and-tooltip'> <a aria-labelledby='share-twitter-desc' class='twitter customer share' href='https://twitter.com/share?url=https://www.citusdata.com/blog/2017/03/23/a-look-into-disaster-recovery-and-high-availability-and-how-they-work-with-postgres-on-citus-cloud/&amp;text=How we implement Disaster Recovery and High Availability with Postgres on Citus Cloud&amp;via=citusdata' role='button' target='_blank'> <i aria-hidden='true' class='icon icon-twitter'> <svg height='42' width='32'> <title>X icon</title> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-x'></use> </svg> </i> </a> <span id='share-twitter-desc' role='tooltip'>Share on X</span> </span> <span class='link-and-tooltip'> <a aria-labelledby='share-linkedin-desc' class='linkedin customer share' href='https://www.linkedin.com/shareArticle?mini=true&amp;url=https://www.citusdata.com/blog/2017/03/23/a-look-into-disaster-recovery-and-high-availability-and-how-they-work-with-postgres-on-citus-cloud/' role='button' target='_blank'> <i aria-hidden='true' class='icon icon-linkedin'> <svg height='42' width='29'> <title>LinkedIn icon</title> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-linkedin'></use> </svg> </i> </a> <span id='share-linkedin-desc' role='tooltip'>Share on<br/>LinkedIn</span> </span> <span class='button-and-tooltip'> <button aria-labelledby='share-link-desc' class='link customer blog-link' data-clipboard-text='https://www.citusdata.com/blog/2017/03/23/a-look-into-disaster-recovery-and-high-availability-and-how-they-work-with-postgres-on-citus-cloud/' data-default-text='Copy link' id='share-link'> <i aria-hidden='true' class='icon icon-link'> <svg height='42' width='31'> <title>link icon</title> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-chain'></use> </svg> </i> </button> <span id='share-link-desc' role='tooltip'>Copy link</span> </span> </p> </div> <div aria-hidden='true' class='white-arrow'></div> </div> <div class='newsletter-signup sidebar-box mini-signup'> <h6 style='line-height: 1.3'>Get our monthly newsletter</h6> <div class='blog-marketo'> <form class='mktoForm' data-formid='1355' data-forminstance='sticky-signup'></form> <p aria-atomic='true' aria-live='assertive' id='newsletter_success_sticky' role='alert'></p> </div> </div> </div> </aside> <section> <div class='blog-article-content__text'> <p>AWS is the leader when it comes to the cloud, and for good reason. AWS is well ahead in the quality and breadth of services they offer.</p> <p>However, when a service is running at the scale of AWS, it is natural to expect some failures to occur. According to AWS EBS availability is designed for 99.999%.</p> <p>The annual failure rate (AFR) is 0.1% - 0.2%, where failure means a complete or partial failure. For example, if you had 1,000 EBS discs, you should expect 1 or 2 to have a failure per year. In our experience, partial failure is significantly more common than a complete loss. Even so, a partial loss can take a lot of time to resolve and can still be debilitating to a business.</p> <p>Over the years, there have been some AWS failures that made news headlines due to havoc caused for both companies and their users. These incidents put a spotlight on AWS’ imperfections. </p> <p>In 2011, a major AWS failure took down hundreds of sites including Quora and Reddit. From this outage, <a href="http://techblog.netflix.com/2011/04/lessons-netflix-learned-from-aws-outage.html">Netflix learnt to always be prepared</a> by intentionally simulating failures with a service called Chaos Monkey. In 2013, one of the biggest failures happened when the AWS U.S. East data center went down, affecting Netflix, Airbnb, Instagram and Amazon.com itself. Just weeks ago on February 28th, AWS S3 storage had a major outage due to high error rates, again in the U.S. East data center. Prominent companies including Atlassian, Slack and Expedia were hit.</p> <h3>Introduction to High Availability and Disaster Recovery</h3> <p>In the real world, insurance is used to manage risk when a natural disaster such as a hurricane or flood strikes. In the database world, there are two critical methods of insurance. High Availability (HA) replicates the latest database version virtually instantly. Disaster Recovery (DR) offers continuous protection by saving every database change, allowing database restoration to any point in time.</p> <p>In what follows, we’ll dig deeper as to what disaster recovery and high availability are, as well as how we’ve implemented them for Citus Cloud.</p> <h3>What is High Availability and Disaster Recovery?</h3> <p>High availability and disaster recovery are both forms of data backups that are mutually exclusive and inter-related. The difference between them, is that HA has a secondary reader database replica (often referred to as stand-by or follower) ready to take over at any moment, but DR just writes to cold storage (in the case of Amazon that’s S3) and has latency in the time for the main database to recover data.</p> <h3>Overview of High Availability</h3> <p><img src="https://cdn.citusdata.com/images/blog/HADR_HA.gif" style="float:right; width:40%;"></p> <p>For HA, any data that is written to a primary database called the Writer is instantly replicated onto a secondary database called the Reader in real-time, through a stream called a <a href="https://www.postgresql.org/docs/9.4/static/wal-intro.html">WAL</a> or Write-Ahead-Log.</p> <p>To ensure HA is functioning properly, Citus Cloud runs health checks every 30 seconds. If the primary fails and data can’t be accessed after six consecutive attempts, a failover is initiated. This means the primary node will be replaced by the standby node and a new standby will be created.</p> <h3>Overview of Disaster Recovery</h3> <p><img src="https://cdn.citusdata.com/images/blog/HADR_DR.gif" style="float:right; width:57%;"></p> <p>For DR, read-only data is replayed from colder storage. On AWS this is from S3, and for Postgres this is downloaded in 16 MB pieces. On Citus Cloud this happens via WAL-E, using precisely the same procedure as creating a new standby for HA. <a href="https://github.com/wal-e/wal-e">WAL-E</a> is an open source tool initially developed by our team, for archiving PostgreSQL WAL (Write Ahead Log) files quickly, continuously and with a low operational burden.</p> <p>This means we can restore your database by fetching the base backup and replaying all of the WAL files on a fresh install in the event of hardware failure, data corruption or other failure modes</p> <p>On Citus Cloud prior to kicking off the DR recovery process, the AWS EC2 instance is automatically restarted. This process usually takes 7±2 minutes. If it restarts without any issues, the setup remains the same. If the EC2 instance fails to restart, a new instance is created. This happens at a rate of at least 30MB/second, so 512GB of data would take around 5 hours.</p> <h3>How High Availability and Disaster Recovery fit together</h3> <p>While some may be familiar many are not acutely aware of the relationship between HA and DR.</p> <p>Although it’s technically possible to have one without the other, they are unified in that the HA streaming replication and DR archiving transmit the same bytes.</p> <p>For HA the primary “writer” database replicates data through streaming replication to the secondary “reader” database. For DR, the same data is read from S3. In both cases, the “reader” database downloads the WAL and applies it incrementally.</p> <p>Since DR and HA gets regularly used for upgrades and side-grades, the DR system is maintained with care. We ourselves rely on it for releasing new production features.</p> <h3>Disaster Recovery takes a little extra work but gives greater reliability</h3> <p>You might think that if HA provides virtually instant backup reliability, so ‘Why bother with DR?’ There are some compelling reasons to use DR in conjunction with HA including cost, reliability and control. </p> <p>From a cost efficiency perspective, since HA based on EBS and EC2 is a mirrored database, you have to pay for every layer of redundancy added. However, DR archives in S3 are often 10-30% of the monthly cost of a database instance. And with Citus Cloud the S3 cost is already covered for you in the standard price of your cluster.</p> <p>From reliability perspective, S3 has proven to be up to a thousand times more reliable than EBS and EC2, though a more reasonable range is ten to a hundred times. S3 archives also have the advantage of immediate restoration, even while teams try to figure out what’s going on. Conversely, sometimes EBS volume availability can be down for hours with uncertainty it will completely restore.</p> <p>From a control perspective, using DR means a standby database can be created while reducing the impact on the primary database. It also has the capability of being able to recover a database from a previous version.</p> <h3>Trade-offs between latency and reliability</h3> <p>There is a long history of trade-offs between latency and reliability, dating back to when the gold standard for backups were on spools of tape.</p> <p>Writing data to and then reading data from S3 offers latencies that are 100 to 1,000 times longer than streaming bytes between two computers as seen in streaming replication. However, S3&#39;s availability and durability are both in excess of ten times better than an EBS volume.</p> <p>On the other hand, the throughput of S3 is excellent: with parallelism, and without downstream bottlenecks, one can achieve multi-gigabit throughput in backup and WAL reading and writing.</p> <h3>How High Availability and Disaster Recovery is used for crash recovery</h3> <p>When many customers entrust your company with their data, it is your duty to keep it safe under all circumstances. So when the most severe database crashes strike, you need to be ready to recover.</p> <p>Our team is battle-hardened from years of experience as the original Heroku Postgres team, managing over 1.5 million databases. Running at that scale with constant risks of failure, meant that it was important to automate recovery processes. </p> <p>Such crashes are a nightmare. But crash recovery is a way to make sure you sleep well at night by making sure none of your or your customers data is lost and your downtime is minimal.</p> <h3>How High Availability and Disaster Recovery is used for upgrades and side-grades</h3> <p>Here at Citus, our customers use the same HA and DR plumbing that we rely on when we upgrade hardware or sidegrade Citus versions.</p> <p>A common example of this is when we run upgrades. In these cases we create a new “reader” database which will get caught up to the “writer” then we’ll pause writes to the “writer”, run the necessary upgrade on the “reader” which could be a Citus version upgrade or even a Postgres major version upgrade. Once that is completed we then conduct the same failover process as we do during HA to turn the “reader” into the “writer”.</p> <h3>In conclusion</h3> <p>We’ve covered a lot on the topic of HA and DR. Yet there is even more we could unpack. In the future, we’ll look deeper into these topics and others around how we manage and operate databases.</p> <p>If you are unfamiliar with Citus Cloud, Citus Cloud is a managed service delivered on top of AWS that allows you to scale out your storage, memory and compute–horizontally scaling your database. If you’re starting to outgrow single node Postgres and need help scaling it out, feel free to <a href="/about/contact_us">contact us</a> if you have any questions.</p> <ul class='meta__tags-list desktop-only'> <li class='meta__tag'> <a class='tag-link' href='/blog/tags/citus-cloud/'>Citus Cloud</a> </li> <li class='meta__tag'> <a class='tag-link' href='/blog/tags/high-availability/'>high availability</a> </li> <li class='meta__tag'> <a class='tag-link' href='/blog/tags/replication/'>replication</a> </li> <li class='meta__tag'> <a class='tag-link' href='/blog/tags/wal/'>WAL</a> </li> </ul> <div class='share-blog'> <div class='social-media-icons new'> <h6><small>SHARE THIS POST</small></h6> <p> <span class='link-and-tooltip'> <a aria-labelledby='share-twitter-desc2' class='twitter customer share' href='https://twitter.com/share?url=https://www.citusdata.com/blog/2017/03/23/a-look-into-disaster-recovery-and-high-availability-and-how-they-work-with-postgres-on-citus-cloud/&amp;text=How we implement Disaster Recovery and High Availability with Postgres on Citus Cloud&amp;via=citusdata' role='button' target='_blank'> <i aria-hidden='true' class='icon icon-twitter'> <svg height='38' width='28'> <title>X icon</title> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-x'></use> </svg> </i> </a> <span id='share-twitter-desc2' role='tooltip'>Share on X</span> </span> <span class='link-and-tooltip'> <a aria-labelledby='share-linkedin-desc2' class='linkedin customer share' href='https://www.linkedin.com/shareArticle?mini=true&amp;url=https://www.citusdata.com/blog/2017/03/23/a-look-into-disaster-recovery-and-high-availability-and-how-they-work-with-postgres-on-citus-cloud/' role='button' target='_blank'> <i aria-hidden='true' class='icon icon-linkedin'> <svg height='38' width='26'> <title>LinkedIn icon</title> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-linkedin'></use> </svg> </i> </a> <span id='share-linkedin-desc2' role='tooltip'>Share on<br/>LinkedIn</span> </span> <span class='button-and-tooltip'> <button aria-labelledby='share-link-desc2' class='link customer blog-link' data-clipboard-text='https://www.citusdata.com/blog/2017/03/23/a-look-into-disaster-recovery-and-high-availability-and-how-they-work-with-postgres-on-citus-cloud/' data-default-text='Copy link' id='share-link2'> <small> <i aria-hidden='true' class='icon icon-link'> <svg class='sm-icon' height='13' width='13'> <title>link icon</title> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-chain'></use> </svg> </i> COPY LINK </small> </button> <span id='share-link-desc2' role='tooltip'>Copy link</span> </span> </p> </div> </div> <div class='meta-info'> <div class='meta-image has-bio'> <picture> <source srcset='../../../../../assets/images/team/blog_authors/daniel-87b469d4.webp' type='image/webp'> <source srcset='../../../../../assets/images/team/blog_authors/daniel-1a88e5c3.jpg' type='image/jpeg'> <img alt='Daniel Farina' loading='lazy' src='../../../../../assets/images/team/blog_authors/daniel-1a88e5c3.jpg'> </picture> </div> <div class='meta-text has-bio'> <small> <p> <strong> Written by <a class='underline' href='/blog/authors/daniel-farina/'>Daniel Farina</a> </strong> </p> <p class='mini-bio'>Former Principal Engineer at Citus Data & Microsoft. Managed thousands of Postgres databases in the cloud at Citus and Heroku. Creator & maintainer of WAL-E. History lover.</p> </small> </div> </div> <ul class='meta__tags-list mobile-only'> <li class='meta__tag'> <a class='tag-link' href='/blog/tags/citus-cloud/'>Citus Cloud</a> </li> <li class='meta__tag'> <a class='tag-link' href='/blog/tags/high-availability/'>high availability</a> </li> <li class='meta__tag'> <a class='tag-link' href='/blog/tags/replication/'>replication</a> </li> <li class='meta__tag'> <a class='tag-link' href='/blog/tags/wal/'>WAL</a> </li> </ul> </div> </section> <aside aria-label='right sidebar' class='sidebar'> <div class='popular-posts sidebar-box'> <h6> <small>POPULAR POSTS</small> </h6> <ul> <li> <a href='/blog/2024/02/08/whats-new-in-postgres-16-query-planner-optimizer/'> What&rsquo;s new in the Postgres 16 query planner &amp; optimizer <small> By David Rowley </small> </a> </li> <li> <a href='https://techcommunity.microsoft.com/t5/azure-database-for-postgresql/what-s-new-with-postgres-at-microsoft-2024-edition/ba-p/4140085' rel='noopener' target='_blank'> <span class='external-link'>What&rsquo;s new with Postgres at Microsoft 2024</span> <small> By Claire Giordano </small> </a> </li> <li> <a href='/blog/2024/07/09/say-hello-to-the-talking-postgres-podcast/'> Say hello to the Talking Postgres podcast <small> By Claire Giordano </small> </a> </li> <li> <a href='/blog/2023/10/19/new-citus-technical-readme-for-distributed-postgres/'> New Citus Technical README for distributed PostgreSQL <small> By Onder Kalaci </small> </a> </li> <li> <a href='/blog/2024/02/28/podcast-about-transitioning-from-dev-to-postgres-specialist/'> Podcast on transition from dev to PostgreSQL specialist with Derk van Veen <small> By Ari Padilla </small> </a> </li> </ul> <p> <a class='btn btn-block' href='/blog/'> LATEST POSTS&nbsp; <i aria-hidden='true' class='icon'> <svg> <title>arrow circle icon</title> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-arrow-circle-right'></use> </svg> </i> </a> </p> </div> <div class='sticky-cta' data-margin-top='90' data-sticky-class='is-stuck' data-sticky-for='900' data-sticky=''> <div class='share-blog sidebar-box'> <div class='social-media-icons new'> <h6 class='has-margin'><small>SHARE THIS POST</small></h6> <p> <span class='link-and-tooltip'> <a aria-labelledby='share-twitter-desc3' class='twitter customer share' href='https://twitter.com/share?url=https://www.citusdata.com/blog/2017/03/23/a-look-into-disaster-recovery-and-high-availability-and-how-they-work-with-postgres-on-citus-cloud/&amp;text=How we implement Disaster Recovery and High Availability with Postgres on Citus Cloud&amp;via=citusdata' role='button' target='_blank'> <i aria-hidden='true' class='icon icon-twitter'> <svg height='42' width='32'> <title>X icon</title> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-x'></use> </svg> </i> </a> <span id='share-twitter-desc3' role='tooltip'>Share on X</span> </span> <span class='link-and-tooltip'> <a aria-labelledby='share-linkedin-desc3' class='linkedin customer share' href='https://www.linkedin.com/shareArticle?mini=true&amp;url=https://www.citusdata.com/blog/2017/03/23/a-look-into-disaster-recovery-and-high-availability-and-how-they-work-with-postgres-on-citus-cloud/' role='button' target='_blank'> <i aria-hidden='true' class='icon icon-linkedin'> <svg height='42' width='29'> <title>LinkedIn icon</title> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-linkedin'></use> </svg> </i> </a> <span id='share-linkedin-desc3' role='tooltip'>Share on<br/>LinkedIn</span> </span> <span class='button-and-tooltip'> <button aria-labelledby='share-link-desc3' class='link customer blog-link' data-clipboard-text='https://www.citusdata.com/blog/2017/03/23/a-look-into-disaster-recovery-and-high-availability-and-how-they-work-with-postgres-on-citus-cloud/' data-default-text='Copy link' id='share-link3'> <i aria-hidden='true' class='icon icon-link'> <svg height='42' width='31'> <title>link icon</title> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-chain'></use> </svg> </i> </button> <span id='share-link-desc3' role='tooltip'>Copy link</span> </span> </p> </div> </div> </div> </aside> </div> </main> <div class='page-content-wrapper'> <div class='related-posts new' role='complementary'> <h6> <small> Related Content </small> </h6> <ul> <li> <a href='/blog/2023/09/22/adding-postgres-16-support-to-citus-12-1/'> <strong class='lead'> Adding Postgres 16 support to Citus 12.1, plus schema-based sharding </strong> <small> By Naisila Puka </small> <strong class='read-more'> Read more <i class='fa fa-arrow-circle-right'></i> </strong> </a> </li> <li> <a href='/blog/2023/06/21/distributed-postgres-benchmarks-using-hammerdb-by-gigaom/'> <strong class='lead'> Distributed PostgreSQL benchmarks using HammerDB </strong> <small> By Marco Slot </small> <strong class='read-more'> Read more <i class='fa fa-arrow-circle-right'></i> </strong> </a> </li> <li> <a href='https://techcommunity.microsoft.com/t5/azure-database-for-postgresql/what-s-new-with-postgres-at-microsoft-2024-edition/ba-p/4140085'> <strong class='lead'> What&rsquo;s new with Postgres at Microsoft, 2024 edition </strong> <small> By Claire Giordano </small> <strong class='read-more'> Read more <i class='fa fa-arrow-circle-right'></i> </strong> </a> </li> </ul> </div> </div> </body> <footer class='new-footer pg-section'> <div class='page-content-wrapper'> <div class='left-side'> <div class='footer-copyright msft'> <div class='logo-group'> <strong>Citus Data is now part of</strong> <img alt='Microsoft logo' height='21' loading='lazy' src='../../../../../assets/images/microsoft-logo-white-56fd3229.svg' style='height:21px' width='98'> </div> </div> <div class='keep-in-touch'> <ul class='footer-list' role='presentation'> <li class='footer-list__careers' role='presentation'> <a href='https://slack.citusdata.com/' target='_blank'>Join our Slack!</a> </li> <li class='footer-list__social' role='presentation'> <span class='link-and-tooltip'> <a aria-labelledby='footer-github-desc' href='https://github.com/citusdata/citus' rel='noopener' target='_blank'> <svg alt='GitHub icon'> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-github'></use> </svg> </a> <span class='noselect bottom-left' id='footer-github-desc' role='tooltip'>GitHub</span> </span> <span class='link-and-tooltip'> <a aria-labelledby='footer-stackoverflow-desc' href='https://stackoverflow.com/questions/tagged/citus' rel='noopener' target='_blank'> <svg alt='Stack Overflow icon'> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-stackoverflow'></use> </svg> </a> <span class='noselect' id='footer-stackoverflow-desc' role='tooltip'>Stack<br>Overflow</span> </span> <span class='link-and-tooltip'> <a aria-labelledby='footer-twitter-desc' href='https://www.twitter.com/citusdata' rel='noopener' target='_blank'> <svg alt='X icon'> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-x'></use> </svg> </a> <span class='noselect' id='footer-twitter-desc' role='tooltip'>X (Twitter)</span> </span> <span class='link-and-tooltip'> <a aria-labelledby='footer-slack-desc' href='https://slack.citusdata.com/' target='_blank'> <svg alt='Slack icon'> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-slack'></use> </svg> </a> <span class='noselect' id='footer-slack-desc' role='tooltip'>Slack</span> </span> <span class='link-and-tooltip'> <a aria-labelledby='footer-youtube-desc' href='https://www.youtube.com/playlist?list=PLixnExCn6lRq261O0iwo4ClYxHpM9qfVy' rel='noopener' target='_blank'> <svg alt='YouTube icon'> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-youtube'></use> </svg> </a> <span class='noselect' id='footer-youtube-desc' role='tooltip'>YouTube</span> </span> <span class='link-and-tooltip'> <a aria-labelledby='footer-rss-desc' href='https://www.citusdata.com/feed.xml' target='_blank'> <svg alt='RSS icon'> <use xlink:href='../../../../../assets/images/sprites/font-icons-main-76fd918f.svg#icon-feed'></use> </svg> </a> <span class='noselect bottom-right' id='footer-rss-desc' role='tooltip'>RSS feed</span> </span> </li> </ul> </div> <div class='copyright-info'> <span class='privacy-links'> <a class='underline' href='https://go.microsoft.com/fwlink/?LinkId=521839' rel='noopener' target='_blank'>Privacy Statement</a> &nbsp;|&nbsp; <a class='underline' href='https://www.microsoft.com/privacy/microsoft-consumer-health-data-privacy-policy' rel='noopener' target='_blank'>Consumer Health Privacy</a> </span> <br> <span class='rights'>&copy;2026 Citus Data, a Microsoft Company. All rights reserved.</span> </div> </div> <div class='footer-links-block'> <ul aria-label='Resources' class='footer-list'> <li aria-hidden='true' class='footer-list__heading' role='presentation'>Resources</li> <li class='footer-list__item'> <a aria-label='Resources, Getting Started' class='footer-list__link' href='/getting-started/'>Getting Started</a> </li> <li class='footer-list__item'> <a aria-label='Resources, Docs' class='footer-list__link' href='https://learn.microsoft.com/postgresql/citus/' rel='noopener' target='_blank'>Docs</a> </li> <li class='footer-list__item'> <a aria-label='Resources, Release Updates' class='footer-list__link' href='/updates/'>Release Updates</a> </li> <li class='footer-list__item'> <a aria-label='Resources, Support' class='footer-list__link' href='/support'>Support</a> </li> <li class='footer-list__item'> <a aria-label='Resources, Blog' class='footer-list__link' href='/blog/'>Blog</a> </li> </ul> <ul aria-label='Technology' class='footer-list'> <li aria-hidden='true' class='footer-list__heading' role='presentation'>Technology</li> <li class='footer-list__item'> <a aria-label='Technology, Overview' class='footer-list__link' href='/overview/'>Overview</a> </li> <li class='footer-list__item'> <a aria-label='Technology, Use Cases' class='footer-list__link' href='/use-cases'>Use Cases</a> </li> <li class='footer-list__item'> <a aria-label='Technology, Download' class='footer-list__link' href='/download/'>Download</a> </li> </ul> <ul aria-label='About' class='footer-list'> <li aria-hidden='true' class='footer-list__heading' role='presentation'>About</li> <li class='footer-list__item'> <a aria-label='About, Contact us' class='footer-list__link' href='/about/contact_us'>Contact Us</a> </li> <li class='footer-list__item'> <a aria-label='About, Our Story' class='footer-list__link' href='/about/our-story/'>Our Story</a> </li> <li class='footer-list__item'> <a aria-label='About, Events' class='footer-list__link' href='/events'>Events</a> </li> <li class='footer-list__item'> <a aria-label='About, Careers' class='footer-list__link' href='/jobs'>Careers</a> </li> <li class='footer-list__item'> <a aria-label='About, FAQ' class='footer-list__link' href='/faq'>FAQ</a> </li> </ul> <ul aria-label='Community' class='footer-list'> <li aria-hidden='true' class='footer-list__heading' role='presentation'>Community</li> <li class='footer-list__item'> <a aria-label='Community, GitHub repo' class='footer-list__link' href='https://github.com/citusdata/citus' rel='noopener' target='_blank'>GitHub repo</a> </li> <li class='footer-list__item'> <a aria-label='Community, Citus Slack' class='footer-list__link' href='https://slack.citusdata.com'>Citus Slack</a> </li> <li class='footer-list__item'> <a aria-label='Community, Stack Overflow' class='footer-list__link' href='https://stackoverflow.com/questions/tagged/citus' rel='noopener' target='_blank'>Stack Overflow</a> </li> <li class='footer-list__item'> <a aria-label='Community, Newsletters' class='footer-list__link' href='/join-newsletter'>Newsletters</a> </li> <li class='footer-list__item'> <a aria-label='Community, POSETTE' class='footer-list__link' href='https://posetteconf.com'>POSETTE</a> </li> <li class='footer-list__item'> <a aria-label='Community, Talking Postgres' class='footer-list__link' href='https://talkingpostgres.com/' rel='noopener' target='_blank'>Talking Postgres</a> </li> </ul> </div> </div> </footer> <script src='../../../../../assets/javascripts/notification.min-39f6128a.js' type='text/javascript'></script> <script>
var cookieName="CitusPromoCookieps26_wrap",hideTop=document.querySelector("#hideTop"),promoLink=document.querySelector(".promo-text a"),closePromo=function(){slideUp(document.getElementById("topbar")),setCookie(cookieName,"Notified",30),dataLayer.push({event:"eventTracking",category:"notification bar",action:"close",label:"ps26_wrap (/blog/2017/03/23/a-look-into-disaster-recovery-and-high-availability-and-how-they-work-with-postgres-on-citus-cloud/)"}),dataLayer.push({event:"notification_bar",action:"close notification",label:"ps26_wrap"})},clickPromo=function(){dataLayer.push({event:"eventTracking",category:"notification bar",action:"click "+event.target.href,label:"ps26_wrap (/blog/2017/03/23/a-look-into-disaster-recovery-and-high-availability-and-how-they-work-with-postgres-on-citus-cloud/)"}),dataLayer.push({event:"notification_bar",action:"click promo link "+event.target.href,label:"ps26_wrap"})};hideTop.addEventListener("click",closePromo),promoLink.addEventListener("click",clickPromo),null==getCookie(cookieName)&&show(document.getElementById("topbar"))
</script> <script>
window.dataLayer = window.dataLayer || [];
</script> <script src='https://wcpstatic.microsoft.com/mscc/lib/v2/wcp-consent.js'></script> <script>
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
</script> <script src="../../../../../assets/javascripts/index-46b22140.js"></script> <script src='../../../../../assets/javascripts/navigation.min-1c4a0365.js'></script> <script>
var useragent=window.navigator.userAgent,tridentBrowser=useragent.indexOf("Trident/");tridentBrowser>0&&loadCSS("../../../../../assets/stylesheets/ie-65b42519.css");
</script> <script src="../../../../../assets/javascripts/svg4everybody.min-928af460.js"></script> <script>
svg4everybody();
</script> <script src='//info.citusdata.com/js/forms2/js/forms2.min.js'></script> <script src="../../../../../assets/javascripts/share.min-0fc4317a.js" async="true"></script> <script src='../../../../../assets/javascripts/sticky.min-5e750bab.js'></script> <script src="../../../../../assets/javascripts/share.min-0fc4317a.js" async="true"></script> <script src="https://cdn.jsdelivr.net/npm/clipboard@2.0.6/dist/clipboard.min.js"></script> <script>
var clipboard=new ClipboardJS(".blog-link"),sharelink=$(".blog-link");function fallbackMessage(e){var t="",r="cut"===e?"X":"C";return/iPhone|iPad/i.test(navigator.userAgent)?"No support :(":/Mac/i.test(navigator.userAgent)?"Press ⌘-"+r+" to "+e:"Press Ctrl-"+r+" to "+e}clipboard.on("success",function(e){$(e.trigger).next('[role="tooltip"]').text("Link copied!"),$(e.trigger).addClass("linkcopied"),e.clearSelection()}),clipboard.on("error",function(e){console.error("Action:",e.action),console.error("Trigger:",e.trigger),showTooltip(e.trigger,fallbackMessage(e.action))}),sharelink.on("mouseleave focusout",function(){if($(this).hasClass("linkcopied")&&$(this).next('[role="tooltip"]').is(":hidden")){var e=$(this).attr("data-default-text");$(this).removeClass("linkcopied"),$(this).next('[role="tooltip"]').text(e)}});var sticky=new Sticky("[data-sticky]"),tableWrap=document.querySelectorAll(".overflow-box");function makeScrollableBlocksFocusable(){for(var e=0;e<tableWrap.length;e++)isOverflown(tableWrap[e])?tableWrap[e].setAttribute("tabindex","0"):tableWrap[e].removeAttribute("tabindex")}function isOverflown(e){return e.scrollHeight>e.clientHeight||e.scrollWidth>e.clientWidth}window.addEventListener("load",e=>{tableWrap.length>0&&(makeScrollableBlocksFocusable(),window.addEventListener("resize",makeScrollableBlocksFocusable))});
var clipboardAnchors=new ClipboardJS(".share-link, .copy-button");clipboardAnchors.on("success",function(t){t.clearSelection(),console.info("Action:",t.action),console.info("Text:",t.text),console.info("Trigger:",t.trigger),t.trigger.classList.contains("copy-button")?t.trigger.textContent="Copied!":showTooltip(t.trigger,"Copied!")}),clipboardAnchors.on("error",function(t){console.error("Action:",t.action),console.error("Trigger:",t.trigger),showTooltip(t.trigger,fallbackMessage(t.action))});for(var btns=document.querySelectorAll(".share-link, .copy-button"),i=0;i<btns.length;i++)btns[i].addEventListener("mouseleave",clearTooltip),btns[i].addEventListener("blur",clearTooltip);function clearTooltip(t){t.currentTarget.classList.contains("copy-button")?t.currentTarget.textContent="Copy":(t.currentTarget.setAttribute("class","icon share-link"),t.currentTarget.setAttribute("aria-label","Copy link"))}function showTooltip(t,e){t.classList.contains("copy-button")||(t.setAttribute("class","icon share-link tooltipped tooltipped-s"),t.setAttribute("aria-label",e))}function fallbackMessage(t){var e="",r="cut"===t?"X":"C";return/iPhone|iPad/i.test(navigator.userAgent)?"No support :(":/Mac/i.test(navigator.userAgent)?"Press ⌘-"+r+" to "+t:"Press Ctrl-"+r+" to "+t}
</script> <div class='new' id='consentBox'> <div class='consent-text'> <p>This site uses cookies for analytics, personalized content and ads. By continuing to browse this site, you agree to this use. <a href="/privacy#cookies" id="cookiesLink" class="text-white">Learn&nbsp;more</a>.</p> </div> <div class='consent-button'> <button aria-label='Close cookie consent popup' class='btn' id='closeConsent'>CLOSE</button> </div> </div> <script>
$(document).ready(function(){if($("#cookie-script").length>0);else{var e=document.createElement("script");e.setAttribute("id","cookie-script"),e.src="../../../../../assets/javascripts/cookieConsent_msftUpdate.min-ba4d4084.js",document.head.appendChild(e)}});
</script> </html>