322 lines
65 KiB
HTML
322 lines
65 KiB
HTML
<!DOCTYPE html><html lang="en" data-astro-cid-ouamjn2i> <head><meta charset="utf-8"><meta name="viewport" content="width=device-width"><title>Budibase Cloud January 9th Incident</title><meta name="generator" content="Astro v6.0.4"><meta name="color-scheme" content="light dark"><meta name="theme-color" content="#faf9f5" media="(prefers-color-scheme: light)"><meta name="theme-color" content="#141413" media="(prefers-color-scheme: dark)"><meta name="apple-mobile-web-app-capable" content="yes"><meta name="apple-mobile-web-app-status-bar-style" content="default"><meta name="mobile-web-app-capable" content="yes"><link rel="manifest" href="/manifest.webmanifest"><script src="https://cdn.kukie.io/s/35056df5-a65d-4cb3-be6d-e29000e4b2da/c.js" async></script><script>
|
||
// Prevent flash of unstyled content by applying theme immediately
|
||
// Solution based on: https://medium.com/@maiowebdesign/astro-dark-mode-flickering-62da94d08ce6
|
||
(function() {
|
||
function getCodeTheme(theme) {
|
||
return theme === 'dark' ? 'github-dark' : 'github-light';
|
||
}
|
||
|
||
function getEffectiveTheme() {
|
||
try {
|
||
const stored = localStorage.getItem('theme');
|
||
if (stored === 'dark' || stored === 'light') {
|
||
return stored;
|
||
}
|
||
} catch (e) {
|
||
// localStorage might not be available
|
||
}
|
||
return window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light';
|
||
}
|
||
|
||
function applyTheme() {
|
||
const effectiveTheme = getEffectiveTheme();
|
||
document.documentElement.classList.toggle('dark', effectiveTheme === 'dark');
|
||
document.documentElement.dataset.theme = getCodeTheme(effectiveTheme);
|
||
}
|
||
|
||
// Apply theme immediately before any rendering
|
||
applyTheme();
|
||
|
||
// Only react to explicit theme changes (avoids flicker on navigation)
|
||
window.addEventListener('storage', (e) => {
|
||
if (e.key === 'theme') {
|
||
applyTheme();
|
||
}
|
||
});
|
||
})();
|
||
</script><script>
|
||
if ("serviceWorker" in navigator) {
|
||
window.addEventListener(
|
||
"load",
|
||
() => {
|
||
navigator.serviceWorker.register("/sw.js").catch(() => {});
|
||
},
|
||
{ once: true },
|
||
);
|
||
}
|
||
</script><script>
|
||
(() => {
|
||
const GTM_ID = "GTM-TW89LL3";
|
||
const PROD_HOSTS = new Set(["budibase.com", "www.budibase.com"]);
|
||
|
||
function shouldLoadGtm() {
|
||
return PROD_HOSTS.has(window.location.hostname);
|
||
}
|
||
|
||
function injectGtm() {
|
||
if (!shouldLoadGtm() || document.getElementById("gtm-loader")) return;
|
||
|
||
window.dataLayer = window.dataLayer || [];
|
||
window.dataLayer.push({
|
||
"gtm.start": Date.now(),
|
||
event: "gtm.js",
|
||
});
|
||
|
||
const script = document.createElement("script");
|
||
script.id = "gtm-loader";
|
||
script.async = true;
|
||
script.src = `https://www.googletagmanager.com/gtm.js?id=${GTM_ID}`;
|
||
document.head.appendChild(script);
|
||
}
|
||
|
||
window.addEventListener(
|
||
"load",
|
||
() => {
|
||
if ("requestIdleCallback" in window) {
|
||
window.requestIdleCallback(injectGtm, { timeout: 3000 });
|
||
} else {
|
||
window.setTimeout(injectGtm, 1500);
|
||
}
|
||
},
|
||
{ once: true },
|
||
);
|
||
})();
|
||
</script><style>
|
||
/* Intentionally left blank */
|
||
</style><style>@font-face{font-family:Inter-960edf7b3ac5a933;src:url("/_astro/fonts/888e30f2e4b4ce0e.woff2?dpl=6a843466bcd1c600081f29af") format("woff2");font-display:swap;font-weight:400;font-style:normal;}@font-face{font-family:Inter-960edf7b3ac5a933;src:url("/_astro/fonts/fae41ae4d7460211.woff2?dpl=6a843466bcd1c600081f29af") format("woff2");font-display:swap;font-weight:500;font-style:normal;}@font-face{font-family:Inter-960edf7b3ac5a933;src:url("/_astro/fonts/a906f59fb011c175.woff2?dpl=6a843466bcd1c600081f29af") format("woff2");font-display:swap;font-weight:600;font-style:normal;}@font-face{font-family:"Inter-960edf7b3ac5a933 fallback: Arial";src:local("Arial");font-display:swap;font-weight:400;font-style:normal;size-adjust:107.1194%;ascent-override:90.4365%;descent-override:22.518%;line-gap-override:0%;}@font-face{font-family:"Inter-960edf7b3ac5a933 fallback: Arial";src:local("Arial");font-display:swap;font-weight:500;font-style:normal;size-adjust:107.1194%;ascent-override:90.4365%;descent-override:22.518%;line-gap-override:0%;}@font-face{font-family:"Inter-960edf7b3ac5a933 fallback: Arial";src:local("Arial");font-display:swap;font-weight:600;font-style:normal;size-adjust:107.1194%;ascent-override:90.4365%;descent-override:22.518%;line-gap-override:0%;}:root{--font-inter:Inter-960edf7b3ac5a933,"Inter-960edf7b3ac5a933 fallback: Arial",sans-serif;}</style><link rel="preload" href="/_astro/fonts/888e30f2e4b4ce0e.woff2?dpl=6a843466bcd1c600081f29af" as="font" type="font/woff2" crossorigin><style>@font-face{font-family:Fraunces-aa3dbfb0314c74e4;src:url("/_astro/fonts/2fe06b9b5210ad5a.woff2?dpl=6a843466bcd1c600081f29af") format("woff2");font-display:swap;font-weight:400 600;font-style:normal;}@font-face{font-family:"Fraunces-aa3dbfb0314c74e4 fallback: Arial";src:local("Arial");font-display:swap;font-weight:400 600;font-style:normal;size-adjust:116.1954%;ascent-override:84.1686%;descent-override:21.9458%;line-gap-override:0%;}:root{--font-fraunces:Fraunces-aa3dbfb0314c74e4,"Fraunces-aa3dbfb0314c74e4 fallback: Arial",sans-serif;}</style><style>@font-face{font-family:"iA Writer Mono-697537adbc77779a";src:url("/_astro/fonts/1bc93e8883967290.woff2?dpl=6a843466bcd1c600081f29af") format("woff2");font-display:swap;font-weight:400;font-style:normal;}@font-face{font-family:"iA Writer Mono-697537adbc77779a fallback: Arial";src:local("Arial");font-display:swap;font-weight:400;font-style:normal;size-adjust:120.906%;ascent-override:84.7766%;descent-override:22.7449%;line-gap-override:0%;}:root{--font-ia-writer-mono:"iA Writer Mono-697537adbc77779a","iA Writer Mono-697537adbc77779a fallback: Arial",sans-serif;}</style><!-- Low Priority Global Metadata --><link rel="icon" href="/favicon.ico"><link rel="shortcut icon" href="/favicon.ico"><link rel="icon" type="image/svg+xml" sizes="any" href="/favicon.svg"><link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png"><link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png"><link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png"><link rel="mask-icon" href="/favicon.svg" color="#8D46E7"><link rel="sitemap" href="/sitemap.xml"><link rel="alternate" type="application/rss+xml" href="/rss.xml" title="RSS"><!-- SEO construct --><link rel="canonical" href="https://budibase.com/blog/updates/2024/2024-01-09-incident/"><meta name="description" content="What caused the Budibase Cloud incident on January 9th, and what we're doing to prevent it happening again."><meta property="og:title" content="Budibase Cloud January 9th Incident"><meta property="og:type" content="article"><meta property="og:url" content="https://budibase.com/blog/updates/2024/2024-01-09-incident/"><meta property="og:locale" content="en"><meta property="og:description" content="What caused the Budibase Cloud incident on January 9th, and what we're doing to prevent it happening again."><meta property="og:site_name" content="Budibase"><meta property="og:image" content="https://res.cloudinary.com/daog6scxm/image/upload/v1705398760/cms/incident-report/Incident_bevbsa.png"><meta property="og:image:alt" content="Budibase Cloud January 9th Incident"><meta name="twitter:card" content="summary_large_image"><meta name="twitter:site" content="budibase"><meta name="twitter:title" content="Budibase Cloud January 9th Incident"><meta name="twitter:description" content="What caused the Budibase Cloud incident on January 9th, and what we're doing to prevent it happening again."><meta name="twitter:image" content="https://res.cloudinary.com/daog6scxm/image/upload/v1705398760/cms/incident-report/Incident_bevbsa.png"><meta name="twitter:image:alt" content="Budibase Cloud January 9th Incident"><meta name="robots" content="max-image-preview:large"><script type="application/ld+json">{"@context":"https://schema.org","@type":"BlogPosting","headline":"Budibase Cloud January 9th Incident","url":"https://budibase.com/blog/updates/2024/2024-01-09-incident/","license":"This work is licensed and owned by Budibase","description":"What caused the Budibase Cloud incident on January 9th, and what we're doing to prevent it happening again.","mainEntityOfPage":{"@type":"WebPage","@id":"https://budibase.com/blog/updates/2024/2024-01-09-incident/"},"publisher":{"@type":"Organization","name":"Budibase","url":"https://budibase.com/","sameAs":["https://www.wikidata.org/wiki/Q131328403","https://x.com/budibase","https://www.linkedin.com/company/budibase/","https://github.com/budibase/budibase"]},"author":{"@type":"Person","name":"Sam Rose"},"datePublished":"2024-01-16T00:00:00.000Z","dateModified":"2024-01-16T00:00:00.000Z"}</script><link rel="stylesheet" href="/_astro/MainLayout.H0G7Kj7w.css"><script type="module" src="/_astro/page.D6yUpseN.js"></script><style>[data-astro-transition-scope="astro-t5k7mbsc-1"] { view-transition-name: updates-2024-2024-01-09-incident-title; }@layer astro { ::view-transition-old(updates-2024-2024-01-09-incident-title) {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeOut; }::view-transition-new(updates-2024-2024-01-09-incident-title) {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeIn; }[data-astro-transition=back]::view-transition-old(updates-2024-2024-01-09-incident-title) {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeOut; }[data-astro-transition=back]::view-transition-new(updates-2024-2024-01-09-incident-title) {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeIn; } }[data-astro-transition-fallback="old"] [data-astro-transition-scope="astro-t5k7mbsc-1"],
|
||
[data-astro-transition-fallback="old"][data-astro-transition-scope="astro-t5k7mbsc-1"] {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeOut; }[data-astro-transition-fallback="new"] [data-astro-transition-scope="astro-t5k7mbsc-1"],
|
||
[data-astro-transition-fallback="new"][data-astro-transition-scope="astro-t5k7mbsc-1"] {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeIn; }[data-astro-transition=back][data-astro-transition-fallback="old"] [data-astro-transition-scope="astro-t5k7mbsc-1"],
|
||
[data-astro-transition=back][data-astro-transition-fallback="old"][data-astro-transition-scope="astro-t5k7mbsc-1"] {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeOut; }[data-astro-transition=back][data-astro-transition-fallback="new"] [data-astro-transition-scope="astro-t5k7mbsc-1"],
|
||
[data-astro-transition=back][data-astro-transition-fallback="new"][data-astro-transition-scope="astro-t5k7mbsc-1"] {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeIn; }</style><style>[data-astro-transition-scope="astro-q3khvcbl-2"] { view-transition-name: updates-2024-2024-01-09-incident-meta; }@layer astro { ::view-transition-old(updates-2024-2024-01-09-incident-meta) {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeOut; }::view-transition-new(updates-2024-2024-01-09-incident-meta) {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeIn; }[data-astro-transition=back]::view-transition-old(updates-2024-2024-01-09-incident-meta) {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeOut; }[data-astro-transition=back]::view-transition-new(updates-2024-2024-01-09-incident-meta) {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeIn; } }[data-astro-transition-fallback="old"] [data-astro-transition-scope="astro-q3khvcbl-2"],
|
||
[data-astro-transition-fallback="old"][data-astro-transition-scope="astro-q3khvcbl-2"] {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeOut; }[data-astro-transition-fallback="new"] [data-astro-transition-scope="astro-q3khvcbl-2"],
|
||
[data-astro-transition-fallback="new"][data-astro-transition-scope="astro-q3khvcbl-2"] {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeIn; }[data-astro-transition=back][data-astro-transition-fallback="old"] [data-astro-transition-scope="astro-q3khvcbl-2"],
|
||
[data-astro-transition=back][data-astro-transition-fallback="old"][data-astro-transition-scope="astro-q3khvcbl-2"] {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeOut; }[data-astro-transition=back][data-astro-transition-fallback="new"] [data-astro-transition-scope="astro-q3khvcbl-2"],
|
||
[data-astro-transition=back][data-astro-transition-fallback="new"][data-astro-transition-scope="astro-q3khvcbl-2"] {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeIn; }</style><style>[data-astro-transition-scope="astro-zscz5gm2-3"] { view-transition-name: site-chrome; }@layer astro { ::view-transition-old(site-chrome) {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeOut; }::view-transition-new(site-chrome) {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeIn; }[data-astro-transition=back]::view-transition-old(site-chrome) {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeOut; }[data-astro-transition=back]::view-transition-new(site-chrome) {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeIn; } }[data-astro-transition-fallback="old"] [data-astro-transition-scope="astro-zscz5gm2-3"],
|
||
[data-astro-transition-fallback="old"][data-astro-transition-scope="astro-zscz5gm2-3"] {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeOut; }[data-astro-transition-fallback="new"] [data-astro-transition-scope="astro-zscz5gm2-3"],
|
||
[data-astro-transition-fallback="new"][data-astro-transition-scope="astro-zscz5gm2-3"] {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeIn; }[data-astro-transition=back][data-astro-transition-fallback="old"] [data-astro-transition-scope="astro-zscz5gm2-3"],
|
||
[data-astro-transition=back][data-astro-transition-fallback="old"][data-astro-transition-scope="astro-zscz5gm2-3"] {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeOut; }[data-astro-transition=back][data-astro-transition-fallback="new"] [data-astro-transition-scope="astro-zscz5gm2-3"],
|
||
[data-astro-transition=back][data-astro-transition-fallback="new"][data-astro-transition-scope="astro-zscz5gm2-3"] {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeIn; }</style><style>[data-astro-transition-scope="astro-tvayyljk-4"] { view-transition-name: site-footer; }@layer astro { ::view-transition-old(site-footer) {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeOut; }::view-transition-new(site-footer) {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeIn; }[data-astro-transition=back]::view-transition-old(site-footer) {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeOut; }[data-astro-transition=back]::view-transition-new(site-footer) {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeIn; } }[data-astro-transition-fallback="old"] [data-astro-transition-scope="astro-tvayyljk-4"],
|
||
[data-astro-transition-fallback="old"][data-astro-transition-scope="astro-tvayyljk-4"] {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeOut; }[data-astro-transition-fallback="new"] [data-astro-transition-scope="astro-tvayyljk-4"],
|
||
[data-astro-transition-fallback="new"][data-astro-transition-scope="astro-tvayyljk-4"] {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeIn; }[data-astro-transition=back][data-astro-transition-fallback="old"] [data-astro-transition-scope="astro-tvayyljk-4"],
|
||
[data-astro-transition=back][data-astro-transition-fallback="old"][data-astro-transition-scope="astro-tvayyljk-4"] {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeOut; }[data-astro-transition=back][data-astro-transition-fallback="new"] [data-astro-transition-scope="astro-tvayyljk-4"],
|
||
[data-astro-transition=back][data-astro-transition-fallback="new"][data-astro-transition-scope="astro-tvayyljk-4"] {
|
||
animation-duration: 180ms;
|
||
animation-timing-function: cubic-bezier(0.76, 0, 0.24, 1);
|
||
animation-fill-mode: both;
|
||
animation-name: astroFadeIn; }</style></head> <body data-astro-cid-ouamjn2i> <div data-astro-cid-ouamjn2i> <main data-astro-cid-ouamjn2i> <div id="hero" class="opacity: 1; transform: none;" data-astro-cid-ouamjn2i> <div class="text-base" data-astro-cid-ouamjn2i> <div class="relative flex flex-col min-h-screen" data-astro-cid-ouamjn2i> <nav id="site-nav" class="nav-shell sticky top-0 z-50 w-full bg-1 transition-colors duration-150" data-astro-transition-persist="site-chrome" data-astro-transition-scope="astro-zscz5gm2-3"> <div class="relative z-10 mx-auto grid h-[var(--site-nav-height)] w-full max-w-7xl grid-cols-[1fr_auto_auto] items-center px-2 md:px-3 lg:grid-cols-[auto_1fr_auto] lg:gap-6 lg:px-0"> <a href="/" aria-label="Budibase home" class="z-10 shrink-0 flex items-center" data-astro-prefetch> <div class="relative w-[120px] flex items-center"> <img src="/_astro/BudibaseLogoBlack.BrzZ_t3P.svg?dpl=6a843466bcd1c600081f29af" alt="Budibase logo" loading="lazy" decoding="async" width="539" height="97" class="block hover:cursor-pointer dark:hidden"> <img src="/_astro/BudibaseLogoWhite.DEpQW1Im.svg?dpl=6a843466bcd1c600081f29af" alt="Budibase logo" loading="lazy" decoding="async" width="200" height="36" class="block hover:cursor-pointer hidden dark:block"> </div> </a> <div class="z-10 flex items-center gap-2 justify-self-end lg:col-start-3 lg:gap-3"> <div class="hidden items-center gap-5 whitespace-nowrap lg:flex xl:gap-7"> <header-drop-down> <details class="group relative"> <summary class="accordion cursor-pointer link flex items-center gap-2 text-sm px-1 text-primary hover:text-primary group-open:text-primary transition-colors"> Product </summary> <ul class="grid grid-cols-[repeat(2,auto)] gap-y-[6px] gap-x-[6px] absolute left-0 top-full translate-y-2 w-fit max-w-[calc(100vw-2rem)] bg-secondary pt-2 px-[15px] pb-3 text-base shadow-xl ring-1 ring-stone-200/60 dark:ring-stone-800/60 invisible group-open:visible group-hover:visible origin-top scale-95 group-open:scale-100 group-hover:scale-100 opacity-0 group-open:opacity-100 group-hover:opacity-100 transition-all duration-300"> <li class="min-w-[7.5rem]"> <a href="/product/agents/" data-astro-prefetch class="nav__sub-nav__link inline-flex items-center gap-2 py-[2px] text-sm font-medium transition-colors text-primary hover:text-primary"> <span>Agents</span> <span class="text-xs leading-none font-mono text-primary"> Beta </span> </a> </li><li class="min-w-[7.5rem]"> <a href="/product/apps/" data-astro-prefetch class="nav__sub-nav__link inline-flex items-center gap-2 py-[2px] text-sm font-medium transition-colors text-primary hover:text-primary"> <span>Apps</span> </a> </li><li class="min-w-[7.5rem]"> <a href="/product/automations/" data-astro-prefetch class="nav__sub-nav__link inline-flex items-center gap-2 py-[2px] text-sm font-medium transition-colors text-primary hover:text-primary"> <span>Automations</span> </a> </li><li class="min-w-[7.5rem]"> <a href="/product/apis/" data-astro-prefetch class="nav__sub-nav__link inline-flex items-center gap-2 py-[2px] text-sm font-medium transition-colors text-primary hover:text-primary"> <span>API explorer</span> </a> </li><li class="min-w-[7.5rem]"> <a href="/product/connections/" data-astro-prefetch class="nav__sub-nav__link inline-flex items-center gap-2 py-[2px] text-sm font-medium transition-colors text-primary hover:text-primary"> <span>Connectors</span> </a> </li><li class="min-w-[7.5rem]"> <a href="/product/data/" data-astro-prefetch class="nav__sub-nav__link inline-flex items-center gap-2 py-[2px] text-sm font-medium transition-colors text-primary hover:text-primary"> <span>Data tables</span> </a> </li> </ul> </details> </header-drop-down> <script type="module">customElements.get("header-drop-down")||customElements.define("header-drop-down",class extends HTMLElement{connectedCallback(){const e=this.querySelector("details");if(!e)return;let t;this.addEventListener("mouseenter",()=>{clearTimeout(t),e.open=!0}),this.addEventListener("mouseleave",()=>{t=setTimeout(()=>{e.matches(":focus-within")||(e.open=!1)},200)}),e.addEventListener("focusout",()=>{e.matches(":focus-within")||setTimeout(()=>{e.open=!1},200)})}});</script> <header-drop-down> <details class="group relative"> <summary class="accordion cursor-pointer link flex items-center gap-2 text-sm px-1 text-primary hover:text-primary group-open:text-primary transition-colors"> Solutions </summary> <ul class="grid grid-cols-[repeat(2,auto)] gap-y-[6px] gap-x-[6px] absolute left-0 top-full translate-y-2 w-fit max-w-[calc(100vw-2rem)] bg-secondary pt-2 px-[15px] pb-3 text-base shadow-xl ring-1 ring-stone-200/60 dark:ring-stone-800/60 invisible group-open:visible group-hover:visible origin-top scale-95 group-open:scale-100 group-hover:scale-100 opacity-0 group-open:opacity-100 group-hover:opacity-100 transition-all duration-300"> <li class="min-w-[7.5rem]"> <a href="/ops/" data-astro-prefetch class="nav__sub-nav__link inline-flex items-center gap-2 py-[2px] text-sm font-medium transition-colors text-primary hover:text-primary"> <span>Use Cases</span> </a> </li><li class="min-w-[7.5rem]"> <a href="/enterprise/" data-astro-prefetch class="nav__sub-nav__link inline-flex items-center gap-2 py-[2px] text-sm font-medium transition-colors text-primary hover:text-primary"> <span>Enterprise</span> </a> </li><li class="min-w-[7.5rem]"> <a href="/security/" data-astro-prefetch class="nav__sub-nav__link inline-flex items-center gap-2 py-[2px] text-sm font-medium transition-colors text-primary hover:text-primary"> <span>Security</span> </a> </li><li class="min-w-[7.5rem]"> <a href="/it/" data-astro-prefetch class="nav__sub-nav__link inline-flex items-center gap-2 py-[2px] text-sm font-medium transition-colors text-primary hover:text-primary"> <span>IT Teams</span> </a> </li> </ul> </details> </header-drop-down> <header-drop-down> <details class="group relative"> <summary class="accordion cursor-pointer link flex items-center gap-2 text-sm px-1 text-primary hover:text-primary group-open:text-primary transition-colors"> Resources </summary> <ul class="grid grid-cols-[repeat(2,auto)] gap-y-[6px] gap-x-[6px] absolute left-0 top-full translate-y-2 w-fit max-w-[calc(100vw-2rem)] bg-secondary pt-2 px-[15px] pb-3 text-base shadow-xl ring-1 ring-stone-200/60 dark:ring-stone-800/60 invisible group-open:visible group-hover:visible origin-top scale-95 group-open:scale-100 group-hover:scale-100 opacity-0 group-open:opacity-100 group-hover:opacity-100 transition-all duration-300"> <li class="min-w-[7.5rem]"> <a href="https://docs.budibase.com" target="_blank" rel="noopener noreferrer" class="nav__sub-nav__link inline-flex items-center gap-2 py-[2px] text-sm font-medium transition-colors text-primary hover:text-primary"> <span>Docs</span> </a> </li><li class="min-w-[7.5rem]"> <a href="/customers/" data-astro-prefetch class="nav__sub-nav__link inline-flex items-center gap-2 py-[2px] text-sm font-medium transition-colors text-primary hover:text-primary"> <span>Customers</span> </a> </li><li class="min-w-[7.5rem]"> <a href="/blog/" data-astro-prefetch class="nav__sub-nav__link inline-flex items-center gap-2 py-[2px] text-sm font-medium transition-colors text-primary hover:text-primary"> <span>Blog</span> </a> </li><li class="min-w-[7.5rem]"> <a href="/changelog/" data-astro-prefetch class="nav__sub-nav__link inline-flex items-center gap-2 py-[2px] text-sm font-medium transition-colors text-primary hover:text-primary"> <span>Changelog</span> </a> </li> </ul> </details> </header-drop-down> <a target="_self" href="/pricing/" class="text-sm nav-link transition-colors" data-astro-prefetch>
|
||
Pricing
|
||
</a> </div> <a href="https://github.com/budibase/budibase" target="_blank" class="relative justify-center items-center px-1.5 text-center ease-out duration-150 rounded-sm transition-all hover:bg-stone-200 dark:hover:bg-stone-800 border-transparent text-xs hidden xl:flex focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-stone-500 focus-visible:ring-offset-2 focus-visible:ring-offset-white dark:focus-visible:ring-offset-stone-900"><span class="truncate"><span class="flex text-sm items-center gap-0.5 nav-link transition-colors"><svg class="w-5 h-5" viewBox="0 0 17 16" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M8.5 2.22168C5.23312 2.22168 2.58496 4.87398 2.58496 8.14677C2.58496 10.7642 4.27962 12.9853 6.63026 13.7684C6.92601 13.8228 7.03366 13.6401 7.03366 13.4827C7.03366 13.3425 7.02893 12.9693 7.02597 12.4754C5.38041 12.8333 5.0332 11.681 5.0332 11.681C4.76465 10.996 4.37663 10.8139 4.37663 10.8139C3.83954 10.4471 4.41744 10.4542 4.41744 10.4542C5.01072 10.4956 5.32303 11.0647 5.32303 11.0647C5.85065 11.9697 6.70774 11.7082 7.04431 11.5568C7.09873 11.1741 7.25134 10.9132 7.42051 10.7654C6.10737 10.6157 4.72621 10.107 4.72621 7.83683C4.72621 7.19031 4.95689 6.66092 5.33486 6.24686C5.27394 6.09721 5.07105 5.49447 5.39283 4.67938C5.39283 4.67938 5.88969 4.51967 7.01947 5.28626C7.502 5.15466 7.99985 5.08763 8.5 5.08692C9.00278 5.08929 9.50851 5.15495 9.98113 5.28626C11.1103 4.51967 11.606 4.67879 11.606 4.67879C11.9289 5.49447 11.7255 6.09721 11.6651 6.24686C12.0437 6.66092 12.2732 7.19031 12.2732 7.83683C12.2732 10.1129 10.8897 10.6139 9.5724 10.7606C9.78475 10.9434 9.97344 11.3048 9.97344 11.8579C9.97344 12.6493 9.96634 13.2887 9.96634 13.4827C9.96634 13.6413 10.0728 13.8258 10.3733 13.7678C11.5512 13.3728 12.5751 12.6175 13.3003 11.6089C14.0256 10.6002 14.4155 9.38912 14.415 8.14677C14.415 4.87398 11.7663 2.22168 8.5 2.22168Z" fill="currentColor"></path> </svg>27.5K</span></span></a> <div class="hidden lg:flex items-center gap-2 lg:gap-3"> <a href="https://account.budibase.app/auth/login" class="inline-flex items-center justify-center rounded-full !font-medium transition-colors duration-150 active:scale-[0.98] no-underline cursor-pointer font-sans focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-bg bg-secondary text hover:bg-surface-2 px-2.5 py-[4.5px] text-sm gap-1" role="button" target="_blank" rel="noopener noreferrer">Log in</a> <a href="/contact/" class="inline-flex items-center justify-center rounded-full !font-medium transition-colors duration-150 active:scale-[0.98] no-underline cursor-pointer font-sans focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-bg bg-secondary text hover:bg-surface-2 px-2.5 py-[4.5px] text-sm gap-1" role="button">Contact sales</a> <a href="https://account.budibase.app/register" class="inline-flex items-center justify-center rounded-full !font-medium transition-colors duration-150 active:scale-[0.98] no-underline cursor-pointer font-sans focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-bg bg-grey-900 text-grey-50 hover:bg-grey-800 dark:bg-grey-100 dark:text-grey-900 dark:hover:bg-grey-200 px-2.5 py-[4.5px] text-sm gap-1" role="button" target="_blank" rel="noopener noreferrer">Try it free</a> </div> <button id="mobile-menu-button" class="lg:hidden p-2 text-stone-800 dark:text-stone-200 hover:bg-stone-200 dark:hover:bg-stone-800 rounded transition-colors" aria-label="Toggle mobile menu" aria-expanded="false"> <svg id="menu-icon" class="w-6 h-6" fill="none" stroke="currentColor" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"> <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 6h16M4 12h16M4 18h16"></path> </svg> <svg id="close-icon" class="w-6 h-6 hidden" fill="none" stroke="currentColor" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"> <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12"></path> </svg> </button> </div> </div> </nav> <div id="mobile-menu" class="mobile-nav-overlay hidden lg:hidden" aria-hidden="true"> <div id="main-menu-view" class="mobile-nav-panel"> <div class="mobile-nav-header"> <a href="/" aria-label="Budibase home" class="mobile-nav-logo" data-astro-prefetch> <div class="relative w-[120px] flex items-center"> <img src="/_astro/BudibaseLogoBlack.BrzZ_t3P.svg?dpl=6a843466bcd1c600081f29af" alt="Budibase logo" loading="lazy" decoding="async" width="539" height="97" class="block dark:hidden"> <img src="/_astro/BudibaseLogoWhite.DEpQW1Im.svg?dpl=6a843466bcd1c600081f29af" alt="Budibase logo" loading="lazy" decoding="async" width="200" height="36" class="hidden dark:block"> </div> </a> <button id="close-menu-button" class="mobile-nav-close-button" aria-label="Close menu"> <svg class="w-6 h-6" fill="none" stroke="currentColor" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"> <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M6 18L18 6M6 6l12 12"></path> </svg> </button> </div> <nav class="mobile-nav-scroll"> <ul class="mobile-nav-list"> <li class="mobile-nav-item"> <details class="mobile-nav-accordion" data-mobile-nav-accordion> <summary class="mobile-nav-row mobile-nav-accordion-summary"> <span class="mobile-nav-row-label nav-link transition-colors">Product</span> <svg class="mobile-nav-plus-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" aria-hidden="true"> <path stroke-linecap="round" stroke-linejoin="round" d="M12 5v14M5 12h14"></path> </svg> </summary> <ul class="mobile-nav-sub-list"> <li class="mobile-nav-sub-item"> <a href="/product/agents/" data-astro-prefetch class="mobile-nav-sub-link nav-link transition-colors"> <span>Agents</span> <span class="text-sm leading-none font-mono text-[var(--text-secondary)]"> Beta </span> </a> </li><li class="mobile-nav-sub-item"> <a href="/product/apps/" data-astro-prefetch class="mobile-nav-sub-link nav-link transition-colors"> <span>Apps</span> </a> </li><li class="mobile-nav-sub-item"> <a href="/product/automations/" data-astro-prefetch class="mobile-nav-sub-link nav-link transition-colors"> <span>Automations</span> </a> </li><li class="mobile-nav-sub-item"> <a href="/product/apis/" data-astro-prefetch class="mobile-nav-sub-link nav-link transition-colors"> <span>API explorer</span> </a> </li><li class="mobile-nav-sub-item"> <a href="/product/connections/" data-astro-prefetch class="mobile-nav-sub-link nav-link transition-colors"> <span>Connectors</span> </a> </li><li class="mobile-nav-sub-item"> <a href="/product/data/" data-astro-prefetch class="mobile-nav-sub-link nav-link transition-colors"> <span>Data tables</span> </a> </li> </ul> </details> </li> <li class="mobile-nav-item"> <a href="/ops/" class="mobile-nav-row nav-link transition-colors" data-astro-prefetch>
|
||
Ops
|
||
</a> </li> <li class="mobile-nav-item"> <a href="https://discord.com/invite/budibase-733030666647765003" target="_blank" class="mobile-nav-row nav-link transition-colors">
|
||
Community
|
||
</a> </li> <li class="mobile-nav-item"> <a href="/pricing/" class="mobile-nav-row nav-link transition-colors" data-astro-prefetch>
|
||
Pricing
|
||
</a> </li> <li class="mobile-nav-item"> <details class="mobile-nav-accordion" data-mobile-nav-accordion> <summary class="mobile-nav-row mobile-nav-accordion-summary"> <span class="mobile-nav-row-label nav-link transition-colors">Resources</span> <svg class="mobile-nav-plus-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" aria-hidden="true"> <path stroke-linecap="round" stroke-linejoin="round" d="M12 5v14M5 12h14"></path> </svg> </summary> <ul class="mobile-nav-sub-list"> <li class="mobile-nav-sub-item"> <a href="https://docs.budibase.com" target="_blank" rel="noopener noreferrer" class="mobile-nav-sub-link nav-link transition-colors"> <span>Docs</span> </a> </li><li class="mobile-nav-sub-item"> <a href="/customers/" data-astro-prefetch class="mobile-nav-sub-link nav-link transition-colors"> <span>Customers</span> </a> </li><li class="mobile-nav-sub-item"> <a href="/blog/" data-astro-prefetch class="mobile-nav-sub-link nav-link transition-colors"> <span>Blog</span> </a> </li><li class="mobile-nav-sub-item"> <a href="/changelog/" data-astro-prefetch class="mobile-nav-sub-link nav-link transition-colors"> <span>Changelog</span> </a> </li> </ul> </details> </li> </ul> </nav> <div class="mobile-nav-actions"> <a href="https://account.budibase.app/auth/login" class="inline-flex items-center justify-center rounded-full !font-medium transition-colors duration-150 active:scale-[0.98] no-underline cursor-pointer font-sans focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-bg bg-secondary text hover:bg-surface-2 px-4 py-2 text-lg gap-2 mobile-nav-action-button" role="button" target="_blank" rel="noopener noreferrer">Log in</a> <a href="https://account.budibase.app/register" class="inline-flex items-center justify-center rounded-full !font-medium transition-colors duration-150 active:scale-[0.98] no-underline cursor-pointer font-sans focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-bg bg-grey-900 text-grey-50 hover:bg-grey-800 dark:bg-grey-100 dark:text-grey-900 dark:hover:bg-grey-200 px-4 py-2 text-lg gap-2 mobile-nav-action-button" role="button" target="_blank" rel="noopener noreferrer">Sign up</a> </div> </div> </div> <script type="module">(function(){if(window.__siteMobileNavBound)return;window.__siteMobileNavBound=!0;const i=document.getElementById("mobile-menu-button"),e=document.getElementById("mobile-menu"),l=document.getElementById("close-menu-button"),s=document.getElementById("menu-icon"),c=document.getElementById("close-icon"),d=window.matchMedia("(min-width: 1024px)");let a="";function o(){return!e?.classList.contains("hidden")}function u(){!e||o()||(a=document.body.style.overflow,e?.classList.remove("hidden"),i?.setAttribute("aria-expanded","true"),e?.setAttribute("aria-hidden","false"),s?.classList.add("hidden"),c?.classList.remove("hidden"),document.body.style.overflow="hidden")}function n(){!e||!o()||(e?.classList.add("hidden"),i?.setAttribute("aria-expanded","false"),e?.setAttribute("aria-hidden","true"),s?.classList.remove("hidden"),c?.classList.add("hidden"),document.body.style.overflow=a,e?.querySelectorAll("[data-mobile-nav-accordion][open]").forEach(t=>t.removeAttribute("open")))}i?.addEventListener("click",()=>{if(o()){n();return}u()}),l?.addEventListener("click",n),e?.addEventListener("click",t=>{t.target===e&&n()}),e?.querySelectorAll("a").forEach(t=>{t.addEventListener("click",n)}),document.addEventListener("keydown",t=>{t.key==="Escape"&&o()&&n()});const r=()=>{d.matches&&n()};d.addEventListener?d.addEventListener("change",r):d.addListener(r)})();</script> <div class="max-w-2xl mx-auto w-full px-5 md:px-0 pt-20"> <a class="opacity-75 font-weight-medium a-alt mt-8" href="/blog/"><- All posts</a> <h1 class="mb-4 mt-4" data-astro-transition-scope="astro-t5k7mbsc-1"> Budibase Cloud January 9th Incident </h1> <div class="post-meta" data-astro-transition-scope="astro-q3khvcbl-2"> <time datetime="2024-01-16T00:00:00.000Z"> Jan 16, 2024 </time> <span> by Sam Rose </span> </div> </div> <div class="max-w-2xl mx-auto w-full px-5 md:px-0"> <article class="article my-8 max-w-2xl mx-auto"> <p>On January 9th, a subset of Budibase Cloud customers were unable to log in. We were able to trace this problem back to data corruption in our production database, and resolved the problem by rolling our database back to a known-good snapshot. The snapshot we rolled back to, at the time we rolled back to it, was 87 minutes old. Any changes made by customers on Budibase Cloud in those 87 minutes were lost.</p>
|
||
<p>We regret that we had to do this, and apologise for the inconvenience it has caused. This post is going to discuss in detail how this came to happen, and what we’re doing to prevent it from happening again.</p>
|
||
<h2 id="what-happened">What happened</h2>
|
||
<p>As part of our ongoing work to make Budibase Cloud a faster and more stable environment, we have been making lots of changes to our technical infrastructure. We've added lots of new observability into our platform, we've introduced continuous profiling and used this to make a number of <a href="https://github.com/Budibase/budibase/pull/12580">performance</a> <a href="https://github.com/Budibase/budibase/pull/12603">improvements</a>, and we've made changes to our health checks to ensure when tasks become unhealthy, they are replaced.</p>
|
||
<p>Another area we've been improving is our production database. Budibase Cloud is backed by a database called CouchDB, which we run across 3 servers (or "nodes"). We do this so that if one of these nodes were to go down or have problems, nobody would notice because we still have 2 other nodes available.</p>
|
||
<p>We noticed at the start of this year that one of our CouchDB nodes had been configured differently to the other 2. All of the nodes in the cluster should be identical, any divergence in their configuration introduces uncertainty. For example, we found that this outlier node does logging differently to the other two. This makes it trickier to track down problems because now we need to remember that one of our 3 nodes logs somewhere different to the rest of them. What we'd prefer is for them all to work in exactly the same way.</p>
|
||
<p>We decided that the easiest way to do this would be to add a new node to the cluster, then turn down the outlier one. We use an "Infrastructure as Code" approach at Budibase, so we have automated processes for adding a new node to our CouchDB cluster.</p>
|
||
<p>So at 13:52 UTC on January 9th, we add the new node to our CouchDB cluster. Straight away, it begins to synchronise data from the other 3 nodes. This is expected, and can be seen in our monitoring.</p>
|
||
<figure class="not-prose my-8 space-y-3">
|
||
<img src="https://res.cloudinary.com/daog6scxm/image/upload/v1705397473/total-databases_y9aucp.png" alt="A graph of total databases per CouchDB node. We see 3 nodes sitting the same value, and at 13:52 a new nodes joins and the number of databases on it starts to increase slowly" class="w-full rounded-sm border" loading="lazy">
|
||
|
||
</figure>
|
||
<p>Our production database cluster is called "chesterfield" (after the <a href="https://en.wiktionary.org/wiki/chesterfield">sofa</a>), and each of the nodes is numbered sequentially. <code>chesterfield-004</code> is the new node in this graph. When it first shows up is what it joined the cluster, then we see the number of databases on it slowly increase as it synchronises, then we see the number of databases plateau when we remove it from the cluster, and then the line disappears when the node is turned down.</p>
|
||
<p>The mistake we made in our automated process for adding nodes was to add the new node to our load balancer before it had fully synchronised. We have a load balancer in front of our CouchDB cluster because any node can service any request, so we round-robin requests between them. You can see that in this graph showing the number of healthy nodes our database load balancer is aware of.</p>
|
||
<figure class="not-prose my-8 space-y-3">
|
||
<img src="https://res.cloudinary.com/daog6scxm/image/upload/v1705397472/healthy-nodes_fcdmqp.png" alt="A graph showing a line that hovers at the value 3 until 13:52, where it jumps to 4, and then at 13:58 goes back down to 3 and stays there" class="w-full rounded-sm border" loading="lazy">
|
||
|
||
</figure>
|
||
<p>CouchDB is what's called a "multi-master" database. Many distributed database systems will have one node that's more important
|
||
than the others (the "master" node) and this is the node responsible for handling new data, while the other nodes can only handle the fetching of data. This is not the case in CouchDB, all nodes are equal and can handle writes. The way this works under the hood is that conflicting writes, e.g. 2 separate writes to the same data on 2 different nodes, are resolved through a conflict-resolution process.</p>
|
||
<p>For historical reasons, most requests that Budibase makes to its internal database first check to make sure that the database they're writing to exists. If it does not, it gets created. For the 6 minutes that <code>chesterfield-004</code> was in the load balancer, most requests to check that a database existed would have reported that no, the database does not exist. A new, empty database would be created and the code would continue. This empty database was then replicated to other nodes, effectively deleting all of their contents. We were able to reproduce this behaviour locally, and you can check our methodology <a href="https://github.com/Budibase/couchdb-outage-repro">here</a>.</p>
|
||
<figure class="not-prose my-8 space-y-3">
|
||
<img src="https://res.cloudinary.com/daog6scxm/image/upload/v1705397472/chesterfield-004-writes_dtxga2.png" alt="A graph showing the number of writes per second to chesterfield-004. It peaks at 1 write per second at 13:54, then trails off until the line disappears at 13:58" class="w-full rounded-sm border" loading="lazy">
|
||
|
||
</figure>
|
||
<p>This manifested for most users as the inability to log in, as their user record had been deleted.</p>
|
||
<figure class="not-prose my-8 space-y-3">
|
||
<img src="https://res.cloudinary.com/daog6scxm/image/upload/v1705397472/login-403s_nutffq.png" alt="A graph showing HTTP requests to Budibase Cloud that had a 403 response, indicating login failure" class="w-full rounded-sm border" loading="lazy">
|
||
|
||
</figure>
|
||
<p>Unfortunately we were not aware that there was a problem until 14:13 UTC, when we started receiving reports from customers that they were not able to log in. We do have automated tests of our login process running every few minutes, but they succeeded throughout because the users we use in the tests were not affected.</p>
|
||
<p>Why, then, had we removed the node from the cluster and turned it down by 13:58?</p>
|
||
<p>When our automation brought up the node, it failed at the point where it was installing CouchDB. We investigated this at the time and found it had used an incorrect CouchDB version string when installing the CouchDB package. We attempted to correct this and re-run the automation, but that didn't work. This prompted us to bring the node down completely, and start again from the beginning.</p>
|
||
<p>This was both lucky and unfortunate. Lucky because we inadvertently fixed what was causing data corruption. The data that had been lost was still gone, but we weren't losing more data. Unfortunate because when this node went, it took valuable debugging information with it.</p>
|
||
<h2 id="how-we-responded">How we responded</h2>
|
||
<p>The first customer reports got to us in the engineering team at 14:13 UTC, and we immediately suspected our work on the database cluster was the cause. The timing lined up perfectly, and it was a substantial change to our infrastructure. By 14:28 UTC we had confirmed data loss in the tenants that had reached out to us and began executing our disaster recovery plan.</p>
|
||
<p>We managed to confirm data loss in 11 customers by looking at our records of failed logins, and given that the rate of failed logins was not increasing we believed that without <code>chesterfield-004</code> in the cluster, the problem was not spreading. However, 11 customers was a lower bound. It was difficult to know what writes had been served by <code>chesterfield-004</code> prior to it being taken down, and the log files on the machine had not been preserved. Because of this, we decided to play it safe and restore all data to the last known-good backup we had, which was from 13:33 UTC.</p>
|
||
<p>We started by taking <code>chesterfield-001</code> offline and restoring its hard disk to the 13:33 UTC snapshot. When that was done, we re-added it to the cluster and to our dismay it immediately synchronised its state with the other two nodes, re-deleting the data we were trying to restore. Our disaster recovery plan covered partial recovery of data for when we know what we're restoring, and recovering from a single node failure. There were no steps for rolling the entire cluster back to a known-good state.</p>
|
||
<p>At around 15:00 UTC we started to discuss how we could restore the full state of the cluster without incurring downtime. Under time pressure, we weren't able to find a solution that we were happy with. The best idea we had was to create a completely new cluster, restored from the 13:33 UTC snapshot, and then change our DNS records to point at this new cluster. But that was complex, hadn't been tested, and would take time.</p>
|
||
<p>In the end, we decided we'd have to accept a small period of downtime. The plan was to take <code>chesterfield-001</code> offline again, restore it to the 13:33 UTC snapshot, but before bringing it back up take <code>chesterfield-002</code> and <code>chesterfield-003</code> offline. We would have downtime until <code>chesterfield-001</code> came back up, but timed well this should only be a minute or two. This way <code>chesterfield-001</code> would not synchronise to the other two nodes, and we could bring the other two nodes back up shortly after, restored to the 13:33 UTC snapshot.</p>
|
||
<figure class="not-prose my-8 space-y-3">
|
||
<img src="https://res.cloudinary.com/daog6scxm/image/upload/v1705397473/restoration-dance_ijjd3m.png" alt="A graph of healthy Chesterfield hosts, showing the total drop from 3 to 0 before rising back up to 3 again" class="w-full rounded-sm border" loading="lazy">
|
||
|
||
</figure>
|
||
<p>By 15:39 UTC we were restored, but running on a single CouchDB node. By 15:49 UTC our cluster was fully restored to the 13:33 UTC snapshot and Budibase Cloud was operational again.</p>
|
||
<p>We had one problem that persisted after the recovery. Some types of data from our CouchDB cluster get cached in the Budibase app, and during the incident window we had cached some reads from <code>chesterfield-004</code> that were empty. These cached keys persisted after the cluster had recovered, but we were able to resolve this problem by clearing that part of the cache and allowing it to rebuild from the newly-restored cluster.</p>
|
||
<h2 id="what-were-going-to-do-next">What we're going to do next</h2>
|
||
<p>This incident has revealed several weaknesses in our processes that need resolving.</p>
|
||
<ol>
|
||
<li>We are going to fix our automation around bringing up new CouchDB nodes and test it thoroughly in a safe, non-production environment. This is an automation we run very infrequently, and in this instance we should have first tested it in a non-production environment instead of trusting that it still worked.</li>
|
||
<li>We are going to write a disaster recovery plan for a full cluster rollback, so that if this happens again in future we will be able to react quicker. Had we had this available to us in this incident, we could have saved approximately 30 minutes in recovery time.</li>
|
||
<li>We were not alerted to this problem automatically, relying instead on the diligence of our customers reaching out to us. It is only through luck that so few customers were affected this time, and we don't want to rely on luck going forward. We are going figure out a good way to be automatically alerted to data loss situations in future.</li>
|
||
<li>The Budibase code should not be lazily creating databases the way that it does. This is technical debt we have carried with us from the days that Budibase was a desktop app, and it's time we paid it down. We are going to prioritise work to only create databases when necessary. This change would likely have prevented much of the data loss we saw.</li>
|
||
</ol>
|
||
<h2 id="conclusion">Conclusion</h2>
|
||
<p>Thank you for reading this incident report. We believe by understanding these incidents in full, coming up with action items that would prevent them in future, and following through on those action items, Budibase will get better every day.</p> </article> <div class="gap-4 lg:gap-5 mx-auto px-4 md:px-6 mt-20 md:mt-40 lg:mt-50 grid-cols-1 lg:grid-cols-6 max-w-homepage lg:px-0 lg:mx-auto box-content flex items-center my-0! py-12 md:py-25 lg:py-35"> <div class="w-full col-span-1 lg:col-span-6 flex flex-col items-center text-center"> <h2 class="text-2xl md:text-3xl lg:text-5xl! text-primary mb-4 max-w-lg px-2">Sign up today.</h2> <p class="text-stone-700 dark:text-stone-200! text-base md:text-lg lg:text-xl! mb-6 md:mb-8! text-center max-w-lg px-2">Save weeks building agents, chat, automations, and apps with your models, your tools and data. </p> <a href="https://account.budibase.app/register" class="inline-flex items-center justify-center rounded-full !font-medium transition-colors duration-150 active:scale-[0.98] no-underline cursor-pointer font-sans focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-bg bg-brand text-on-brand hover:bg-brand-hover active:bg-brand-hover dark:text-grey-50 px-4 py-2 text-lg gap-2" role="button">Get Budibase free</a> </div> </div> </div> <footer class="footer bg-1 border-t-[0.5px] chrome-border" data-astro-transition-persist="site-footer" data-astro-cid-sz7xmlte data-astro-transition-scope="astro-tvayyljk-4"> <div class="footer-container mx-auto w-full max-w-7xl px-2 md:px-3 lg:px-0" data-astro-cid-sz7xmlte> <!-- Upper section: Navigation columns --> <div class="footer-nav" data-astro-cid-sz7xmlte> <div class="footer-column" data-astro-cid-sz7xmlte> <p data-astro-cid-sz7xmlte>Product</p> <ul class="text-stone-500! flex flex-col gap-1.5! sm:gap-0.5!" data-astro-cid-sz7xmlte> <li data-astro-cid-sz7xmlte> <a href="/product/agents/" target="_self" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> Agents <!-- allow optional icon/content --> </a> </li><li data-astro-cid-sz7xmlte> <a href="/product/automations/" target="_self" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> Automations <!-- allow optional icon/content --> </a> </li><li data-astro-cid-sz7xmlte> <a href="/product/apps/" target="_self" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> Apps <!-- allow optional icon/content --> </a> </li><li data-astro-cid-sz7xmlte> <a href="/pricing/" target="_self" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> Pricing <!-- allow optional icon/content --> </a> </li><li data-astro-cid-sz7xmlte> <a href="/ops/" target="_self" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> Use cases <!-- allow optional icon/content --> </a> </li> </ul> </div><div class="footer-column" data-astro-cid-sz7xmlte> <p data-astro-cid-sz7xmlte>Resources</p> <ul class="text-stone-500! flex flex-col gap-1.5! sm:gap-0.5!" data-astro-cid-sz7xmlte> <li data-astro-cid-sz7xmlte> <a href="/blog/" target="_self" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> Blog <!-- allow optional icon/content --> </a> </li><li data-astro-cid-sz7xmlte> <a href="/changelog/" target="_self" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> Changelog <!-- allow optional icon/content --> </a> </li><li data-astro-cid-sz7xmlte> <a href="/customers/" target="_self" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> Customers <!-- allow optional icon/content --> </a> </li><li data-astro-cid-sz7xmlte> <a href="https://docs.budibase.com" target="_blank" rel="noopener noreferrer" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> Docs <!-- allow optional icon/content --> </a> </li><li data-astro-cid-sz7xmlte> <a href="/security/" target="_self" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> Security <!-- allow optional icon/content --> </a> </li><li data-astro-cid-sz7xmlte> <a href="/it/" target="_self" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> IT Teams <!-- allow optional icon/content --> </a> </li> </ul> </div><div class="footer-column" data-astro-cid-sz7xmlte> <p data-astro-cid-sz7xmlte>Legal</p> <ul class="text-stone-500! flex flex-col gap-1.5! sm:gap-0.5!" data-astro-cid-sz7xmlte> <li data-astro-cid-sz7xmlte> <a href="/support/" target="_self" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> Support Policy <!-- allow optional icon/content --> </a> </li><li data-astro-cid-sz7xmlte> <a href="/terms/" target="_self" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> Terms of Service <!-- allow optional icon/content --> </a> </li><li data-astro-cid-sz7xmlte> <a href="/privacy/" target="_self" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> Privacy Policy <!-- allow optional icon/content --> </a> </li> </ul> </div><div class="footer-column" data-astro-cid-sz7xmlte> <p data-astro-cid-sz7xmlte>Connect</p> <ul class="text-stone-500! flex flex-col gap-1.5! sm:gap-0.5!" data-astro-cid-sz7xmlte> <li data-astro-cid-sz7xmlte> <a href="https://discord.com/invite/budibase-733030666647765003" target="_blank" rel="noopener noreferrer" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> Discord <!-- allow optional icon/content --> </a> </li><li data-astro-cid-sz7xmlte> <a href="https://linkedin.com/company/budibase" target="_blank" rel="noopener noreferrer" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> LinkedIn <!-- allow optional icon/content --> </a> </li><li data-astro-cid-sz7xmlte> <a href="https://youtube.com/@budibase" target="_blank" rel="noopener noreferrer" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> YouTube <!-- allow optional icon/content --> </a> </li><li data-astro-cid-sz7xmlte> <a href="/partners/" target="_self" class="alink small secondary text-stone-500 dark:text-stone-400 hover:text-stone-700 dark:hover:text-stone-300 footer-link min-h-12 py-1.5 sm:min-h-0 sm:py-0" data-astro-cid-mbqdmgin> Partners <!-- allow optional icon/content --> </a> </li> </ul> </div> </div> <!-- Lower section: Copyright and controls --> <div class="footer-bottom" data-astro-cid-sz7xmlte> <div class="footer-copyright" data-astro-cid-sz7xmlte> <span data-astro-cid-sz7xmlte>© 2026 Budibase, Inc.</span> <span class="footer-certification" data-astro-cid-sz7xmlte> <svg class="footer-check-icon" data-astro-cid-sz7xmlte="true" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 256" fill="currentColor"><path d="M128,136a8,8,0,0,1-8,8H72a8,8,0,0,1,0-16h48A8,8,0,0,1,128,136Zm-8-40H72a8,8,0,0,0,0,16h48a8,8,0,0,0,0-16Zm112,65.47V224A8,8,0,0,1,220,231l-24-13.74L172,231A8,8,0,0,1,160,224V200H40a16,16,0,0,1-16-16V56A16,16,0,0,1,40,40H216a16,16,0,0,1,16,16V86.53a51.88,51.88,0,0,1,0,74.94ZM160,184V161.47A52,52,0,0,1,216,76V56H40V184Zm56-12a51.88,51.88,0,0,1-40,0v38.22l16-9.16a8,8,0,0,1,7.94,0l16,9.16Zm16-48a36,36,0,1,0-36,36A36,36,0,0,0,232,124Z"></path></svg> <span data-astro-cid-sz7xmlte>ISO 27001 Certified</span> </span> <span class="footer-certification" data-astro-cid-sz7xmlte> <svg class="footer-check-icon" data-astro-cid-sz7xmlte="true" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 256" fill="currentColor"><path d="M173.66,98.34a8,8,0,0,1,0,11.32l-56,56a8,8,0,0,1-11.32,0l-24-24a8,8,0,0,1,11.32-11.32L112,148.69l50.34-50.35A8,8,0,0,1,173.66,98.34ZM232,128A104,104,0,1,1,128,24,104.11,104.11,0,0,1,232,128Zm-16,0a88,88,0,1,0-88,88A88.1,88.1,0,0,0,216,128Z"></path></svg> <span data-astro-cid-sz7xmlte>GDPR Compliant</span> </span> <span class="footer-certification" data-astro-cid-sz7xmlte> <svg class="footer-check-icon" data-astro-cid-sz7xmlte="true" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 256" fill="currentColor"><path d="M128,24h0A104,104,0,1,0,232,128,104.12,104.12,0,0,0,128,24Zm87.62,96H175.79C174,83.49,159.94,57.67,148.41,42.4A88.19,88.19,0,0,1,215.63,120ZM96.23,136h63.54c-2.31,41.61-22.23,67.11-31.77,77C118.45,203.1,98.54,177.6,96.23,136Zm0-16C98.54,78.39,118.46,52.89,128,43c9.55,9.93,29.46,35.43,31.77,77Zm11.36-77.6C96.06,57.67,82,83.49,80.21,120H40.37A88.19,88.19,0,0,1,107.59,42.4ZM40.37,136H80.21c1.82,36.51,15.85,62.33,27.38,77.6A88.19,88.19,0,0,1,40.37,136Zm108,77.6c11.53-15.27,25.56-41.09,27.38-77.6h39.84A88.19,88.19,0,0,1,148.41,213.6Z"></path></svg> <span data-astro-cid-sz7xmlte>Made in Europe</span> </span> </div> <div class="footer-controls" data-astro-cid-sz7xmlte> <div class="flex bg-stone-100 dark:bg-stone-900 rounded-lg p-0.5 gap-0.5" role="group" aria-label="Theme selection" data-astro-cid-x3pjskd3> <!-- Hidden element to ensure Tailwind includes dynamically added classes --> <div class="hidden bg-stone-700 dark:bg-stone-600 text-stone-50 dark:text-stone-50" data-astro-cid-x3pjskd3></div> <button type="button" class="theme-button flex items-center justify-center w-11 h-11 p-0 border-none bg-transparent rounded-md cursor-pointer transition-all duration-150 text-stone-600 dark:text-stone-400 hover:bg-stone-200 dark:hover:bg-stone-800 hover:text-stone-800 dark:hover:text-stone-200 sm:w-8 sm:h-8" data-theme="system" aria-label="System theme" title="Use system theme" data-astro-cid-x3pjskd3> <svg class="w-4 h-4" style="width: 16px; height: 16px;" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 256" fill="currentColor"><path d="M208,40H48A24,24,0,0,0,24,64V176a24,24,0,0,0,24,24H208a24,24,0,0,0,24-24V64A24,24,0,0,0,208,40Zm8,136a8,8,0,0,1-8,8H48a8,8,0,0,1-8-8V64a8,8,0,0,1,8-8H208a8,8,0,0,1,8,8Zm-48,48a8,8,0,0,1-8,8H96a8,8,0,0,1,0-16h64A8,8,0,0,1,168,224Z"></path></svg> </button> <button type="button" class="theme-button flex items-center justify-center w-11 h-11 p-0 border-none bg-transparent rounded-md cursor-pointer transition-all duration-150 text-stone-600 dark:text-stone-400 hover:bg-stone-200 dark:hover:bg-stone-800 hover:text-stone-800 dark:hover:text-stone-200 sm:w-8 sm:h-8" data-theme="light" aria-label="Light theme" title="Light theme" data-astro-cid-x3pjskd3> <svg class="w-4 h-4" style="width: 16px; height: 16px;" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 256" fill="currentColor"><path d="M120,40V16a8,8,0,0,1,16,0V40a8,8,0,0,1-16,0Zm72,88a64,64,0,1,1-64-64A64.07,64.07,0,0,1,192,128Zm-16,0a48,48,0,1,0-48,48A48.05,48.05,0,0,0,176,128ZM58.34,69.66A8,8,0,0,0,69.66,58.34l-16-16A8,8,0,0,0,42.34,53.66Zm0,116.68-16,16a8,8,0,0,0,11.32,11.32l16-16a8,8,0,0,0-11.32-11.32ZM192,72a8,8,0,0,0,5.66-2.34l16-16a8,8,0,0,0-11.32-11.32l-16,16A8,8,0,0,0,192,72Zm5.66,114.34a8,8,0,0,0-11.32,11.32l16,16a8,8,0,0,0,11.32-11.32ZM48,128a8,8,0,0,0-8-8H16a8,8,0,0,0,0,16H40A8,8,0,0,0,48,128Zm80,80a8,8,0,0,0-8,8v24a8,8,0,0,0,16,0V216A8,8,0,0,0,128,208Zm112-88H216a8,8,0,0,0,0,16h24a8,8,0,0,0,0-16Z"></path></svg> </button> <button type="button" class="theme-button flex items-center justify-center w-11 h-11 p-0 border-none bg-transparent rounded-md cursor-pointer transition-all duration-150 text-stone-600 dark:text-stone-400 hover:bg-stone-200 dark:hover:bg-stone-800 hover:text-stone-800 dark:hover:text-stone-200 sm:w-8 sm:h-8" data-theme="dark" aria-label="Dark theme" title="Dark theme" data-astro-cid-x3pjskd3> <svg class="w-4 h-4" style="width: 16px; height: 16px;" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 256" fill="currentColor"><path d="M233.54,142.23a8,8,0,0,0-8-2,88.08,88.08,0,0,1-109.8-109.8,8,8,0,0,0-10-10,104.84,104.84,0,0,0-52.91,37A104,104,0,0,0,136,224a103.09,103.09,0,0,0,62.52-20.88,104.84,104.84,0,0,0,37-52.91A8,8,0,0,0,233.54,142.23ZM188.9,190.34A88,88,0,0,1,65.66,67.11a89,89,0,0,1,31.4-26A106,106,0,0,0,96,56,104.11,104.11,0,0,0,200,160a106,106,0,0,0,14.92-1.06A89,89,0,0,1,188.9,190.34Z"></path></svg> </button> </div> <script type="module">(function(){function s(e){return e==="dark"?"github-dark":"github-light"}function c(){const e=localStorage.getItem("theme");return e==="dark"||e==="light"?e:window.matchMedia("(prefers-color-scheme: dark)").matches?"dark":"light"}function n(e=!1){e&&document.documentElement.classList.add("no-transitions");const o=c();document.documentElement.classList.toggle("dark",o==="dark"),document.documentElement.dataset.theme=s(o),e&&(document.documentElement.offsetHeight,document.documentElement.classList.remove("no-transitions")),d()}function d(){const e=document.querySelector('[role="group"][aria-label="Theme selection"]');if(!e)return;const o=e.querySelectorAll(".theme-button"),a=localStorage.getItem("theme")||"system";o.forEach(t=>{t.getAttribute("data-theme")===a?(t.classList.add("bg-stone-200","dark:bg-stone-800","text-stone-950","dark:text-stone-50"),t.classList.remove("text-stone-600","dark:text-stone-400","bg-transparent")):(t.classList.remove("bg-stone-700","dark:bg-stone-600","text-stone-50","dark:text-stone-50"),t.classList.add("text-stone-600","dark:text-stone-400","bg-transparent"))})}function m(e){e==="system"?localStorage.removeItem("theme"):localStorage.setItem("theme",e),n(!0)}function r(){const e=document.querySelector('[role="group"][aria-label="Theme selection"]');if(!e)return;if(e.dataset.themeToggleReady==="true"){n();return}e.dataset.themeToggleReady="true";const o=e.querySelectorAll(".theme-button");n(),o.forEach(a=>{a.addEventListener("click",()=>{const t=a.getAttribute("data-theme");t&&m(t)})})}document.readyState==="loading"?document.addEventListener("DOMContentLoaded",r):r(),document.addEventListener("astro:after-swap",()=>{n(!0),setTimeout(r,0)}),window.matchMedia("(prefers-color-scheme: dark)").addEventListener("change",()=>{localStorage.getItem("theme")||n()})})();</script> </div> </div> </div> </footer> </div> </div> </div> </main> </div> <script>
|
||
(() => {
|
||
const root = document.documentElement;
|
||
|
||
const setNavHeight = () => {
|
||
const nav = document.getElementById("site-nav");
|
||
if (!nav) return;
|
||
const { height } = nav.getBoundingClientRect();
|
||
root.style.setProperty("--site-nav-height", `${Math.round(height)}px`);
|
||
};
|
||
|
||
const onReady = () => {
|
||
setNavHeight();
|
||
if (!window.__siteNavHeightListenerBound) {
|
||
window.addEventListener("resize", setNavHeight, { passive: true });
|
||
window.__siteNavHeightListenerBound = true;
|
||
}
|
||
};
|
||
|
||
if (document.readyState === "loading") {
|
||
document.addEventListener("DOMContentLoaded", onReady, { once: true });
|
||
} else {
|
||
onReady();
|
||
}
|
||
|
||
document.addEventListener("astro:after-swap", () => {
|
||
requestAnimationFrame(setNavHeight);
|
||
});
|
||
})();
|
||
</script> </body></html> |