Files
nexus/sreweekly/articles/531/07-how-tailscale-helped-find-the-sqlite-wal-reset-bug.html

1 line
191 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html><head><meta charSet="utf-8"/><title>How Tailscale helped find the SQLite WAL-Reset bug</title><meta name="robots" content="index,follow"/><meta name="description" content="Tailscale and SQLite developers traced maddening corruption incidents to find the WAL-Reset data race, then uncovered a second stale expression index bug."/><meta property="og:title" content="How Tailscale helped find the SQLite WAL-Reset bug"/><meta property="og:description" content="Tailscale and SQLite developers traced maddening corruption incidents to find the WAL-Reset data race, then uncovered a second stale expression index bug."/><meta property="og:url" content="https://tailscale.com/blog/sqlite-wal-reset-bug"/><meta property="og:image" content="https://tailscale.com/api/og-image?id=45389fc1-da6e-4e0d-82f6-140651c6ba57"/><link rel="canonical" href="https://tailscale.com/blog/sqlite-wal-reset-bug"/><link rel="alternate" type="application/rss+xml" href="https://tailscale.com/blog/index.xml"/><link rel="alternate" type="application/rss+xml" href="https://tailscale.com/changelog/index.xml"/><link rel="alternate" type="application/rss+xml" href="https://tailscale.com/security-bulletins/index.xml"/><meta name="viewport" content="initial-scale=1.0, width=device-width, maximum-scale=1"/><link rel="icon" href="/favicon.png" type="image/png"/><link rel="icon" href="/favicon.svg" type="image/svg+xml"/><meta name="msapplication-TileColor" content="#492847"/><meta name="theme-color" content="#ffffff"/><script async="" src="https://bwa.marketplace.awsstatic.com/assets/partner.js"></script><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:title" content="How Tailscale helped find the SQLite WAL-Reset bug"/><meta name="twitter:image" content="https://tailscale.com/api/og-image?id=45389fc1-da6e-4e0d-82f6-140651c6ba57"/><meta name="twitter:description" content="Tailscale and SQLite developers traced maddening corruption incidents to find the WAL-Reset data race, then uncovered a second stale expression index bug."/><meta name="twitter:site" content="@tailscale"/><meta name="next-head-count" content="23"/><link rel="preload" href="/_next/static/media/97a52bce187043ec-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/media/e4af272ccee01ff0-s.p.woff2" as="font" type="font/woff2" crossorigin="anonymous" data-next-font="size-adjust"/><link rel="preload" href="/_next/static/css/1ab5b6ae56568d1f.css" as="style"/><link rel="stylesheet" href="/_next/static/css/1ab5b6ae56568d1f.css" data-n-g=""/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-42372ed130431b0a.js"></script><script src="/_next/static/chunks/webpack-bd5564c61acfdf0e.js" defer=""></script><script src="/_next/static/chunks/framework-5c59b3faca784f4c.js" defer=""></script><script src="/_next/static/chunks/main-fc51a042ada8f325.js" defer=""></script><script src="/_next/static/chunks/pages/_app-6904be075d7b128c.js" defer=""></script><script src="/_next/static/chunks/47397-1c0811c5767dd37e.js" defer=""></script><script src="/_next/static/chunks/48354-1c6772f2b5b4e807.js" defer=""></script><script src="/_next/static/chunks/63448-1e4bb16637393fff.js" defer=""></script><script src="/_next/static/chunks/88556-68c54805ab62c57a.js" defer=""></script><script src="/_next/static/chunks/8100-98b2867a0c4566e6.js" defer=""></script><script src="/_next/static/chunks/26485-d62792b96ea3b48b.js" defer=""></script><script src="/_next/static/chunks/92099-5ae4e0be6ff96794.js" defer=""></script><script src="/_next/static/chunks/84328-88d7cfb3810d9315.js" defer=""></script><script src="/_next/static/chunks/5769-fb22ae43ed03c83e.js" defer=""></script><script src="/_next/static/chunks/44810-76288fe13347663d.js" defer=""></script><script src="/_next/static/chunks/69018-676c20cf88cc7bdb.js" defer=""></script><script src="/_next/static/chunks/pages/blog/%5Bslug%5D-0739ad507bb92110.js" defer=""></script><script src="/_next/static/YcO3-STY7pBVXk7rkvXgF/_buildManifest.js" defer=""></script><script src="/_next/static/YcO3-STY7pBVXk7rkvXgF/_ssgManifest.js" defer=""></script><style id="__jsx-459994682">:root{--font-inter:'__Inter_98cf97', '__Inter_Fallback_98cf97';--font-mdio:'__MDIOFont_8d6c39', '__MDIOFont_Fallback_8d6c39'}</style></head><body><div id="__next"><!--$--><!--/$--><div class="sticky top-0 z-[20000]"><div class="z-[20000] w-full p-4 font-mdio text-xs uppercase !tracking-[0.6px] xl-small:p-1.5 bg-gray-700 text-white"><div class="mx-auto flex w-full max-w-[1360px] items-center justify-center px-6 xl-small:max-w-[1440px] xl-small:justify-between xl-small:px-16"><div><a class="group flex flex-wrap items-center justify-center gap-x-2 text-pretty pt-0.5 text-center font-medium text-white sm:text-left" href="https://tailscale.com/tailscaleup-date-week-26"><span class="hidden sm:block"><svg width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg" style="color:#BE8FE1" aria-hidden="true"><path d="M12 3L10.088 8.813C9.99015 9.11051 9.82379 9.38088 9.60234 9.60234C9.38088 9.82379 9.11051 9.99015 8.813 10.088L3 12L8.813 13.912C9.11051 14.0099 9.38088 14.1762 9.60234 14.3977C9.82379 14.6191 9.99015 14.8895 10.088 15.187L12 21L13.912 15.187C14.0099 14.8895 14.1762 14.6191 14.3977 14.3977C14.6191 14.1762 14.8895 14.0099 15.187 13.912L21 12L15.187 10.088C14.8895 9.99015 14.6191 9.82379 14.3977 9.60234C14.1762 9.38088 14.0099 9.11051 13.912 8.813L12 3Z" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path><path d="M5 3V7" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path><path d="M19 17V21" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path><path d="M3 5H7" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path><path d="M17 19H21" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></span>See everything you missed at TailscaleUp.<span class="pl-2 text-gray-0/80 underline group-hover:no-underline">Read the recap →</span></a></div><nav class="hidden xl-small:block"><ul class="flex gap-6"><li><a class="block py-3.5 font-medium text-gray-0/80 hover:underline" href="/blog">Blog</a></li><li><a class="block py-3.5 font-medium text-gray-0/80 hover:underline" href="/docs">Docs</a></li><li><a class="block py-3.5 font-medium text-gray-0/80 hover:underline" href="/download">Download</a></li><li><a class="block py-3.5 font-medium text-gray-0/80 hover:underline" href="/contact/sales">Contact Sales</a></li></ul></nav></div></div><header class="bg-white transition-colors duration-300"><div class="mx-auto flex h-[75px] w-full max-w-[1360px] items-center justify-between gap-[4px] px-6 md:max-w-[1440px] lg:px-16"><a href="/"><svg width="145" height="28" viewBox="0 0 145 28" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M3.5 17.5C5.43299 17.5 6.99999 15.933 6.99999 14C6.99999 12.067 5.43299 10.5 3.5 10.5C1.567 10.5 0 12.067 0 14C0 15.933 1.567 17.5 3.5 17.5Z" fill="#232222"></path><path d="M14 17.5C15.933 17.5 17.5 15.933 17.5 14C17.5 12.067 15.933 10.5 14 10.5C12.067 10.5 10.5 12.067 10.5 14C10.5 15.933 12.067 17.5 14 17.5Z" fill="#232222"></path><path d="M14 28C15.933 28 17.5 26.433 17.5 24.5C17.5 22.567 15.933 21 14 21C12.067 21 10.5 22.567 10.5 24.5C10.5 26.433 12.067 28 14 28Z" fill="#232222"></path><path d="M24.5 17.5C26.433 17.5 28 15.933 28 14C28 12.067 26.433 10.5 24.5 10.5C22.567 10.5 21 12.067 21 14C21 15.933 22.567 17.5 24.5 17.5Z" fill="#232222"></path><g opacity="0.4"><path d="M3.5 28C5.43299 28 6.99999 26.433 6.99999 24.5C6.99999 22.567 5.43299 21 3.5 21C1.567 21 0 22.567 0 24.5C0 26.433 1.567 28 3.5 28Z" fill="#232222"></path><path d="M24.5 28C26.433 28 28 26.433 28 24.5C28 22.567 26.433 21 24.5 21C22.567 21 21 22.567 21 24.5C21 26.433 22.567 28 24.5 28Z" fill="#232222"></path><path d="M3.5 6.99999C5.43299 6.99999 6.99999 5.43299 6.99999 3.5C6.99999 1.567 5.43299 0 3.5 0C1.567 0 0 1.567 0 3.5C0 5.43299 1.567 6.99999 3.5 6.99999Z" fill="#232222"></path><path d="M14 6.99999C15.933 6.99999 17.5 5.43299 17.5 3.5C17.5 1.567 15.933 0 14 0C12.067 0 10.5 1.567 10.5 3.5C10.5 5.43299 12.067 6.99999 14 6.99999Z" fill="#232222"></path><path d="M24.5 6.99999C26.433 6.99999 28 5.43299 28 3.5C28 1.567 26.433 0 24.5 0C22.567 0 21 1.567 21 3.5C21 5.43299 22.567 6.99999 24.5 6.99999Z" fill="#232222"></path></g><path d="M46.1746 24.5C47.0875 24.5 47.8293 24.4144 48.7137 24.2432V20.9334C48.1717 21.1331 47.5725 21.1902 47.0019 21.1902C45.604 21.1902 45.0904 20.5054 45.0904 19.1929V12.4592H48.7137V9.14945H45.0904V4.24185H41.0962V9.14945H38.5V12.4592H41.0962V19.5638C41.0962 22.7024 42.751 24.5 46.1746 24.5Z" fill="#232222"></path><path d="M55.2368 24.5C57.3766 24.5 58.8031 23.7867 59.5163 22.6739C59.5734 23.159 59.7446 23.8152 59.9728 24.2147H63.5676C63.3679 23.5299 63.2252 22.4456 63.2252 21.7323V13.9144C63.2252 10.6902 60.9143 8.86412 57.0057 8.86412C54.0386 8.86412 51.7277 9.9769 50.4438 11.6033L52.7262 13.8573C53.7533 12.716 55.0657 12.1168 56.6633 12.1168C58.6034 12.1168 59.4593 12.7731 59.4593 13.7432C59.4593 14.5706 58.8887 15.1413 55.8074 15.1413C52.8403 15.1413 49.8447 16.3682 49.8447 19.7921C49.8447 22.9592 52.1271 24.5 55.2368 24.5ZM56.0357 21.4755C54.4951 21.4755 53.7818 20.8193 53.7818 19.6494C53.7818 18.6223 54.6377 17.9375 56.0642 17.9375C58.3466 17.9375 58.9743 17.7948 59.4593 17.3098V18.5367C59.4593 20.163 58.0898 21.4755 56.0357 21.4755Z" fill="#232222"></path><path d="M65.8217 7.23777H69.9871V3.5H65.8217V7.23777ZM65.9073 24.2147H69.9015V9.14945H65.9073V24.2147Z" fill="#232222"></path><path d="M72.8113 24.2147H76.8055V3.95652H72.8113V24.2147Z" fill="#232222"></path><path d="M85.1075 24.5C89.5011 24.5 91.755 22.1888 91.755 19.7636C91.755 17.5666 90.6423 15.9402 86.9905 15.284C84.4228 14.8274 82.8251 14.2568 82.8251 13.3723C82.8251 12.6019 83.7381 12.0598 85.2787 12.0598C86.6481 12.0598 87.7893 12.5163 88.6452 13.5149L91.1844 11.375C89.9005 9.77717 87.7893 8.86412 85.2787 8.86412C81.4271 8.86412 79.0306 10.9185 79.0306 13.4293C79.0306 16.1685 81.5412 17.3954 84.2516 17.8804C86.5055 18.2799 87.7608 18.7079 87.7608 19.678C87.7608 20.5054 86.8763 21.1902 85.1931 21.1902C83.5383 21.1902 82.2545 20.4198 81.6554 19.2785L78.2888 20.8763C79.1162 22.9307 81.9407 24.5 85.1075 24.5Z" fill="#232222"></path><path d="M100.685 24.5C103.538 24.5 105.592 23.3587 107.132 20.7907L103.937 19.1644C103.338 20.3628 102.368 21.1902 100.685 21.1902C98.1455 21.1902 96.776 19.1359 96.776 16.6535C96.776 14.1712 98.2311 12.1739 100.685 12.1739C102.225 12.1739 103.252 13.0014 103.851 14.2568L107.075 12.5448C105.991 10.1766 103.851 8.86412 100.685 8.86412C95.4636 8.86412 92.7818 12.6875 92.7818 16.6535C92.7818 20.9619 96.0342 24.5 100.685 24.5Z" fill="#232222"></path><path d="M113.495 24.5C115.635 24.5 117.061 23.7867 117.774 22.6739C117.831 23.159 118.003 23.8152 118.231 24.2147H121.826C121.626 23.5299 121.483 22.4456 121.483 21.7323V13.9144C121.483 10.6902 119.172 8.86412 115.264 8.86412C112.297 8.86412 109.986 9.9769 108.702 11.6033L110.984 13.8573C112.011 12.716 113.324 12.1168 114.921 12.1168C116.861 12.1168 117.717 12.7731 117.717 13.7432C117.717 14.5706 117.147 15.1413 114.065 15.1413C111.098 15.1413 108.103 16.3682 108.103 19.7921C108.103 22.9592 110.385 24.5 113.495 24.5ZM114.294 21.4755C112.753 21.4755 112.04 20.8193 112.04 19.6494C112.04 18.6223 112.896 17.9375 114.322 17.9375C116.605 17.9375 117.232 17.7948 117.717 17.3098V18.5367C117.717 20.163 116.348 21.4755 114.294 21.4755Z" fill="#232222"></path><path d="M123.831 24.2147H127.825V3.95652H123.831V24.2147Z" fill="#232222"></path><path d="M137.411 24.5C140.35 24.5 142.718 23.3301 144.03 21.2473L141.006 19.3071C140.207 20.5625 139.151 21.2473 137.411 21.2473C135.442 21.2473 134.016 20.163 133.645 18.0516H145V16.6535C145 12.6875 142.575 8.86412 137.382 8.86412C132.447 8.86412 129.765 12.716 129.765 16.6821C129.765 22.3315 134.016 24.5 137.411 24.5ZM133.788 14.913C134.415 13.0014 135.728 12.1168 137.468 12.1168C139.437 12.1168 140.635 13.2867 141.034 14.913H133.788Z" fill="#232222"></path></svg></a><div class="flex w-full items-center justify-end gap-3 xl-small:justify-between"><nav class="website-nav m-auto hidden xl-small:relative xl-small:block"><ul class="flex gap-x-[4px]"><li><button type="button" class="flex items-center gap-[4px] rounded-md px-3 py-2.5 text-sm font-medium hover:bg-gray-200">Platform<svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M2.91675 7H11.0834" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round"></path><path d="M7 2.9165V11.0832" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round" style="opacity:1;transform:scaleY(1);transform-origin:center;transition:opacity 150ms ease-in-out, transform 150ms ease-in-out"></path></svg></button><div class="pointer-events-none absolute left-1/2 top-full z-[100] mt-3 -translate-x-1/2 pt-4 opacity-0"><div class="overflow-hidden rounded-2xl border border-gray-200 shadow-8"><div class="grid auto-cols-[268px] grid-flow-col gap-4 border-b border-gray-200 bg-white p-8 shadow-2"><div class="flex flex-col gap-4"><div class="flex items-center gap-2.5 px-3"><p class="whitespace-nowrap font-mdio text-[14px] font-medium uppercase !tracking-[0.6px] text-gray-700/80">Meet Tailscale</p><div class="h-px w-full bg-gray-200"></div></div><ul class="flex flex-col gap-1"><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;How Tailscale Works&quot;}" href="/blog/how-tailscale-works">How Tailscale Works</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;WireGuard® for Enterprises&quot;}" href="/wireguard-vpn">WireGuard® for Enterprises</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Features&quot;}" href="/features">Features</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Integrations&quot;}" href="/integrations">Integrations</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Docs&quot;}" href="/docs">Docs</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Download&quot;}" href="/download">Download</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Compare Tailscale&quot;}" href="/compare">Compare Tailscale</a></li></ul></div><div class="flex flex-col gap-4"><div class="flex items-center gap-2.5 px-3"><p class="whitespace-nowrap font-mdio text-[14px] font-medium uppercase !tracking-[0.6px] text-gray-700/80">Products</p><div class="h-px w-full bg-gray-200"></div></div><ul class="flex flex-col gap-1"><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Business VPN&quot;}" href="/use-cases/business-vpn">Business VPN</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Tailscale PAM&quot;}" href="/use-cases/pam">Tailscale PAM</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;CI/CD Connectivity&quot;}" href="/use-cases/ci-cd">CI/CD Connectivity</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Secure Access to AI&quot;}" href="/use-cases/securing-ai">Secure Access to AI</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Workload Connectivity&quot;}" href="/use-cases/workload-connectivity">Workload Connectivity</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Edge &amp; IoT&quot;}" href="/use-cases/edge-iot">Edge &amp; IoT</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Homelab&quot;}" href="/use-cases/homelab">Homelab</a></li></ul></div><a class="flex flex-col self-stretch overflow-hidden rounded-md border border-gray-200 bg-gray-100 shadow-2" href="https://tailscale.com/use-cases/securing-ai"><div class="relative flex flex-1 items-center"><img alt="aperture dashboard" loading="lazy" width="268" height="150" decoding="async" data-nimg="1" class="h-auto w-full rounded-none border-none object-cover" style="color:transparent" src="https://cdn.sanity.io/images/w77i7m8x/production-v2/a88cf9aad8777f4b0641202a5ed6c02bbb005d8c-266x342.svg"/></div><div class="bg-gray-50 p-8"><p class="mb-1 font-mdio text-[14px] font-medium uppercase !tracking-[0.6px] text-gray-700">Aperture by Tailscale</p><p class="text-base text-gray-700/80">Unified AI governance for AI agents and users. </p></div></a></div><div class="bg-gray-50 p-8"><div class="flex items-center justify-between"><div class="flex flex-col gap-1"><p class="font-mdio text-[14px] font-medium uppercase !tracking-[0.6px] text-gray-700">Bring Tailscale to work</p><p class="text-base text-gray-700/80">See how Tailscale fits the systems your team already runs.</p></div><a class="inline-flex h-9 shrink-0 items-center justify-center rounded-md border border-gray-800 bg-gray-800 px-3 text-base font-medium text-white shadow-2 transition-all duration-200 hover:opacity-80 hover:shadow-4" href="https://tailscale.com/bring-tailscale-to-work">Make the case</a></div></div></div></div></li><li><button type="button" class="flex items-center gap-[4px] rounded-md px-3 py-2.5 text-sm font-medium hover:bg-gray-200">Solutions<svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M2.91675 7H11.0834" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round"></path><path d="M7 2.9165V11.0832" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round" style="opacity:1;transform:scaleY(1);transform-origin:center;transition:opacity 150ms ease-in-out, transform 150ms ease-in-out"></path></svg></button><div class="pointer-events-none absolute left-1/2 top-full z-[100] mt-3 -translate-x-1/2 pt-4 opacity-0"><div class="overflow-hidden rounded-2xl border border-gray-200 shadow-8"><div class="grid auto-cols-[268px] grid-flow-col gap-4 border-b border-gray-200 bg-white p-8 shadow-2"><div class="flex flex-col gap-4"><div class="flex items-center gap-2.5 px-3"><p class="whitespace-nowrap font-mdio text-[14px] font-medium uppercase !tracking-[0.6px] text-gray-700/80">Solutions</p><div class="h-px w-full bg-gray-200"></div></div><ul class="flex flex-col gap-1"><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Cloud Connectivity&quot;}" href="/use-cases/cloud-connectivity">Cloud Connectivity</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Infrastructure Access&quot;}" href="/use-cases/infrastructure-access">Infrastructure Access</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Zero Trust Networking&quot;}" href="/use-cases/zero-trust-networking">Zero Trust Networking</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Remote Access&quot;}" href="/use-cases/remote-access">Remote Access</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Kubernetes Networking&quot;}" href="/use-cases/kubernetes">Kubernetes Networking</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Secure SaaS&quot;}" href="/use-cases/secure-saas">Secure SaaS</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Secure AI Agent Connectivity&quot;}" href="/use-cases/secure-ai-agent-connectivity">Secure AI Agent Connectivity</a></li></ul></div></div></div></div></li><li><button type="button" class="flex items-center gap-[4px] rounded-md px-3 py-2.5 text-sm font-medium hover:bg-gray-200">Customers<svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M2.91675 7H11.0834" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round"></path><path d="M7 2.9165V11.0832" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round" style="opacity:1;transform:scaleY(1);transform-origin:center;transition:opacity 150ms ease-in-out, transform 150ms ease-in-out"></path></svg></button><div class="pointer-events-none absolute left-1/2 top-full z-[100] mt-3 -translate-x-1/2 pt-4 opacity-0"><div class="overflow-hidden rounded-2xl border border-gray-200 shadow-8"><div class="grid auto-cols-[268px] grid-flow-col gap-4 border-b border-gray-200 bg-white p-8 shadow-2"><div class="flex flex-col gap-4"><div class="flex items-center gap-2.5 px-3"><p class="whitespace-nowrap font-mdio text-[14px] font-medium uppercase !tracking-[0.6px] text-gray-700/80">Customer Stories</p><div class="h-px w-full bg-gray-200"></div></div><ul class="flex flex-col gap-1"><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Instacart&quot;}" href="/customers/instacart">Instacart</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Cribl&quot;}" href="/customers/cribl">Cribl</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Mercury&quot;}" href="/customers/mercury">Mercury</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;All Customer Stories&quot;}" href="/customers">All Customer Stories</a></li></ul></div></div></div></div></li><li><button type="button" class="flex items-center gap-[4px] rounded-md px-3 py-2.5 text-sm font-medium hover:bg-gray-200">Community<svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M2.91675 7H11.0834" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round"></path><path d="M7 2.9165V11.0832" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round" style="opacity:1;transform:scaleY(1);transform-origin:center;transition:opacity 150ms ease-in-out, transform 150ms ease-in-out"></path></svg></button><div class="pointer-events-none absolute left-1/2 top-full z-[100] mt-3 -translate-x-1/2 pt-4 opacity-0"><div class="overflow-hidden rounded-2xl border border-gray-200 shadow-8"><div class="grid auto-cols-[268px] grid-flow-col gap-4 border-b border-gray-200 bg-white p-8 shadow-2"><div class="flex flex-col gap-4"><div class="flex items-center gap-2.5 px-3"><p class="whitespace-nowrap font-mdio text-[14px] font-medium uppercase !tracking-[0.6px] text-gray-700/80">Join the Community</p><div class="h-px w-full bg-gray-200"></div></div><ul class="flex flex-col gap-1"><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;About Community&quot;}" href="/about-community">About Community</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Tailscale Insiders&quot;}" href="/insiders-program">Tailscale Insiders</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Community Projects&quot;}" href="/community/community-projects">Community Projects</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Bring Tailscale to Work&quot;}" href="/bring-tailscale-to-work">Bring Tailscale to Work</a></li></ul></div><div class="flex flex-col gap-4"><div class="flex items-center gap-2.5 px-3"><p class="whitespace-nowrap font-mdio text-[14px] font-medium uppercase !tracking-[0.6px] text-gray-700/80">Events</p><div class="h-px w-full bg-gray-200"></div></div><ul class="flex flex-col gap-1"><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Events&quot;}" href="/events">Events</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Webinars&quot;}" href="/webinars">Webinars</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;TailscaleUp&quot;}" href="/tailscaleup">TailscaleUp</a></li></ul></div><div class="flex flex-col gap-4"><div class="flex items-center gap-2.5 px-3"><p class="whitespace-nowrap font-mdio text-[14px] font-medium uppercase !tracking-[0.6px] text-gray-700/80">Learn more</p><div class="h-px w-full bg-gray-200"></div></div><ul class="flex flex-col gap-1"><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Docs&quot;}" href="/docs">Docs</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Blog&quot;}" href="/blog">Blog</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Changelog&quot;}" href="/changelog">Changelog</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Press&quot;}" href="/press">Press</a></li></ul></div></div><div class="bg-gray-50 p-8"><div class="flex items-center justify-between"><div class="flex flex-col gap-1"><p class="font-mdio text-[14px] font-medium uppercase !tracking-[0.6px] text-gray-700">Bring Tailscale to work</p><p class="text-base text-gray-700/80">Start with your own devices. Bring the same simplicity to your team.</p></div><a class="inline-flex h-9 shrink-0 items-center justify-center rounded-md border border-gray-800 bg-gray-800 px-3 text-base font-medium text-white shadow-2 transition-all duration-200 hover:opacity-80 hover:shadow-4" href="https://tailscale.com/bring-tailscale-to-work">Make the case</a></div></div></div></div></li><li><button type="button" class="flex items-center gap-[4px] rounded-md px-3 py-2.5 text-sm font-medium hover:bg-gray-200">Partnerships<svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M2.91675 7H11.0834" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round"></path><path d="M7 2.9165V11.0832" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round" style="opacity:1;transform:scaleY(1);transform-origin:center;transition:opacity 150ms ease-in-out, transform 150ms ease-in-out"></path></svg></button><div class="pointer-events-none absolute left-1/2 top-full z-[100] mt-3 -translate-x-1/2 pt-4 opacity-0"><div class="overflow-hidden rounded-2xl border border-gray-200 shadow-8"><div class="grid auto-cols-[268px] grid-flow-col gap-4 border-b border-gray-200 bg-white p-8 shadow-2"><div class="flex flex-col gap-4"><div class="flex items-center gap-2.5 px-3"><p class="whitespace-nowrap font-mdio text-[14px] font-medium uppercase !tracking-[0.6px] text-gray-700/80">Partner Opportunities</p><div class="h-px w-full bg-gray-200"></div></div><ul class="flex flex-col gap-1"><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Become a Partner&quot;}" href="/partnerships">Become a Partner</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Our Partners&quot;}" href="/partnerships/directory">Our Partners</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Integrations&quot;}" href="/integrations">Integrations</a></li><li><a class="block rounded-md p-3 text-base font-medium text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Contact Partnerships Team&quot;}" href="/contact/partnerships">Contact Partnerships Team</a></li></ul></div></div></div></div></li><li><a href="/pricing" class="flex items-center gap-4 rounded-md px-3 py-2.5 text-sm font-medium hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Pricing&quot;}">Pricing</a></li></ul></nav><nav class="hidden sm:ml-auto sm:block lg:ml-0"><ul class="flex items-center gap-x-[4px] font-medium"><li><a href="https://login.tailscale.com/welcome" class="block rounded-md px-3 py-2.5 text-sm hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Login&quot;}">Login</a></li><li><a class="inline-flex h-9 items-center justify-center rounded-[0.5rem] border border-gray-800 bg-gray-800 px-3 text-base font-medium text-white shadow-2 transition-all duration-200 hover:opacity-80 hover:shadow-4" href="https://login.tailscale.com/start">Get started - it’s free!</a></li></ul></nav><button type="button" class="h-[36px] rounded-[0.5rem] bg-gray-800 px-[5px] xl-small:hidden"><svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M4 12H20" stroke="#fff" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path><path d="M4 6H20" stroke="#fff" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path><path d="M4 18H20" stroke="#fff" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></button></div></div></header><div class="pointer-events-none absolute left-0 top-full w-full bg-white opacity-0 transition-opacity duration-150 md:hidden"><div class="bg-gray-700"><nav><ul class="flex justify-center gap-6"><li><a class="block py-2.5 font-mdio text-xs uppercase !tracking-[0.6px] text-gray-0/80 hover:underline" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Blog&quot;}" href="/blog">Blog</a></li><li><a class="block py-2.5 font-mdio text-xs uppercase !tracking-[0.6px] text-gray-0/80 hover:underline" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Docs&quot;}" href="/docs">Docs</a></li><li><a class="block py-2.5 font-mdio text-xs uppercase !tracking-[0.6px] text-gray-0/80 hover:underline" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Download&quot;}" href="/download">Download</a></li><li><a class="block py-2.5 font-mdio text-xs uppercase !tracking-[0.6px] text-gray-0/80 hover:underline" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Contact Sales&quot;}" href="/contact/sales">Contact Sales</a></li></ul></nav></div><nav class="max-h-[60vh] overflow-auto rounded-b-md p-6"><details class="group"><summary class="flex items-center justify-between rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] hover:bg-gray-200 group-open:bg-gray-200">Platform<span class="relative"><span class="group-open:hidden"><svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M2.91675 7H11.0834" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round"></path><path d="M7 2.9165V11.0832" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round" style="opacity:1;transform:scaleY(1);transform-origin:center;transition:opacity 150ms ease-in-out, transform 150ms ease-in-out"></path></svg></span><span class="hidden group-open:block"><svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M2.91675 7H11.0834" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round"></path><path d="M7 2.9165V11.0832" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round" style="opacity:0;transform:scaleY(0);transform-origin:center;transition:opacity 150ms ease-in-out, transform 150ms ease-in-out"></path></svg></span></span></summary><div class="mb-4 border-b border-gray-200 py-4"><div class="mb-6"><p class="mb-3 flex items-center gap-2.5 whitespace-nowrap px-2 font-mdio text-[12px] font-medium uppercase !tracking-[0.6px] text-gray-700/80 after:h-px after:w-full after:bg-gray-200 after:content-[&#x27;&#x27;]">Meet Tailscale</p><ul><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;How Tailscale Works&quot;}" href="/blog/how-tailscale-works">How Tailscale Works</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;WireGuard® for Enterprises&quot;}" href="/wireguard-vpn">WireGuard® for Enterprises</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Features&quot;}" href="/features">Features</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Integrations&quot;}" href="/integrations">Integrations</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Docs&quot;}" href="/docs">Docs</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Download&quot;}" href="/download">Download</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Compare Tailscale&quot;}" href="/compare">Compare Tailscale</a></li></ul></div><div class="mb-6"><p class="mb-3 flex items-center gap-2.5 whitespace-nowrap px-2 font-mdio text-[12px] font-medium uppercase !tracking-[0.6px] text-gray-700/80 after:h-px after:w-full after:bg-gray-200 after:content-[&#x27;&#x27;]">Products</p><ul><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Business VPN&quot;}" href="/use-cases/business-vpn">Business VPN</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Tailscale PAM&quot;}" href="/use-cases/pam">Tailscale PAM</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;CI/CD Connectivity&quot;}" href="/use-cases/ci-cd">CI/CD Connectivity</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Secure Access to AI&quot;}" href="/use-cases/securing-ai">Secure Access to AI</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Workload Connectivity&quot;}" href="/use-cases/workload-connectivity">Workload Connectivity</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Edge &amp; IoT&quot;}" href="/use-cases/edge-iot">Edge &amp; IoT</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Homelab&quot;}" href="/use-cases/homelab">Homelab</a></li></ul></div></div></details><details class="group"><summary class="flex items-center justify-between rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] hover:bg-gray-200 group-open:bg-gray-200">Solutions<span class="relative"><span class="group-open:hidden"><svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M2.91675 7H11.0834" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round"></path><path d="M7 2.9165V11.0832" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round" style="opacity:1;transform:scaleY(1);transform-origin:center;transition:opacity 150ms ease-in-out, transform 150ms ease-in-out"></path></svg></span><span class="hidden group-open:block"><svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M2.91675 7H11.0834" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round"></path><path d="M7 2.9165V11.0832" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round" style="opacity:0;transform:scaleY(0);transform-origin:center;transition:opacity 150ms ease-in-out, transform 150ms ease-in-out"></path></svg></span></span></summary><div class="mb-4 border-b border-gray-200 py-4"><div class="mb-6"><p class="mb-3 flex items-center gap-2.5 whitespace-nowrap px-2 font-mdio text-[12px] font-medium uppercase !tracking-[0.6px] text-gray-700/80 after:h-px after:w-full after:bg-gray-200 after:content-[&#x27;&#x27;]">Solutions</p><ul><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Cloud Connectivity&quot;}" href="/use-cases/cloud-connectivity">Cloud Connectivity</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Infrastructure Access&quot;}" href="/use-cases/infrastructure-access">Infrastructure Access</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Zero Trust Networking&quot;}" href="/use-cases/zero-trust-networking">Zero Trust Networking</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Remote Access&quot;}" href="/use-cases/remote-access">Remote Access</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Kubernetes Networking&quot;}" href="/use-cases/kubernetes">Kubernetes Networking</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Secure SaaS&quot;}" href="/use-cases/secure-saas">Secure SaaS</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Secure AI Agent Connectivity&quot;}" href="/use-cases/secure-ai-agent-connectivity">Secure AI Agent Connectivity</a></li></ul></div></div></details><details class="group"><summary class="flex items-center justify-between rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] hover:bg-gray-200 group-open:bg-gray-200">Customers<span class="relative"><span class="group-open:hidden"><svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M2.91675 7H11.0834" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round"></path><path d="M7 2.9165V11.0832" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round" style="opacity:1;transform:scaleY(1);transform-origin:center;transition:opacity 150ms ease-in-out, transform 150ms ease-in-out"></path></svg></span><span class="hidden group-open:block"><svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M2.91675 7H11.0834" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round"></path><path d="M7 2.9165V11.0832" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round" style="opacity:0;transform:scaleY(0);transform-origin:center;transition:opacity 150ms ease-in-out, transform 150ms ease-in-out"></path></svg></span></span></summary><div class="mb-4 border-b border-gray-200 py-4"><div class="mb-6"><p class="mb-3 flex items-center gap-2.5 whitespace-nowrap px-2 font-mdio text-[12px] font-medium uppercase !tracking-[0.6px] text-gray-700/80 after:h-px after:w-full after:bg-gray-200 after:content-[&#x27;&#x27;]">Customer Stories</p><ul><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Instacart&quot;}" href="/customers/instacart">Instacart</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Cribl&quot;}" href="/customers/cribl">Cribl</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Mercury&quot;}" href="/customers/mercury">Mercury</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;All Customer Stories&quot;}" href="/customers">All Customer Stories</a></li></ul></div></div></details><details class="group"><summary class="flex items-center justify-between rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] hover:bg-gray-200 group-open:bg-gray-200">Community<span class="relative"><span class="group-open:hidden"><svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M2.91675 7H11.0834" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round"></path><path d="M7 2.9165V11.0832" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round" style="opacity:1;transform:scaleY(1);transform-origin:center;transition:opacity 150ms ease-in-out, transform 150ms ease-in-out"></path></svg></span><span class="hidden group-open:block"><svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M2.91675 7H11.0834" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round"></path><path d="M7 2.9165V11.0832" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round" style="opacity:0;transform:scaleY(0);transform-origin:center;transition:opacity 150ms ease-in-out, transform 150ms ease-in-out"></path></svg></span></span></summary><div class="mb-4 border-b border-gray-200 py-4"><div class="mb-6"><p class="mb-3 flex items-center gap-2.5 whitespace-nowrap px-2 font-mdio text-[12px] font-medium uppercase !tracking-[0.6px] text-gray-700/80 after:h-px after:w-full after:bg-gray-200 after:content-[&#x27;&#x27;]">Join the Community</p><ul><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;About Community&quot;}" href="/about-community">About Community</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Tailscale Insiders&quot;}" href="/insiders-program">Tailscale Insiders</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Community Projects&quot;}" href="/community/community-projects">Community Projects</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Bring Tailscale to Work&quot;}" href="/bring-tailscale-to-work">Bring Tailscale to Work</a></li></ul></div><div class="mb-6"><p class="mb-3 flex items-center gap-2.5 whitespace-nowrap px-2 font-mdio text-[12px] font-medium uppercase !tracking-[0.6px] text-gray-700/80 after:h-px after:w-full after:bg-gray-200 after:content-[&#x27;&#x27;]">Events</p><ul><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Events&quot;}" href="/events">Events</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Webinars&quot;}" href="/webinars">Webinars</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;TailscaleUp&quot;}" href="/tailscaleup">TailscaleUp</a></li></ul></div><div class="mb-6"><p class="mb-3 flex items-center gap-2.5 whitespace-nowrap px-2 font-mdio text-[12px] font-medium uppercase !tracking-[0.6px] text-gray-700/80 after:h-px after:w-full after:bg-gray-200 after:content-[&#x27;&#x27;]">Learn more</p><ul><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Docs&quot;}" href="/docs">Docs</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Blog&quot;}" href="/blog">Blog</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Changelog&quot;}" href="/changelog">Changelog</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Press&quot;}" href="/press">Press</a></li></ul></div></div></details><details class="group"><summary class="flex items-center justify-between rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] hover:bg-gray-200 group-open:bg-gray-200">Partnerships<span class="relative"><span class="group-open:hidden"><svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M2.91675 7H11.0834" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round"></path><path d="M7 2.9165V11.0832" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round" style="opacity:1;transform:scaleY(1);transform-origin:center;transition:opacity 150ms ease-in-out, transform 150ms ease-in-out"></path></svg></span><span class="hidden group-open:block"><svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M2.91675 7H11.0834" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round"></path><path d="M7 2.9165V11.0832" stroke="#2E2D2D" stroke-width="1.16667" stroke-linecap="round" stroke-linejoin="round" style="opacity:0;transform:scaleY(0);transform-origin:center;transition:opacity 150ms ease-in-out, transform 150ms ease-in-out"></path></svg></span></span></summary><div class="mb-4 border-b border-gray-200 py-4"><div class="mb-6"><p class="mb-3 flex items-center gap-2.5 whitespace-nowrap px-2 font-mdio text-[12px] font-medium uppercase !tracking-[0.6px] text-gray-700/80 after:h-px after:w-full after:bg-gray-200 after:content-[&#x27;&#x27;]">Partner Opportunities</p><ul><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Become a Partner&quot;}" href="/partnerships">Become a Partner</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Our Partners&quot;}" href="/partnerships/directory">Our Partners</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Integrations&quot;}" href="/integrations">Integrations</a></li><li class="space-y-1"><a class="block rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] text-gray-700 hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Contact Partnerships Team&quot;}" href="/contact/partnerships">Contact Partnerships Team</a></li></ul></div></div></details><a class="flex justify-between rounded-md px-2 py-3 text-[14px] font-medium tracking-[-0.14px] hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Pricing&quot;}" href="/pricing">Pricing</a></nav><nav class="mx-6 border-t border-gray-200 py-6"><ul class="flex items-center justify-between gap-x-4 font-medium"><li><a href="https://login.tailscale.com/welcome" class="block rounded-md px-3 py-2.5 text-sm hover:bg-gray-200" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Login&quot;}">Login</a></li><li><a class="inline-flex h-9 items-center justify-center rounded-md border border-gray-800 bg-gray-800 px-3 text-base font-medium text-white shadow-2 transition-all duration-200 hover:opacity-80 hover:shadow-4" href="https://login.tailscale.com/start">Get started - it’s free!</a></li></ul></nav></div></div><div class="website-nav-backdrop z-100 pointer-events-none fixed left-0 top-0 h-full w-full bg-gray-1000/25 opacity-0 transition-opacity duration-300"></div><main><section class="pt-20 md:pt-[110px]"><script type="application/ld+json">{"@context":"https://schema.org","@type":"BlogPosting","image":"https://cdn.sanity.io/images/w77i7m8x/production/6ca0577c6bb689a7af13ec1407ebc8afd2de8136-4608x2372.png","url":"https://tailscale.com/blog/sqlite-wal-reset-bug","headline":"How we tracked down a 16-year-old SQLite bug","datePublished":"2026-08-12T14:00:00.000Z","description":"SQLite corruption? Sorted.","abstract":"SQLite corruption? Sorted.","author":[{"@type":"Person","name":"Alex Chan"}],"articleBody":null}</script><div class="container-new justify-start gap-8 lg:flex"><div class="lg:w-max lg:max-w-3xl"><div class="md:px-8"><div class="flex items-center text-gray-500"><a type="button" aria-label="Go back" class="t-18 flex items-center gap-[5px] transition-colors duration-200 hover:text-gray-1000" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Go Back&quot;}" href="/blog"><svg width="12" height="12" viewBox="0 0 12 12" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M7.98047 2.21094L4.01953 6.17188L7.98047 10.1328" stroke="url(#paint0_radial_243_61489)" stroke-width="1.2" stroke-miterlimit="10" stroke-linecap="square"></path><defs><radialGradient id="paint0_radial_243_61489" cx="0" cy="0" r="1" gradientUnits="userSpaceOnUse" gradientTransform="translate(7.98047 7.22813) rotate(-165.069) scale(4.09935 7.79047)"><stop stop-color="#373636"></stop><stop offset="1" stop-color="#212121"></stop></radialGradient></defs></svg><span>Blog</span></a><span class="mx-3">|</span><span class="inline-flex h-fit items-center gap-1 rounded-brand-sm p-brand-8 text-sm font-medium capitalize" style="color:#571F0D;background:#F8E5B9">insights</span><span class="ml-3">August 12, 2026</span></div><h1 class="h1-new mb-8 mt-7 text-heading-black">How we tracked down a 16-year-old SQLite bug</h1></div><div class="mb-4 flex flex-col gap-6 border-gray-300 md:flex-row md:p-8 lg:hidden"><div class="flex flex-1 flex-grow flex-col items-start gap-2 px-3"><p class="body-new mb-1 font-medium">Author</p><div class="flex items-center gap-1 text-sm font-normal leading-[0.875rem] tracking-[-0.018rem]"><img alt="Alex Chan" loading="lazy" width="1729" height="1729" decoding="async" data-nimg="1" class="h-8 w-8 text-xs border-white rounded-full border-[1.5px]" style="color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url(&quot;data:image/svg+xml;charset=utf-8,%3Csvg xmlns=&#x27;http://www.w3.org/2000/svg&#x27; viewBox=&#x27;0 0 1729 1729&#x27;%3E%3Cfilter id=&#x27;b&#x27; color-interpolation-filters=&#x27;sRGB&#x27;%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3CfeColorMatrix values=&#x27;1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1&#x27; result=&#x27;s&#x27;/%3E%3CfeFlood x=&#x27;0&#x27; y=&#x27;0&#x27; width=&#x27;100%25&#x27; height=&#x27;100%25&#x27;/%3E%3CfeComposite operator=&#x27;out&#x27; in=&#x27;s&#x27;/%3E%3CfeComposite in2=&#x27;SourceGraphic&#x27;/%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3C/filter%3E%3Cimage width=&#x27;100%25&#x27; height=&#x27;100%25&#x27; x=&#x27;0&#x27; y=&#x27;0&#x27; preserveAspectRatio=&#x27;none&#x27; style=&#x27;filter: url(%23b);&#x27; href=&#x27;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/wcAAwAB/ucNC4UAAAAASUVORK5CYII=&#x27;/%3E%3C/svg%3E&quot;)" srcSet="https://cdn.sanity.io/images/w77i7m8x/production/7c909f6cf559dc9719380a3557fb7ff48e5db08a-1729x1729.jpg?w=1920&amp;q=75&amp;fit=clip&amp;auto=format 1x, https://cdn.sanity.io/images/w77i7m8x/production/7c909f6cf559dc9719380a3557fb7ff48e5db08a-1729x1729.jpg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format 2x" src="https://cdn.sanity.io/images/w77i7m8x/production/7c909f6cf559dc9719380a3557fb7ff48e5db08a-1729x1729.jpg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format"/>Alex Chan</div></div></div><img alt="Light orange and dark yellow shapes, like ovals, squares, circles, and quarter-circles, against a pale yellow background." loading="lazy" width="4608" height="2372" decoding="async" data-nimg="1" class="mb-6 w-full rounded-2xl md:mb-12" style="color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url(&quot;data:image/svg+xml;charset=utf-8,%3Csvg xmlns=&#x27;http://www.w3.org/2000/svg&#x27; viewBox=&#x27;0 0 4608 2372&#x27;%3E%3Cfilter id=&#x27;b&#x27; color-interpolation-filters=&#x27;sRGB&#x27;%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3CfeColorMatrix values=&#x27;1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1&#x27; result=&#x27;s&#x27;/%3E%3CfeFlood x=&#x27;0&#x27; y=&#x27;0&#x27; width=&#x27;100%25&#x27; height=&#x27;100%25&#x27;/%3E%3CfeComposite operator=&#x27;out&#x27; in=&#x27;s&#x27;/%3E%3CfeComposite in2=&#x27;SourceGraphic&#x27;/%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3C/filter%3E%3Cimage width=&#x27;100%25&#x27; height=&#x27;100%25&#x27; x=&#x27;0&#x27; y=&#x27;0&#x27; preserveAspectRatio=&#x27;none&#x27; style=&#x27;filter: url(%23b);&#x27; href=&#x27;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/wcAAwAB/ucNC4UAAAAASUVORK5CYII=&#x27;/%3E%3C/svg%3E&quot;)" srcSet="https://cdn.sanity.io/images/w77i7m8x/production/6ca0577c6bb689a7af13ec1407ebc8afd2de8136-4608x2372.png?w=3840&amp;q=75&amp;fit=clip&amp;auto=format 1x" src="https://cdn.sanity.io/images/w77i7m8x/production/6ca0577c6bb689a7af13ec1407ebc8afd2de8136-4608x2372.png?w=3840&amp;q=75&amp;fit=clip&amp;auto=format"/><div><div class="blog-content-prose Markdown-new"><p>At the end of last year, our uptime was <a target="" rel="noreferrer" href="https://tailscale.com/blog/hypergrowth-isnt-always-easy">pretty shaky</a>. You can see this trend on <a target="" rel="noreferrer" href="https://status.tailscale.com/history">our status page</a>, and that instability continued into the new year. Many of these outages were caused by a single bug, deep in <a target="" rel="noreferrer" href="https://sqlite.org/index.html">SQLite</a>. It took months of intense forensics to track it down.</p><p>Now we’re in summer, we’re confident that we’ve found the bug, that we understand it—and more importantly, that we’ve fixed it.</p><p>We know our customers expect Tailscale to be a reliable service, and for several months we didn’t live up to that promise. That’s disruptive, and we’re sorry. We’re publishing this blog post to explain what went wrong, how we responded, and how we ultimately helped to uncover a long-standing bug in the heart of the SQLite database.</p><h2 id="tailscales-database-architecture"><a class="header-link group select-none !no-underline" href="#tailscales-database-architecture">Tailscale’s database architecture<svg class="ml-2 inline w-3.5 opacity-0 transition-all duration-200 group-hover:opacity-100" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M15 7h3a5 5 0 0 1 5 5 5 5 0 0 1-5 5h-3m-6 0H6a5 5 0 0 1-5-5 5 5 0 0 1 5-5h3"></path><line x1="8" y1="12" x2="16" y2="12"></line></svg></a></h2><p>While our clients interact with our <a target="" rel="noreferrer" href="https://tailscale.com/docs/concepts/control-data-planes">control plane</a> as a single public endpoint (<a target="" rel="noreferrer" href="http://controlplane.tailscale.com">controlplane.tailscale.com</a>), internally, our control plane is split into a series of coordination servers (or “shards”). Each tailnet lives on one internal shard at a time, but can migrate seamlessly from one to another. These shards are an internal implementation detail: you don’t know what shard your tailnet is on, and you never need to.</p><p>Each shard has an SQLite database that holds all the information about the tailnets on that shard. A single Go process exclusively accesses that database, and serves the control plane for those tailnets. This single-writer design is exactly how SQLite is meant to be used.</p><figure id="" class="mb-6 scroll-mt-20 text-center"><img _type="asset" video="[object Object]" alt="Architecture diagram illustrating how the Tailscale control plane is made of isolated shards, each of which has an individual SQLite database." loading="lazy" width="1920" height="1080" decoding="async" data-nimg="1" class="w-full will-change-transform md:block mx-auto" style="color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url(&quot;data:image/svg+xml;charset=utf-8,%3Csvg xmlns=&#x27;http://www.w3.org/2000/svg&#x27; viewBox=&#x27;0 0 1920 1080&#x27;%3E%3Cfilter id=&#x27;b&#x27; color-interpolation-filters=&#x27;sRGB&#x27;%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3CfeColorMatrix values=&#x27;1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1&#x27; result=&#x27;s&#x27;/%3E%3CfeFlood x=&#x27;0&#x27; y=&#x27;0&#x27; width=&#x27;100%25&#x27; height=&#x27;100%25&#x27;/%3E%3CfeComposite operator=&#x27;out&#x27; in=&#x27;s&#x27;/%3E%3CfeComposite in2=&#x27;SourceGraphic&#x27;/%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3C/filter%3E%3Cimage width=&#x27;100%25&#x27; height=&#x27;100%25&#x27; x=&#x27;0&#x27; y=&#x27;0&#x27; preserveAspectRatio=&#x27;none&#x27; style=&#x27;filter: url(%23b);&#x27; href=&#x27;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/wcAAwAB/ucNC4UAAAAASUVORK5CYII=&#x27;/%3E%3C/svg%3E&quot;)" srcSet="https://cdn.sanity.io/images/w77i7m8x/production/6db766d5b5c004085237e7498faf57a0e8d3205a-1920x1080.svg?w=1920&amp;q=75&amp;fit=clip&amp;auto=format 1x, https://cdn.sanity.io/images/w77i7m8x/production/6db766d5b5c004085237e7498faf57a0e8d3205a-1920x1080.svg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format 2x" src="https://cdn.sanity.io/images/w77i7m8x/production/6db766d5b5c004085237e7498faf57a0e8d3205a-1920x1080.svg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format"/></figure><p>We’ve used SQLite as our primary database <a target="" rel="noreferrer" href="https://tailscale.com/blog/database-for-2022">since 2022</a>, and we chose it because it&#x27;s well-known, reliable, and widely used. SQLite is <a target="" rel="noreferrer" href="https://mcfunley.com/choose-boring-technology">“boring technology”</a>—in a good way. Many companies use SQLite in much larger deployments without issue, and we expected the same stress-free usage.</p><p>In our current backup pipeline, we take a complete snapshot of the database every few minutes, then upload the entire SQLite file to an S3 bucket. We’d been running this setup without incident since early 2023.</p><p>Fast forward to August last year, when a data pipeline that reads those S3 backups reported an error in one of our databases. We ran SQLite’s <code>PRAGMA integrity_check</code> command against the backup, and found it was indeed corrupted. SQLite corruption <a target="" rel="noreferrer" href="https://www.sqlite.org/howtocorrupt.html">is possible</a>, but it’s highly unusual and not something you should encounter in normal operation. We repaired the affected database, and investigated the cause, but to no avail.</p><p>When operating at scale, even rare events can occur with some frequency, so we should have been unsurprised when it happened again—and again, and again, and again. In total, we faced 19 separate instances of database corruption over six months before we finally resolved the underlying bug.</p><p>When you hear the phrase “database corruption”, it’s natural to worry about data loss. Because our control plane only handles configuration data, these databases contain metadata about your tailnet and devices, but never your private encryption keys or network traffic. In the earliest incidents, the recovery process meant a handful of newly added devices or configuration changes didn’t persist, and a small amount of metadata had to be re-entered.</p><figure id="" class="mb-6 scroll-mt-20 text-center"><img _type="asset" video="[object Object]" alt="Architecture diagram illustrating how the Tailscale control plane copes with a database issue on a single shard. When the SQLite database has database corruption, it affects traffic on that shard, but other shards are unaffected." loading="lazy" width="1920" height="1080" decoding="async" data-nimg="1" class="w-full will-change-transform md:block mx-auto" style="color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url(&quot;data:image/svg+xml;charset=utf-8,%3Csvg xmlns=&#x27;http://www.w3.org/2000/svg&#x27; viewBox=&#x27;0 0 1920 1080&#x27;%3E%3Cfilter id=&#x27;b&#x27; color-interpolation-filters=&#x27;sRGB&#x27;%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3CfeColorMatrix values=&#x27;1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1&#x27; result=&#x27;s&#x27;/%3E%3CfeFlood x=&#x27;0&#x27; y=&#x27;0&#x27; width=&#x27;100%25&#x27; height=&#x27;100%25&#x27;/%3E%3CfeComposite operator=&#x27;out&#x27; in=&#x27;s&#x27;/%3E%3CfeComposite in2=&#x27;SourceGraphic&#x27;/%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3C/filter%3E%3Cimage width=&#x27;100%25&#x27; height=&#x27;100%25&#x27; x=&#x27;0&#x27; y=&#x27;0&#x27; preserveAspectRatio=&#x27;none&#x27; style=&#x27;filter: url(%23b);&#x27; href=&#x27;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/wcAAwAB/ucNC4UAAAAASUVORK5CYII=&#x27;/%3E%3C/svg%3E&quot;)" srcSet="https://cdn.sanity.io/images/w77i7m8x/production/eec436b49f783f30c56edb32b6eeacff62ebfdf4-1920x1080.svg?w=1920&amp;q=75&amp;fit=clip&amp;auto=format 1x, https://cdn.sanity.io/images/w77i7m8x/production/eec436b49f783f30c56edb32b6eeacff62ebfdf4-1920x1080.svg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format 2x" src="https://cdn.sanity.io/images/w77i7m8x/production/eec436b49f783f30c56edb32b6eeacff62ebfdf4-1920x1080.svg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format"/></figure><p>Whenever corruption occurred, we had to stop the control plane process on the shard while we repaired or restored the database. This was painful for tailnets on that shard, because their entire control plane disappeared during that recovery window. In the early incidents, that downtime was over an hour, but we gradually sped up the recovery process over subsequent incidents.</p><p>Each tailnet is a mesh network, where devices make peer-to-peer WireGuard® connections to each other. When a device joins the tailnet, it has to get a list of other devices from the control plane before it can establish new connections—so if a device came online during the SQLite downtime, it couldn’t connect. While the database was being repaired, devices already online remained connected to each other, but they couldn’t learn about changes to the network. Those tailnets also temporarily lost access to the web-based admin console and the Tailscale API.</p><p>There’s also a broader impact on trust. We post a global incident on <a target="" rel="noreferrer" href="https://status.tailscale.com/">our status page</a> even when only a small number of tailnets are affected. Many people saw a status page event for an incident that didn’t affect them. Indeed, the majority of shards and tailnets were never involved in a database corruption incident! Nonetheless, repeated downtime erodes trust, whether or not you’re directly affected.</p><p>From the very first instance of corruption, we knew this was a serious threat to our reliability, and we threw a lot of engineering time at the problem—but the fix wasn’t easy.</p><h2 id="trying-to-find-the-fault"><a class="header-link group select-none !no-underline" href="#trying-to-find-the-fault">Trying to find the fault<svg class="ml-2 inline w-3.5 opacity-0 transition-all duration-200 group-hover:opacity-100" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M15 7h3a5 5 0 0 1 5 5 5 5 0 0 1-5 5h-3m-6 0H6a5 5 0 0 1-5-5 5 5 0 0 1 5-5h3"></path><line x1="8" y1="12" x2="16" y2="12"></line></svg></a></h2><p>This bug resisted all our initial attempts to find it.</p><p>We looked at recent changes, but there weren’t any that seemed relevant. Nobody had been working on our low-level code that interacts with SQLite, because it had all been written years ago and presented no issues up until that point. We re-reviewed all of that code with a fine-toothed comb to look for previously missed bugs, but we didn’t find anything that would cause the corruption we were seeing.</p><p>We looked for common factors between corruption incidents, but we couldn’t find any. It wasn’t tied to a single shard, or customer, or tailnet feature, or time of day, or load level. We were at a loss for what might be triggering the behaviour.</p><p>This lack of reliable trigger conditions meant we couldn’t reproduce the bug synthetically. Instead, we had to rely on deploying passive, forensic telemetry in our live environment to catch the corruption red-handed. Gathering live diagnostics for a database issue is the last thing we wanted to do, but we had no choice.</p><p>As an additional complication, the corruption didn’t occur on a regular schedule. Sometimes incidents would be hours apart, other times weeks. This made it difficult to predict progress or plan further work, because we were never sure when we’d get our next diagnostic dump. We had a six-week period between October and December when there were no corruption incidents, before they returned as an unwelcome Christmas present.</p><p>Because this wouldn’t be a quick or easy fix, we reached out to the SQLite developers for a <a target="" rel="noreferrer" href="https://sqlite.org/prosupport.html">professional support contract</a>. This was a great decision. It gave us direct access to their deep expertise and experience, and we had many detailed technical conversations about our architecture and our incidents.</p><p>Between Tailscale engineering and the SQLite core developers, we mapped out several theories for what might be causing the corruption—including <a target="" rel="noreferrer" href="https://www.sqlite.org/howtocorrupt.html#_posix_advisory_locks_canceled_by_a_separate_thread_doing_close_">broken POSIX locks on close()</a>, mismanaging memory owned by SQLite, or accidentally using SQLite from multiple threads <a target="" rel="noreferrer" href="https://sqlite.org/compile.html#threadsafe">while disabling thread safety</a>. After every incident, we gathered more data, added more diagnostics, and systematically ruled out these theories. We were gradually converging on the true bug.</p><h2 id="the-transactions-that-didnt-bark"><a class="header-link group select-none !no-underline" href="#the-transactions-that-didnt-bark">The transactions that didn’t bark<svg class="ml-2 inline w-3.5 opacity-0 transition-all duration-200 group-hover:opacity-100" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M15 7h3a5 5 0 0 1 5 5 5 5 0 0 1-5 5h-3m-6 0H6a5 5 0 0 1-5-5 5 5 0 0 1 5-5h3"></path><line x1="8" y1="12" x2="16" y2="12"></line></svg></a></h2><p>While we were investigating the root cause, we still had a live platform to run. We took aggressive steps to automate recovery and minimize downtime:</p><ul><li>Configuring our control plane shards to hard-stop immediately upon encountering corruption</li><li>Deploying an automated backup monitor that continuously ran <code>PRAGMA integrity_check</code> over our backups</li><li>Improving our runbooks and on-call training</li></ul><p>These efforts cut our response time to under an hour—and then we discovered an unexpected clue.</p><p>We wanted a way to restore service that didn’t involve rolling back to the last known-good backup (which would lose a lot of data) or repairing the known-corrupted database (which was potentially risky).</p><p>To do this, we built a transaction logging pipeline. We streamed every SQL statement that modified the database to a separate log file. Because SQLite is <a target="" rel="noreferrer" href="https://sqlite.org/faq.html#:~:text=only%20one%20process%20can%20be%20making%20changes%20to%20the%20database%20at%20any%20moment%20in%20time">a single-writer database</a> with <a target="" rel="noreferrer" href="https://www.sqlite.org/isolation.html">serialisable transactions</a>, our transaction history was completely linear and deterministic. (This wouldn’t be true in a multi-writer database like Postgres or MySQL.) Replaying those transactions against the latest known-good backup should restore the database to its most recent state, safely bypassing the corruption.</p><figure id="" class="mb-6 scroll-mt-20 text-center"><img _type="asset" video="[object Object]" alt="Diagram illustrating how the changes between two backups can be reconstructed by replaying the transactions that occurred between them." loading="lazy" width="1920" height="1080" decoding="async" data-nimg="1" class="w-full will-change-transform md:block mx-auto" style="color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url(&quot;data:image/svg+xml;charset=utf-8,%3Csvg xmlns=&#x27;http://www.w3.org/2000/svg&#x27; viewBox=&#x27;0 0 1920 1080&#x27;%3E%3Cfilter id=&#x27;b&#x27; color-interpolation-filters=&#x27;sRGB&#x27;%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3CfeColorMatrix values=&#x27;1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1&#x27; result=&#x27;s&#x27;/%3E%3CfeFlood x=&#x27;0&#x27; y=&#x27;0&#x27; width=&#x27;100%25&#x27; height=&#x27;100%25&#x27;/%3E%3CfeComposite operator=&#x27;out&#x27; in=&#x27;s&#x27;/%3E%3CfeComposite in2=&#x27;SourceGraphic&#x27;/%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3C/filter%3E%3Cimage width=&#x27;100%25&#x27; height=&#x27;100%25&#x27; x=&#x27;0&#x27; y=&#x27;0&#x27; preserveAspectRatio=&#x27;none&#x27; style=&#x27;filter: url(%23b);&#x27; href=&#x27;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/wcAAwAB/ucNC4UAAAAASUVORK5CYII=&#x27;/%3E%3C/svg%3E&quot;)" srcSet="https://cdn.sanity.io/images/w77i7m8x/production/34832281499b7d1388317372478dc0001770b16e-1920x1080.svg?w=1920&amp;q=75&amp;fit=clip&amp;auto=format 1x, https://cdn.sanity.io/images/w77i7m8x/production/34832281499b7d1388317372478dc0001770b16e-1920x1080.svg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format 2x" src="https://cdn.sanity.io/images/w77i7m8x/production/34832281499b7d1388317372478dc0001770b16e-1920x1080.svg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format"/></figure><p>This pipeline worked, but then it did something even better: it gave us a clue.</p><p>In two incidents, our transaction logs failed to replay cleanly. Upon closer inspection, we discovered that data written and committed by one transaction was inexplicably invisible to later transactions. A write had vanished into thin air without raising an error. That should be impossible!</p><h2 id="the-writing-on-the-wal"><a class="header-link group select-none !no-underline" href="#the-writing-on-the-wal">The writing on the WAL<svg class="ml-2 inline w-3.5 opacity-0 transition-all duration-200 group-hover:opacity-100" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M15 7h3a5 5 0 0 1 5 5 5 5 0 0 1-5 5h-3m-6 0H6a5 5 0 0 1-5-5 5 5 0 0 1 5-5h3"></path><line x1="8" y1="12" x2="16" y2="12"></line></svg></a></h2><p>As these incidents were ongoing, the SQLite developers had been developing a new debugging tool. For a while, we’d suspected that the bug was somewhere in the checkpoint process. They were building a new tool to give better visibility into what was happening during checkpoints.</p><p>To understand what this tool found, we need to briefly explain how SQLite checkpoints work.</p><p>A SQLite database is made of a series of “pages”, tiny blocks of information. When you update the database, some of those pages need to be replaced with new pages with the updated information.</p><p>For better performance and greater concurrency, we run SQLite with Write-Ahead Logging, which means new pages aren&#x27;t written directly to the database file. Instead, they’re written to the &quot;write-ahead log&quot; or &quot;WAL file&quot;.</p><figure id="" class="mb-6 scroll-mt-20 text-center"><img _type="asset" video="[object Object]" alt="Architecture diagram illustrating the difference between the database file and the write-ahead log (also known as the “WAL file”). Both are made of individual “pages”, and new pages are written to the WAL file first." loading="lazy" width="1920" height="1080" decoding="async" data-nimg="1" class="w-full will-change-transform md:block mx-auto" style="color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url(&quot;data:image/svg+xml;charset=utf-8,%3Csvg xmlns=&#x27;http://www.w3.org/2000/svg&#x27; viewBox=&#x27;0 0 1920 1080&#x27;%3E%3Cfilter id=&#x27;b&#x27; color-interpolation-filters=&#x27;sRGB&#x27;%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3CfeColorMatrix values=&#x27;1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1&#x27; result=&#x27;s&#x27;/%3E%3CfeFlood x=&#x27;0&#x27; y=&#x27;0&#x27; width=&#x27;100%25&#x27; height=&#x27;100%25&#x27;/%3E%3CfeComposite operator=&#x27;out&#x27; in=&#x27;s&#x27;/%3E%3CfeComposite in2=&#x27;SourceGraphic&#x27;/%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3C/filter%3E%3Cimage width=&#x27;100%25&#x27; height=&#x27;100%25&#x27; x=&#x27;0&#x27; y=&#x27;0&#x27; preserveAspectRatio=&#x27;none&#x27; style=&#x27;filter: url(%23b);&#x27; href=&#x27;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/wcAAwAB/ucNC4UAAAAASUVORK5CYII=&#x27;/%3E%3C/svg%3E&quot;)" srcSet="https://cdn.sanity.io/images/w77i7m8x/production/e80993ba5a6df5095f6cad9649545d52c61bd4c6-1920x1080.svg?w=1920&amp;q=75&amp;fit=clip&amp;auto=format 1x, https://cdn.sanity.io/images/w77i7m8x/production/e80993ba5a6df5095f6cad9649545d52c61bd4c6-1920x1080.svg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format 2x" src="https://cdn.sanity.io/images/w77i7m8x/production/e80993ba5a6df5095f6cad9649545d52c61bd4c6-1920x1080.svg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format"/></figure><p>New pages can&#x27;t be written to the WAL file indefinitely; at some point they have to be copied back to the main database file. This process is called “checkpointing”.</p><figure id="" class="mb-6 scroll-mt-20 text-center"><img _type="asset" video="[object Object]" alt="Architecture diagram illustrating the SQLite checkpoint procedure. Pages in the WAL file are copied back into the database file. New pages can replace existing pages anywhere in the database file, or be appended to the end of the file." loading="lazy" width="1920" height="1080" decoding="async" data-nimg="1" class="w-full will-change-transform md:block mx-auto" style="color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url(&quot;data:image/svg+xml;charset=utf-8,%3Csvg xmlns=&#x27;http://www.w3.org/2000/svg&#x27; viewBox=&#x27;0 0 1920 1080&#x27;%3E%3Cfilter id=&#x27;b&#x27; color-interpolation-filters=&#x27;sRGB&#x27;%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3CfeColorMatrix values=&#x27;1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1&#x27; result=&#x27;s&#x27;/%3E%3CfeFlood x=&#x27;0&#x27; y=&#x27;0&#x27; width=&#x27;100%25&#x27; height=&#x27;100%25&#x27;/%3E%3CfeComposite operator=&#x27;out&#x27; in=&#x27;s&#x27;/%3E%3CfeComposite in2=&#x27;SourceGraphic&#x27;/%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3C/filter%3E%3Cimage width=&#x27;100%25&#x27; height=&#x27;100%25&#x27; x=&#x27;0&#x27; y=&#x27;0&#x27; preserveAspectRatio=&#x27;none&#x27; style=&#x27;filter: url(%23b);&#x27; href=&#x27;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/wcAAwAB/ucNC4UAAAAASUVORK5CYII=&#x27;/%3E%3C/svg%3E&quot;)" srcSet="https://cdn.sanity.io/images/w77i7m8x/production/9499bb50a8a15bcf41faed009237de042cf3969e-1920x1080.svg?w=1920&amp;q=75&amp;fit=clip&amp;auto=format 1x, https://cdn.sanity.io/images/w77i7m8x/production/9499bb50a8a15bcf41faed009237de042cf3969e-1920x1080.svg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format 2x" src="https://cdn.sanity.io/images/w77i7m8x/production/9499bb50a8a15bcf41faed009237de042cf3969e-1920x1080.svg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format"/></figure><p>In most deployments, SQLite itself decides when to do a checkpoint, and the process is invisible to the end user and developer. In our control plane, we take manual control of the checkpoint process so we can run fast and consistent backups. This non-standard approach seemed suspicious as we steadily eliminated potential causes.</p><p>One clue was that during corruption incidents, our metrics showed that SQLite would report copying more pages from the WAL file than were actually available. If there are 10 pages in the WAL file and 20 pages get copied to the database, something is clearly wrong.</p><p>To understand what was happening during these faulty checkpoints, the SQLite developers created a new debugging tool for the virtual filesystem layer.</p><p>SQLite is split into several layers. The top layer is the parser and code generator, which converts SQL statements into SQLite’s internal data structures. These data structures get passed to the pager, which splits them into the individual pages to be written to disk. Actually writing them to disk is handled by the OS interface, or “virtual filesystem”. Currently SQLite has two mainstream virtual filesystem implementations—Unix and Windows.</p><figure id="" class="mb-6 scroll-mt-20 text-center"><img _type="asset" video="[object Object]" alt="Architecture diagram illustrating the internals of SQLite. It receives SQL statements as input, which pass through three layers: the parser/code generator, the pager, and the OS interface/virtual filesystem. The filesystem layer writes the changes to disk." loading="lazy" width="1920" height="1080" decoding="async" data-nimg="1" class="w-full will-change-transform md:block mx-auto" style="color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url(&quot;data:image/svg+xml;charset=utf-8,%3Csvg xmlns=&#x27;http://www.w3.org/2000/svg&#x27; viewBox=&#x27;0 0 1920 1080&#x27;%3E%3Cfilter id=&#x27;b&#x27; color-interpolation-filters=&#x27;sRGB&#x27;%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3CfeColorMatrix values=&#x27;1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1&#x27; result=&#x27;s&#x27;/%3E%3CfeFlood x=&#x27;0&#x27; y=&#x27;0&#x27; width=&#x27;100%25&#x27; height=&#x27;100%25&#x27;/%3E%3CfeComposite operator=&#x27;out&#x27; in=&#x27;s&#x27;/%3E%3CfeComposite in2=&#x27;SourceGraphic&#x27;/%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3C/filter%3E%3Cimage width=&#x27;100%25&#x27; height=&#x27;100%25&#x27; x=&#x27;0&#x27; y=&#x27;0&#x27; preserveAspectRatio=&#x27;none&#x27; style=&#x27;filter: url(%23b);&#x27; href=&#x27;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/wcAAwAB/ucNC4UAAAAASUVORK5CYII=&#x27;/%3E%3C/svg%3E&quot;)" srcSet="https://cdn.sanity.io/images/w77i7m8x/production/fc1ab3b4825fab7a103b2023b8122f6f6c8f258c-1920x1080.svg?w=1920&amp;q=75&amp;fit=clip&amp;auto=format 1x, https://cdn.sanity.io/images/w77i7m8x/production/fc1ab3b4825fab7a103b2023b8122f6f6c8f258c-1920x1080.svg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format 2x" src="https://cdn.sanity.io/images/w77i7m8x/production/fc1ab3b4825fab7a103b2023b8122f6f6c8f258c-1920x1080.svg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format"/></figure><p>If you&#x27;re interested in a deeper dive on these internals, I recommend <a target="" rel="noreferrer" href="https://www.youtube.com/watch?v=ZSKLA81tBis">this lecture by Richard Hipp</a>, the primary author of SQLite.</p><p>This approach allows you to replace different layers with different implementations, or wrap an existing layer to get more information. To help diagnose our problem, the SQLite developers created a wrapper around the virtual filesystem that writes additional tracing information and logs about changes to the database. This wrapper is called the <code>tmstmpvfs</code> shim, and the source code is available in the <a target="" rel="noreferrer" href="https://sqlite.org/src/file/ext/misc/tmstmpvfs.c?proof=394548354">SQLite public repository</a>.</p><figure id="" class="mb-6 scroll-mt-20 text-center"><img _type="asset" video="[object Object]" alt="Architecture diagram illustrating the internals of SQLite with our new debugging layer. The OS interface/virtual filesystem layer has now been wrapped in a tmstmpvfs shim." loading="lazy" width="1920" height="1080" decoding="async" data-nimg="1" class="w-full will-change-transform md:block mx-auto" style="color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url(&quot;data:image/svg+xml;charset=utf-8,%3Csvg xmlns=&#x27;http://www.w3.org/2000/svg&#x27; viewBox=&#x27;0 0 1920 1080&#x27;%3E%3Cfilter id=&#x27;b&#x27; color-interpolation-filters=&#x27;sRGB&#x27;%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3CfeColorMatrix values=&#x27;1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1&#x27; result=&#x27;s&#x27;/%3E%3CfeFlood x=&#x27;0&#x27; y=&#x27;0&#x27; width=&#x27;100%25&#x27; height=&#x27;100%25&#x27;/%3E%3CfeComposite operator=&#x27;out&#x27; in=&#x27;s&#x27;/%3E%3CfeComposite in2=&#x27;SourceGraphic&#x27;/%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3C/filter%3E%3Cimage width=&#x27;100%25&#x27; height=&#x27;100%25&#x27; x=&#x27;0&#x27; y=&#x27;0&#x27; preserveAspectRatio=&#x27;none&#x27; style=&#x27;filter: url(%23b);&#x27; href=&#x27;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/wcAAwAB/ucNC4UAAAAASUVORK5CYII=&#x27;/%3E%3C/svg%3E&quot;)" srcSet="https://cdn.sanity.io/images/w77i7m8x/production/2c42cf49f1419d6d8c07a56873b7ef1d327d1efe-1920x1080.svg?w=1920&amp;q=75&amp;fit=clip&amp;auto=format 1x, https://cdn.sanity.io/images/w77i7m8x/production/2c42cf49f1419d6d8c07a56873b7ef1d327d1efe-1920x1080.svg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format 2x" src="https://cdn.sanity.io/images/w77i7m8x/production/2c42cf49f1419d6d8c07a56873b7ef1d327d1efe-1920x1080.svg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format"/></figure><p>We deployed the shim into our live environment, and waited for the next corruption to occur. Fortunately, we didn&#x27;t have to wait long.</p><h2 id="the-wal-reset-bug"><a class="header-link group select-none !no-underline" href="#the-wal-reset-bug">The WAL-Reset bug<svg class="ml-2 inline w-3.5 opacity-0 transition-all duration-200 group-hover:opacity-100" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M15 7h3a5 5 0 0 1 5 5 5 5 0 0 1-5 5h-3m-6 0H6a5 5 0 0 1-5-5 5 5 0 0 1 5-5h3"></path><line x1="8" y1="12" x2="16" y2="12"></line></svg></a></h2><p>After our next corruption incident, the additional logs from the new <code>tmstmpvfs</code> shim allowed the SQLite developers to find and fix the bug: a rare data race in the SQLite source code between a checkpoint and a write transaction.</p><p>In particular, if a write occurs at a specific time during a checkpoint, the checkpointing process gets confused—it thinks some of the pages have been copied from the WAL into the main database file, but they haven’t. Those pages never get written to the database file, and that data is permanently lost. The database file becomes corrupt, because other pages which reference those pages—such as an index—are written to the database.</p><p>The SQLite developers named this the <a target="" rel="noreferrer" href="https://sqlite.org/wal.html#the_wal_reset_bug">“WAL-Reset bug”</a>, and they estimate it was present in SQLite for at least 16 years. It could exist that long because it was rare—so rare, the SQLite developers had to add code to deliberately trigger it in their testing environments. Their fix adds <a target="" rel="noreferrer" href="https://sqlite.org/src/info/7168988acbec2d8d">an additional check to the checkpointing function</a> which detects when the WAL has been reset by another thread.</p><p>They confirmed that this bug caused all of the baffling behaviour we’d seen. It explained the corruption, the transaction logs that wouldn’t apply cleanly, and the inconsistent checkpoint statistics. They also explained why we were more likely to hit the bug than other SQLite users: we take manual control of the checkpointing process, and we checkpoint very aggressively. Even a bug triggered by a rare condition was bound to hit us eventually.</p><p>This was an exciting moment. After months of confusion and uncertainty, we finally had a plausible theory for why the corruption was occurring, and a fix we could deploy to prevent it.</p><p>The SQLite developers released the fix as <a target="" rel="noreferrer" href="https://sqlite.org/changes.html#version_3_52_0">SQLite 3.52.0</a>, and we prepared to deploy it as soon as it was available.</p><h2 id="fixed-with-a-false-alarm"><a class="header-link group select-none !no-underline" href="#fixed-with-a-false-alarm">Fixed, with a false alarm<svg class="ml-2 inline w-3.5 opacity-0 transition-all duration-200 group-hover:opacity-100" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M15 7h3a5 5 0 0 1 5 5 5 5 0 0 1-5 5h-3m-6 0H6a5 5 0 0 1-5-5 5 5 0 0 1 5-5h3"></path><line x1="8" y1="12" x2="16" y2="12"></line></svg></a></h2><p>We rolled out SQLite 3.52.0 carefully—first to a few canary shards, then, when we saw it running smoothly, we deployed it to the rest of the control plane.</p><p>Our backup monitor promptly turned red, and reported corruption in <strong>13 different databases</strong>. This was extremely alarming, but we followed our recovery procedures to fix all the supposed corruption, and everything was happy. It turned out these databases had not suffered real corruption, but were subject to a second problem in the version of SQLite.</p><p>We shared our errors with the SQLite developers, which uncovered a bug in SQLite related to <a target="" rel="noreferrer" href="https://sqlite.org/staleexpridx.html">stale expression indexes</a>. If you create an index on a computed value, and then the computation changes, the index will contain mismatched values, which gets reported as corruption by <code>PRAGMA integrity_check</code>.</p><p>In our case, we were storing some high-precision timestamps as text, converting them to a floating-point number in a VIRTUAL generated column, and the SQLite 3.52.0 release that fixed our data race also made <a target="" rel="noreferrer" href="https://research.swtch.com/fp">an optimisation</a> that subtly changed the rounding behaviour for text-to-floating-point conversions. Our canary shards didn’t have any timestamps that triggered the changed rounding behaviour, so we missed this in our phased rollout.</p><p>Because this change caused false corruption warnings, the SQLite developers withdrew the 3.52.0 release and instead published 3.51.3, which only contained a fix for the WAL-Reset bug.</p><p>We fixed the issue on our side by reducing the precision of our timestamps to integer seconds; text-to-integer conversions are unambiguous. Meanwhile, the SQLite developers created an automated, <a target="" rel="noreferrer" href="https://sqlite.org/staleexpridx.html#selfheal">self-healing index feature</a> in 3.53.0, which prevents the stale expression index problem.</p><h2 id="party-time"><a class="header-link group select-none !no-underline" href="#party-time">Party time!<svg class="ml-2 inline w-3.5 opacity-0 transition-all duration-200 group-hover:opacity-100" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M15 7h3a5 5 0 0 1 5 5 5 5 0 0 1-5 5h-3m-6 0H6a5 5 0 0 1-5-5 5 5 0 0 1 5-5h3"></path><line x1="8" y1="12" x2="16" y2="12"></line></svg></a></h2><p>With the fix rolled out to our entire control plane, we were ready to declare victory, but we were still cautious. An absence of corruption incidents doesn’t mean things are fixed—we’d already had one six-week period of deceptive calm.</p><p>We wanted positive proof that this data race was actively occurring in our production environment. Now that we understood the cause of the bug—a collision between a write transaction and a WAL-reset—we patched our SQLite driver to <a target="" rel="noreferrer" href="https://github.com/tailscale/sqlite/commit/070c921dea03f793cbbbd9d39551d0ad26136113#diff-fcff16fa73210758d3a31f4cfa4609ad5b3ca692b66fc7fb7587c08a1ab11048">log a warning</a> when these two operations overlap. If the warning fired but the database remained uncorrupted, we’d know the fix had saved us from a potential corruption incident.</p><p>We deployed the warning, and we waited. And we waited. And waited. And waited. As weeks slipped by, we began to wonder why we didn’t see it. Was the warning broken? Was our theory wrong? Was the true bug still lurking in the darkness?</p><p>Then, two months later, the alert we were waiting for finally fired:</p><figure id="" class="mb-6 scroll-mt-20 text-center"><img _type="asset" video="[object Object]" alt="Alert Manager notification showing SQLitePartyMode warning: SQLite attempted corruption on shard2.corp.ts.net:8383 in party mode, but the system prevented it. Details include warning code, host, instance, job, namespace, severity, and shard information. Message advises checking server logs for corruption incident details." loading="lazy" width="1458" height="684" decoding="async" data-nimg="1" class="w-full will-change-transform md:block mx-auto" style="color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url(&quot;data:image/svg+xml;charset=utf-8,%3Csvg xmlns=&#x27;http://www.w3.org/2000/svg&#x27; viewBox=&#x27;0 0 1458 684&#x27;%3E%3Cfilter id=&#x27;b&#x27; color-interpolation-filters=&#x27;sRGB&#x27;%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3CfeColorMatrix values=&#x27;1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1&#x27; result=&#x27;s&#x27;/%3E%3CfeFlood x=&#x27;0&#x27; y=&#x27;0&#x27; width=&#x27;100%25&#x27; height=&#x27;100%25&#x27;/%3E%3CfeComposite operator=&#x27;out&#x27; in=&#x27;s&#x27;/%3E%3CfeComposite in2=&#x27;SourceGraphic&#x27;/%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3C/filter%3E%3Cimage width=&#x27;100%25&#x27; height=&#x27;100%25&#x27; x=&#x27;0&#x27; y=&#x27;0&#x27; preserveAspectRatio=&#x27;none&#x27; style=&#x27;filter: url(%23b);&#x27; href=&#x27;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/wcAAwAB/ucNC4UAAAAASUVORK5CYII=&#x27;/%3E%3C/svg%3E&quot;)" srcSet="https://cdn.sanity.io/images/w77i7m8x/production/19ae089ae209ad7e395e188b7f6e4a81be29c431-1458x684.png?w=1920&amp;q=75&amp;fit=clip&amp;auto=format 1x, https://cdn.sanity.io/images/w77i7m8x/production/19ae089ae209ad7e395e188b7f6e4a81be29c431-1458x684.png?w=3840&amp;q=75&amp;fit=clip&amp;auto=format 2x" src="https://cdn.sanity.io/images/w77i7m8x/production/19ae089ae209ad7e395e188b7f6e4a81be29c431-1458x684.png?w=3840&amp;q=75&amp;fit=clip&amp;auto=format"/></figure><p>This alert proved that the precise conditions for the WAL-Reset bug <em>do</em> occur in our production environment, which means it was the likely culprit for our six months of shaky uptime.</p><p>Since that weirdly joyous alert fired, we’ve run for another four months without any database incidents, as of this writing. Finally, we could breathe a sigh of relief.</p><h2 id="off-the-well-trodden-path"><a class="header-link group select-none !no-underline" href="#off-the-well-trodden-path">Off the well-trodden path<svg class="ml-2 inline w-3.5 opacity-0 transition-all duration-200 group-hover:opacity-100" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M15 7h3a5 5 0 0 1 5 5 5 5 0 0 1-5 5h-3m-6 0H6a5 5 0 0 1-5-5 5 5 0 0 1 5-5h3"></path><line x1="8" y1="12" x2="16" y2="12"></line></svg></a></h2><p>Nobody wanted us to spend six months looking for bugs in SQLite. This was an immensely frustrating experience for both our customers and staff, and we’re all glad to put this instability behind us.</p><p>This investigation is a useful reminder: <strong>running boring technology in a non-standard way is a risk. </strong>The common paths and standard configurations are incredibly well-tested and reliable. Most people use SQLite in a standard configuration and never face this sort of issue. Everything we were doing was a public, documented, supported configuration—but by taking manual control of the checkpointing process and running at our own aggressive pace, we stepped off the well-trodden operational path.</p><p>Resolving these incidents was a massive, cross-functional effort involving dozens of people—including Tailscale&#x27;s engineering and support teams, and the core maintainers of SQLite. It is to all of their credit that the impact of these incidents was not much worse.</p><p>We know that repeated downtime erodes trust, no matter how many people are affected, and we’re grateful to our customers for their patience and support while we chased this down.</p><p>Frustrating as this period was, we’re left in a stronger position than we were before. The long-standing bug in SQLite has been patched, and we fixed dozens of other incidental issues that we spotted while looking for it. We funded the open-source SQLite VFS shim that helped isolate the race condition almost immediately, and will help track down similar bugs in the future. Finally, we’ve refined our database backup and recovery processes, and live-tested them over a dozen times.</p><p>Hopefully there won’t be another database incident like this—but if there is, we’ll be ready.</p></div></div><div class="caption-new flex items-center gap-2 font-normal text-gray-500 md:px-8 lg:hidden">Share<div class="flex rounded-full bg-gray-700 fill-white p-1 transition duration-300 hover:bg-gray-300 hover:fill-black"><button aria-label="twitter" class="react-share__ShareButton" style="background-color:transparent;border:none;padding:0;font:inherit;color:inherit;cursor:pointer"><svg width="18" height="19" viewBox="0 0 18 19" xmlns="http://www.w3.org/2000/svg"><path d="M5.30916 5.93023H5.18497L5.26216 6.02752L8.41217 9.99815L5.24625 13.2224L5.14606 13.3244H5.28906H6.00994H6.03511L6.05275 13.3065L8.80942 10.499L11.033 13.3017L11.051 13.3244H11.08H13.5331H13.6573L13.5801 13.2271L10.251 9.03083L13.1953 6.03227L13.2955 5.93023H13.1525H12.4316H12.4064L12.3888 5.9482L9.8538 8.53001L7.80934 5.95294L7.79132 5.93023H7.76234H5.30916ZM11.3749 12.7038L6.49346 6.55082H7.46727L12.3487 12.7038H11.3749Z" fill="curentColor" stroke="curentColor" stroke-width="0.12"></path></svg></button></div><div class="flex rounded-full bg-gray-700 fill-white transition duration-300 hover:bg-gray-300 hover:fill-black"><button aria-label="facebook" class="react-share__ShareButton" style="background-color:transparent;border:none;padding:0;font:inherit;color:inherit;cursor:pointer"><svg width="28" height="28" viewBox="0 0 28 28" xmlns="http://www.w3.org/2000/svg"><path d="M12.434 19.6598L12.4179 14.8081H10.3008V12.7289H12.4179V11.3427C12.4179 9.47188 13.5974 8.57031 15.2966 8.57031C16.1106 8.57031 16.8101 8.62983 17.014 8.65643V10.6115L15.8355 10.612C14.9114 10.612 14.7324 11.0433 14.7324 11.6762V12.7289H17.3577L16.652 14.8081H14.7324V19.6598H12.434Z" fill="curentColor"></path></svg></button></div><div class="flex rounded-full bg-gray-700 fill-white transition duration-300 hover:bg-gray-300 hover:fill-black"><button aria-label="linkedin" class="react-share__ShareButton" style="background-color:transparent;border:none;padding:0;font:inherit;color:inherit;cursor:pointer"><svg width="28" height="28" viewBox="0 0 28 28" xmlns="http://www.w3.org/2000/svg"><path d="M8.68685 18.6518H10.8825V11.5871H8.68685V18.6518Z" fill="curentColor"></path><path d="M8.50195 9.34036C8.50195 10.0352 9.07976 10.6143 9.77312 10.6143C10.4896 10.6143 11.0443 10.0584 11.0443 9.34036C11.0443 8.64547 10.4665 8.06641 9.77312 8.06641C9.07976 8.06641 8.50195 8.64547 8.50195 9.34036Z" fill="curentColor"></path><path d="M16.8917 18.6518H19.0873V14.7836C19.0873 12.8843 18.6713 11.425 16.4525 11.425C15.3894 11.425 14.6729 12.0041 14.3724 12.56H14.3493V11.5871H12.2461V18.6518H14.4418V15.1542C14.4418 14.2509 14.6267 13.3475 15.7592 13.3475C16.8686 13.3475 16.8917 14.413 16.8917 15.2237V18.6518Z" fill="curentColor"></path></svg></button></div></div><div class="my-16 flex w-full justify-center"><svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg"><ellipse cx="2.65813" cy="9.96471" rx="2.44719" ry="2.44128" fill="#232222"></ellipse><ellipse cx="10.0019" cy="9.96471" rx="2.44719" ry="2.44128" fill="#232222"></ellipse><ellipse opacity="0.2" cx="2.65813" cy="17.2889" rx="2.44719" ry="2.44128" fill="#232222"></ellipse><ellipse opacity="0.2" cx="17.3378" cy="17.2889" rx="2.44719" ry="2.44128" fill="#232222"></ellipse><ellipse cx="10.0019" cy="17.2889" rx="2.44719" ry="2.44128" fill="#232222"></ellipse><ellipse cx="17.3378" cy="9.96471" rx="2.44719" ry="2.44128" fill="#232222"></ellipse><ellipse opacity="0.2" cx="2.65813" cy="2.64049" rx="2.44719" ry="2.44128" fill="#232222"></ellipse><ellipse opacity="0.2" cx="10.0019" cy="2.64049" rx="2.44719" ry="2.44128" fill="#232222"></ellipse><ellipse opacity="0.2" cx="17.3378" cy="2.64049" rx="2.44719" ry="2.44128" fill="#232222"></ellipse></svg></div></div><div class="mt-14 hidden h-fit w-full flex-col gap-6 lg:flex"><div class="flex flex-col items-start gap-2 border-t-[1px] border-gray-300 px-2 pt-2"><p class="body-new mb-1 font-medium">Author</p><div class="flex items-center gap-1 text-sm font-normal leading-[0.875rem] tracking-[-0.018rem]"><img alt="Alex Chan" loading="lazy" width="1729" height="1729" decoding="async" data-nimg="1" class="h-8 w-8 text-xs border-white rounded-full border-[1.5px]" style="color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url(&quot;data:image/svg+xml;charset=utf-8,%3Csvg xmlns=&#x27;http://www.w3.org/2000/svg&#x27; viewBox=&#x27;0 0 1729 1729&#x27;%3E%3Cfilter id=&#x27;b&#x27; color-interpolation-filters=&#x27;sRGB&#x27;%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3CfeColorMatrix values=&#x27;1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1&#x27; result=&#x27;s&#x27;/%3E%3CfeFlood x=&#x27;0&#x27; y=&#x27;0&#x27; width=&#x27;100%25&#x27; height=&#x27;100%25&#x27;/%3E%3CfeComposite operator=&#x27;out&#x27; in=&#x27;s&#x27;/%3E%3CfeComposite in2=&#x27;SourceGraphic&#x27;/%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3C/filter%3E%3Cimage width=&#x27;100%25&#x27; height=&#x27;100%25&#x27; x=&#x27;0&#x27; y=&#x27;0&#x27; preserveAspectRatio=&#x27;none&#x27; style=&#x27;filter: url(%23b);&#x27; href=&#x27;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/wcAAwAB/ucNC4UAAAAASUVORK5CYII=&#x27;/%3E%3C/svg%3E&quot;)" srcSet="https://cdn.sanity.io/images/w77i7m8x/production/7c909f6cf559dc9719380a3557fb7ff48e5db08a-1729x1729.jpg?w=1920&amp;q=75&amp;fit=clip&amp;auto=format 1x, https://cdn.sanity.io/images/w77i7m8x/production/7c909f6cf559dc9719380a3557fb7ff48e5db08a-1729x1729.jpg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format 2x" src="https://cdn.sanity.io/images/w77i7m8x/production/7c909f6cf559dc9719380a3557fb7ff48e5db08a-1729x1729.jpg?w=3840&amp;q=75&amp;fit=clip&amp;auto=format"/>Alex Chan</div></div><div class="caption-new flex items-center gap-2 px-2 font-normal text-gray-500">Share<div class="flex rounded-full bg-gray-700 fill-white p-1 transition duration-300 hover:bg-gray-300 hover:fill-black"><button aria-label="twitter" class="react-share__ShareButton" style="background-color:transparent;border:none;padding:0;font:inherit;color:inherit;cursor:pointer"><svg width="18" height="19" viewBox="0 0 18 19" xmlns="http://www.w3.org/2000/svg"><path d="M5.30916 5.93023H5.18497L5.26216 6.02752L8.41217 9.99815L5.24625 13.2224L5.14606 13.3244H5.28906H6.00994H6.03511L6.05275 13.3065L8.80942 10.499L11.033 13.3017L11.051 13.3244H11.08H13.5331H13.6573L13.5801 13.2271L10.251 9.03083L13.1953 6.03227L13.2955 5.93023H13.1525H12.4316H12.4064L12.3888 5.9482L9.8538 8.53001L7.80934 5.95294L7.79132 5.93023H7.76234H5.30916ZM11.3749 12.7038L6.49346 6.55082H7.46727L12.3487 12.7038H11.3749Z" fill="curentColor" stroke="curentColor" stroke-width="0.12"></path></svg></button></div><div class="flex rounded-full bg-gray-700 fill-white transition duration-300 hover:bg-gray-300 hover:fill-black"><button aria-label="facebook" class="react-share__ShareButton" style="background-color:transparent;border:none;padding:0;font:inherit;color:inherit;cursor:pointer"><svg width="28" height="28" viewBox="0 0 28 28" xmlns="http://www.w3.org/2000/svg"><path d="M12.434 19.6598L12.4179 14.8081H10.3008V12.7289H12.4179V11.3427C12.4179 9.47188 13.5974 8.57031 15.2966 8.57031C16.1106 8.57031 16.8101 8.62983 17.014 8.65643V10.6115L15.8355 10.612C14.9114 10.612 14.7324 11.0433 14.7324 11.6762V12.7289H17.3577L16.652 14.8081H14.7324V19.6598H12.434Z" fill="curentColor"></path></svg></button></div><div class="flex rounded-full bg-gray-700 fill-white transition duration-300 hover:bg-gray-300 hover:fill-black"><button aria-label="linkedin" class="react-share__ShareButton" style="background-color:transparent;border:none;padding:0;font:inherit;color:inherit;cursor:pointer"><svg width="28" height="28" viewBox="0 0 28 28" xmlns="http://www.w3.org/2000/svg"><path d="M8.68685 18.6518H10.8825V11.5871H8.68685V18.6518Z" fill="curentColor"></path><path d="M8.50195 9.34036C8.50195 10.0352 9.07976 10.6143 9.77312 10.6143C10.4896 10.6143 11.0443 10.0584 11.0443 9.34036C11.0443 8.64547 10.4665 8.06641 9.77312 8.06641C9.07976 8.06641 8.50195 8.64547 8.50195 9.34036Z" fill="curentColor"></path><path d="M16.8917 18.6518H19.0873V14.7836C19.0873 12.8843 18.6713 11.425 16.4525 11.425C15.3894 11.425 14.6729 12.0041 14.3724 12.56H14.3493V11.5871H12.2461V18.6518H14.4418V15.1542C14.4418 14.2509 14.6267 13.3475 15.7592 13.3475C16.8686 13.3475 16.8917 14.413 16.8917 15.2237V18.6518Z" fill="curentColor"></path></svg></button></div></div></div></div><div class="container">Loading...</div></section></main><section class=" pb-[50px] pt-10 lg:pt-[175px] bg-gray-0"><div class="is-wide container flex flex-col gap-6 md-large:flex-row"><div class="flex flex-1 flex-col justify-between rounded-[14px] bg-blue-900 overflow-hidden"><div class="relative z-10 flex-1 overflow-hidden px-[48px] py-[80px]"><div class="absolute -right-[1.5px] bottom-[0%] h-full w-auto opacity-25 xs:opacity-100 md:opacity-100 md-large:opacity-25 xl:opacity-100"><div class="flex h-full w-auto justify-end"><svg width="100%" height="100%" viewBox="0 0 264 394" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M0.666016 262.667C0.666019 190.118 59.4508 131.333 131.999 131.333L131.999 262.667L0.666016 262.667Z" fill="#6294F1"/><path d="M132 394C59.4515 394 0.666676 335.215 0.666682 262.667L132 262.667L132 394Z" fill="#253570"/><path d="M263.332 394C190.783 394 131.999 335.215 131.999 262.667L263.332 262.667L263.332 394Z" fill="#6294F1"/><path d="M132 131.333C59.4515 131.333 0.666673 72.5485 0.666676 -8.08502e-05L132 -7.62939e-05L132 131.333Z" fill="#253570"/><rect x="263.332" y="131.333" width="131.333" height="131.333" rx="65.6667" transform="rotate(90 263.332 131.333)" fill="#253570"/><rect x="263.332" width="131.333" height="131.333" rx="65.6667" transform="rotate(90 263.332 0)" fill="#6294F1"/></svg></div></div><div class="text-left mx-0 relative z-10"><h2 class="t-h3 text-white mb-4 md:mb-8" style="max-width:292px;color:#6C94EC">Try Tailscale for <span class="text-white">free</span></h2><div class="w-full xs:w-auto flex justify-start"><div class="flex w-full !w-auto flex-col gap-y-4 xs:w-auto xs:flex-row xs:items-center xs:space-x-5 md:space-x-[30px]"><a target="" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;Get started&quot;}" href="https://login.tailscale.com/start"><div class="inline-flex shrink-0 border transition-colors duration-200 w-full xs:w-auto items-center justify-center text-center relative focus:outline-none group transition-all overflow-hidden font-body font-medium rounded-lg group gap-[11px] py-2 px-[17px] leading-[1.46] tracking-[-0.16px] bg-heading-white border-white text-heading-black hover:bg-gray-50 hover:border-gray-50"><div>Get started</div><div class="relative "><span class="block will-change-transform" style="opacity:1;transform:none"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 18" class="w-4"><use href="/sprite.svg#arrowRight"></use></svg></span><span class="absolute inset-0 block will-change-transform" style="opacity:0;transform:translateX(-15px) translateY(0px) translateZ(0)"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 18" class="w-4"><use href="/sprite.svg#arrowRight"></use></svg></span></div></div></a></div></div></div></div><div class="relative w-full rounded-b-[14px] px-[48px] py-[60px] bg-blue-300" style="color:#232222"><div class="t-h5 !font-normal">Schedule a demo</div><a class="mt-3 block" data-track="Link Clicked" data-track-properties="{&quot;label&quot;: &quot;/contact/sales&quot;}" href="/contact/sales"><div class="inline-flex shrink-0 border transition-colors duration-200 w-full xs:w-auto items-center justify-center text-center relative focus:outline-none group transition-all overflow-hidden font-body font-medium rounded-lg group gap-[11px] py-2 px-[17px] leading-[1.46] tracking-[-0.16px] !w-auto bg-heading-black border-heading-black text-white hover:bg-gray-900 hover:border-gray-900 "><div>Contact sales</div></div></a></div></div><div class="flex flex-1 items-end justify-center overflow-hidden rounded-[14px] bg-blue-900 pt-[50px]"><div class="px-5" style="transform:translateY(100px) translateZ(0)"><img _type="asset" alt="cta phone" loading="lazy" width="362" height="567" decoding="async" data-nimg="1" class="md:block mx-auto" style="color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url(&quot;data:image/svg+xml;charset=utf-8,%3Csvg xmlns=&#x27;http://www.w3.org/2000/svg&#x27; viewBox=&#x27;0 0 362 567&#x27;%3E%3Cfilter id=&#x27;b&#x27; color-interpolation-filters=&#x27;sRGB&#x27;%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3CfeColorMatrix values=&#x27;1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1&#x27; result=&#x27;s&#x27;/%3E%3CfeFlood x=&#x27;0&#x27; y=&#x27;0&#x27; width=&#x27;100%25&#x27; height=&#x27;100%25&#x27;/%3E%3CfeComposite operator=&#x27;out&#x27; in=&#x27;s&#x27;/%3E%3CfeComposite in2=&#x27;SourceGraphic&#x27;/%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3C/filter%3E%3Cimage width=&#x27;100%25&#x27; height=&#x27;100%25&#x27; x=&#x27;0&#x27; y=&#x27;0&#x27; preserveAspectRatio=&#x27;none&#x27; style=&#x27;filter: url(%23b);&#x27; href=&#x27;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/wcAAwAB/ucNC4UAAAAASUVORK5CYII=&#x27;/%3E%3C/svg%3E&quot;)" srcSet="https://cdn.sanity.io/images/w77i7m8x/production/90e2ca60be7a63ed677b56ad293b23f1aae42baa-362x567.svg?w=384&amp;q=75&amp;fit=clip&amp;auto=format 1x, https://cdn.sanity.io/images/w77i7m8x/production/90e2ca60be7a63ed677b56ad293b23f1aae42baa-362x567.svg?w=750&amp;q=75&amp;fit=clip&amp;auto=format 2x" src="https://cdn.sanity.io/images/w77i7m8x/production/90e2ca60be7a63ed677b56ad293b23f1aae42baa-362x567.svg?w=750&amp;q=75&amp;fit=clip&amp;auto=format"/></div></div></div><div class="container mt-[60px]"><div><div class="flex flex-wrap items-center justify-center gap-4 lg:justify-between"><div class="flex w-[calc(33%-16px)] flex-col items-center gap-2 sm:w-[calc(33%-16px)] md:w-auto"><img alt="mercury" loading="lazy" width="199" height="81" decoding="async" data-nimg="1" class="mx-auto undefined" style="color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url(&quot;data:image/svg+xml;charset=utf-8,%3Csvg xmlns=&#x27;http://www.w3.org/2000/svg&#x27; viewBox=&#x27;0 0 199 81&#x27;%3E%3Cfilter id=&#x27;b&#x27; color-interpolation-filters=&#x27;sRGB&#x27;%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3CfeColorMatrix values=&#x27;1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1&#x27; result=&#x27;s&#x27;/%3E%3CfeFlood x=&#x27;0&#x27; y=&#x27;0&#x27; width=&#x27;100%25&#x27; height=&#x27;100%25&#x27;/%3E%3CfeComposite operator=&#x27;out&#x27; in=&#x27;s&#x27;/%3E%3CfeComposite in2=&#x27;SourceGraphic&#x27;/%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3C/filter%3E%3Cimage width=&#x27;100%25&#x27; height=&#x27;100%25&#x27; x=&#x27;0&#x27; y=&#x27;0&#x27; preserveAspectRatio=&#x27;none&#x27; style=&#x27;filter: url(%23b);&#x27; href=&#x27;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/wcAAwAB/ucNC4UAAAAASUVORK5CYII=&#x27;/%3E%3C/svg%3E&quot;)" srcSet="https://cdn.sanity.io/images/w77i7m8x/production/4590b93f6cc3b4573e5cd5cefe90e62f935e5aa7-199x81.svg?w=256&amp;q=75&amp;fit=clip&amp;auto=format 1x, https://cdn.sanity.io/images/w77i7m8x/production/4590b93f6cc3b4573e5cd5cefe90e62f935e5aa7-199x81.svg?w=640&amp;q=75&amp;fit=clip&amp;auto=format 2x" src="https://cdn.sanity.io/images/w77i7m8x/production/4590b93f6cc3b4573e5cd5cefe90e62f935e5aa7-199x81.svg?w=640&amp;q=75&amp;fit=clip&amp;auto=format"/></div><div class="flex w-[calc(33%-16px)] flex-col items-center gap-2 sm:w-[calc(33%-16px)] md:w-auto"><img alt="instacrt" loading="lazy" width="199" height="81" decoding="async" data-nimg="1" class="mx-auto undefined" style="color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url(&quot;data:image/svg+xml;charset=utf-8,%3Csvg xmlns=&#x27;http://www.w3.org/2000/svg&#x27; viewBox=&#x27;0 0 199 81&#x27;%3E%3Cfilter id=&#x27;b&#x27; color-interpolation-filters=&#x27;sRGB&#x27;%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3CfeColorMatrix values=&#x27;1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1&#x27; result=&#x27;s&#x27;/%3E%3CfeFlood x=&#x27;0&#x27; y=&#x27;0&#x27; width=&#x27;100%25&#x27; height=&#x27;100%25&#x27;/%3E%3CfeComposite operator=&#x27;out&#x27; in=&#x27;s&#x27;/%3E%3CfeComposite in2=&#x27;SourceGraphic&#x27;/%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3C/filter%3E%3Cimage width=&#x27;100%25&#x27; height=&#x27;100%25&#x27; x=&#x27;0&#x27; y=&#x27;0&#x27; preserveAspectRatio=&#x27;none&#x27; style=&#x27;filter: url(%23b);&#x27; href=&#x27;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/wcAAwAB/ucNC4UAAAAASUVORK5CYII=&#x27;/%3E%3C/svg%3E&quot;)" srcSet="https://cdn.sanity.io/images/w77i7m8x/production/698d5ead889b4e5a57f60549e74102b82ad7aae8-199x81.svg?w=256&amp;q=75&amp;fit=clip&amp;auto=format 1x, https://cdn.sanity.io/images/w77i7m8x/production/698d5ead889b4e5a57f60549e74102b82ad7aae8-199x81.svg?w=640&amp;q=75&amp;fit=clip&amp;auto=format 2x" src="https://cdn.sanity.io/images/w77i7m8x/production/698d5ead889b4e5a57f60549e74102b82ad7aae8-199x81.svg?w=640&amp;q=75&amp;fit=clip&amp;auto=format"/></div><div class="flex w-[calc(33%-16px)] flex-col items-center gap-2 sm:w-[calc(33%-16px)] md:w-auto"><img alt="Retool" loading="lazy" width="199" height="82" decoding="async" data-nimg="1" class="mx-auto undefined" style="color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url(&quot;data:image/svg+xml;charset=utf-8,%3Csvg xmlns=&#x27;http://www.w3.org/2000/svg&#x27; viewBox=&#x27;0 0 199 82&#x27;%3E%3Cfilter id=&#x27;b&#x27; color-interpolation-filters=&#x27;sRGB&#x27;%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3CfeColorMatrix values=&#x27;1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1&#x27; result=&#x27;s&#x27;/%3E%3CfeFlood x=&#x27;0&#x27; y=&#x27;0&#x27; width=&#x27;100%25&#x27; height=&#x27;100%25&#x27;/%3E%3CfeComposite operator=&#x27;out&#x27; in=&#x27;s&#x27;/%3E%3CfeComposite in2=&#x27;SourceGraphic&#x27;/%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3C/filter%3E%3Cimage width=&#x27;100%25&#x27; height=&#x27;100%25&#x27; x=&#x27;0&#x27; y=&#x27;0&#x27; preserveAspectRatio=&#x27;none&#x27; style=&#x27;filter: url(%23b);&#x27; href=&#x27;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/wcAAwAB/ucNC4UAAAAASUVORK5CYII=&#x27;/%3E%3C/svg%3E&quot;)" srcSet="https://cdn.sanity.io/images/w77i7m8x/production/258b299c38fba23800176525d02e7ce713f74820-199x82.svg?w=256&amp;q=75&amp;fit=clip&amp;auto=format 1x, https://cdn.sanity.io/images/w77i7m8x/production/258b299c38fba23800176525d02e7ce713f74820-199x82.svg?w=640&amp;q=75&amp;fit=clip&amp;auto=format 2x" src="https://cdn.sanity.io/images/w77i7m8x/production/258b299c38fba23800176525d02e7ce713f74820-199x82.svg?w=640&amp;q=75&amp;fit=clip&amp;auto=format"/></div><div class="flex w-[calc(33%-16px)] flex-col items-center gap-2 sm:w-[calc(33%-16px)] md:w-auto"><img alt="duolingo" loading="lazy" width="199" height="81" decoding="async" data-nimg="1" class="mx-auto undefined" style="color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url(&quot;data:image/svg+xml;charset=utf-8,%3Csvg xmlns=&#x27;http://www.w3.org/2000/svg&#x27; viewBox=&#x27;0 0 199 81&#x27;%3E%3Cfilter id=&#x27;b&#x27; color-interpolation-filters=&#x27;sRGB&#x27;%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3CfeColorMatrix values=&#x27;1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1&#x27; result=&#x27;s&#x27;/%3E%3CfeFlood x=&#x27;0&#x27; y=&#x27;0&#x27; width=&#x27;100%25&#x27; height=&#x27;100%25&#x27;/%3E%3CfeComposite operator=&#x27;out&#x27; in=&#x27;s&#x27;/%3E%3CfeComposite in2=&#x27;SourceGraphic&#x27;/%3E%3CfeGaussianBlur stdDeviation=&#x27;20&#x27;/%3E%3C/filter%3E%3Cimage width=&#x27;100%25&#x27; height=&#x27;100%25&#x27; x=&#x27;0&#x27; y=&#x27;0&#x27; preserveAspectRatio=&#x27;none&#x27; style=&#x27;filter: url(%23b);&#x27; href=&#x27;data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/wcAAwAB/ucNC4UAAAAASUVORK5CYII=&#x27;/%3E%3C/svg%3E&quot;)" srcSet="https://cdn.sanity.io/images/w77i7m8x/production/b74adcdab85e2515c997140d08827f084bccfa2a-199x81.svg?w=256&amp;q=75&amp;fit=clip&amp;auto=format 1x, https://cdn.sanity.io/images/w77i7m8x/production/b74adcdab85e2515c997140d08827f084bccfa2a-199x81.svg?w=640&amp;q=75&amp;fit=clip&amp;auto=format 2x" src="https://cdn.sanity.io/images/w77i7m8x/production/b74adcdab85e2515c997140d08827f084bccfa2a-199x81.svg?w=640&amp;q=75&amp;fit=clip&amp;auto=format"/></div></div></div></div></section><footer class="mt-16 bg-white py-12"><div class="mx-auto w-full max-w-[1360px] px-6 md:max-w-[1440px] md:px-16"><div class="grid grid-cols-1 gap-4 sm:grid-cols-2 md:grid-cols-12"><div class="px-3 sm:col-span-2 md:col-span-4 md:px-8 md:py-8"><a class="block w-[110px]" title="Homepage" href="/"><svg width="48" height="48" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true"><path d="M3 15C4.65685 15 6 13.6569 6 12C6 10.3431 4.65685 9 3 9C1.34315 9 0 10.3431 0 12C0 13.6569 1.34315 15 3 15Z" fill="currentColor"></path><path d="M12 15C13.6569 15 15 13.6569 15 12C15 10.3431 13.6569 9 12 9C10.3431 9 9 10.3431 9 12C9 13.6569 10.3431 15 12 15Z" fill="currentColor"></path><path d="M12 24C13.6569 24 15 22.6569 15 21C15 19.3431 13.6569 18 12 18C10.3431 18 9 19.3431 9 21C9 22.6569 10.3431 24 12 24Z" fill="currentColor"></path><path d="M21 15C22.6569 15 24 13.6569 24 12C24 10.3431 22.6569 9 21 9C19.3431 9 18 10.3431 18 12C18 13.6569 19.3431 15 21 15Z" fill="currentColor"></path><path d="M3 24C4.65685 24 6 22.6569 6 21C6 19.3431 4.65685 18 3 18C1.34315 18 0 19.3431 0 21C0 22.6569 1.34315 24 3 24Z" fill="currentColor" fill-opacity="0.4"></path><path d="M21 24C22.6569 24 24 22.6569 24 21C24 19.3431 22.6569 18 21 18C19.3431 18 18 19.3431 18 21C18 22.6569 19.3431 24 21 24Z" fill="currentColor" fill-opacity="0.4"></path><path d="M3 6C4.65685 6 6 4.65685 6 3C6 1.34315 4.65685 0 3 0C1.34315 0 0 1.34315 0 3C0 4.65685 1.34315 6 3 6Z" fill="currentColor" fill-opacity="0.4"></path><path d="M12 6C13.6569 6 15 4.65685 15 3C15 1.34315 13.6569 0 12 0C10.3431 0 9 1.34315 9 3C9 4.65685 10.3431 6 12 6Z" fill="currentColor" fill-opacity="0.4"></path><path d="M21 6C22.6569 6 24 4.65685 24 3C24 1.34315 22.6569 0 21 0C19.3431 0 18 1.34315 18 3C18 4.65685 19.3431 6 21 6Z" fill="currentColor" fill-opacity="0.4"></path></svg></a></div><div class="py-3 sm:col-span-1 md:col-span-2 md:py-8"><h3 class="flex flex-row items-center gap-2.5 px-3 pb-3 font-mdio text-xs font-medium uppercase !tracking-[0.6px] text-gray-700/80"><span class="shrink-0">Company</span><hr class="flex-1 border-gray-200"/></h3><ul class="flex flex-col gap-1"><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="/company">About Tailscale</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="/careers">Careers</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="/press">Press</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="/opensource">Open Source</a></li></ul></div><div class="py-3 sm:col-span-1 md:col-span-2 md:py-8"><h3 class="flex flex-row items-center gap-2.5 px-3 pb-3 font-mdio text-xs font-medium uppercase !tracking-[0.6px] text-gray-700/80"><span class="shrink-0">Help &amp; Support</span><hr class="flex-1 border-gray-200"/></h3><ul class="flex flex-col gap-1"><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="/contact/support">Support</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="/contact/sales">Sales</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="/partnerships">Partnerships</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="/security">Security</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="/changelog">Changelog</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a target="_blank" rel="noopener noreferrer" class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="https://status.tailscale.com">Tailscale Status</a></li></ul></div><div class="py-3 sm:col-span-1 md:col-span-2 md:py-8"><h3 class="flex flex-row items-center gap-2.5 px-3 pb-3 font-mdio text-xs font-medium uppercase !tracking-[0.6px] text-gray-700/80"><span class="shrink-0">Legal</span><hr class="flex-1 border-gray-200"/></h3><ul class="flex flex-col gap-1"><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="/terms">Terms of Service</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="/privacy">Privacy Policy</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="/privacy-policy#california-notice">California Notice</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="/cookie-notice">Cookie Notice</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><button type="button" class="flex h-[20px] items-center gap-2 text-left text-sm font-medium text-gray-700">Your privacy choices<img alt="Check mark and x on a white and blue pill button" loading="lazy" width="30" height="14" decoding="async" data-nimg="1" style="color:transparent" src="https://cdn.sanity.io/images/w77i7m8x/production/07d853f507039b2489d9818cb6ee7442c1b60e2a-30x14.svg"/></button></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="/legal">All Legal</a></li></ul></div><div class="py-3 sm:col-span-1 md:col-span-2 md:py-8"><h3 class="flex flex-row items-center gap-2.5 px-3 pb-3 font-mdio text-xs font-medium uppercase !tracking-[0.6px] text-gray-700/80"><span class="shrink-0">Social</span><hr class="flex-1 border-gray-200"/></h3><ul class="flex flex-col gap-1"><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a target="_blank" rel="noopener noreferrer" class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="https://discord.gg/tailscale">Discord</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a target="_blank" rel="noopener noreferrer" class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="https://github.com/tailscale">GitHub</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a target="_blank" rel="noopener noreferrer" class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="https://www.linkedin.com/company/tailscale">LinkedIn</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a target="_blank" rel="noopener noreferrer" class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="https://hachyderm.io/@tailscale">Mastodon</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a target="_blank" rel="noopener noreferrer" class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="https://www.reddit.com/r/Tailscale">Reddit</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a target="_blank" rel="noopener noreferrer" class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="https://www.youtube.com/@Tailscale">YouTube</a></li><li class="rounded-md p-3 transition-colors duration-200 hover:bg-gray-200"><a target="_blank" rel="noopener noreferrer" class="flex h-[20px] items-center text-sm font-medium text-gray-700 no-underline" href="https://twitter.com/tailscale">X (Twitter)</a></li></ul></div></div><div class="mx-3 flex flex-col items-start gap-3 border-gray-200 sm:border-t sm:pt-6 md:mx-8 md:flex-row md:items-center md:border-t-0 md:pt-0"><span class="text-xs text-gray-500">© 2026 Tailscale Inc.</span><hr class="hidden flex-1 self-center border-gray-200 md:inline"/><span class="flex flex-col text-xs text-gray-500 md:block"><span>Tailscale is a registered trademark of Tailscale Inc.</span><span class="hidden md:inline"> | </span><span>WireGuard is a registered trademark of Jason A. Donenfeld</span></span></div></div></footer></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"post":{"_id":"45389fc1-da6e-4e0d-82f6-140651c6ba57","authors":[{"_createdAt":"2026-02-17T19:39:49Z","_id":"1640153a-a496-451a-8807-6307ce3c2f50","_rev":"gz9XuCcqKJ5aRLQDDJ2DtZ","_system":{"base":{"id":"1640153a-a496-451a-8807-6307ce3c2f50","rev":"UhQG1jCdoCqTZDHLXbdaNy"}},"_type":"teamMember","_updatedAt":"2026-08-12T14:38:50Z","member":{"image":{"_type":"sanityImage","alt":"Alex Chan","asset":{"_ref":"image-7c909f6cf559dc9719380a3557fb7ff48e5db08a-1729x1729-jpg","_type":"reference"}},"jobTitle":"Engineer, developer experience","name":"Alex Chan"},"title":"Alex Chan"}],"category":"insights","content":null,"contributors":null,"excerpt":"SQLite corruption? Sorted.","featuredImage":{"_type":"image","alt":"Light orange and dark yellow shapes, like ovals, squares, circles, and quarter-circles, against a pale yellow background.","asset":{"_ref":"image-6ca0577c6bb689a7af13ec1407ebc8afd2de8136-4608x2372-png","_type":"reference"}},"flexContent":[{"_key":"74daba3c49ac","_type":"block","children":[{"_key":"1dca11c0ab61","_type":"span","marks":[],"text":"At the end of last year, our uptime was "},{"_key":"ee2781c27e5b","_type":"span","marks":["b38d9d5857a2"],"text":"pretty shaky"},{"_key":"3670643e18e9","_type":"span","marks":[],"text":". You can see this trend on "},{"_key":"2193b7620678","_type":"span","marks":["23aae48f6a51"],"text":"our status page"},{"_key":"75e811b995b0","_type":"span","marks":[],"text":", and that instability continued into the new year. Many of these outages were caused by a single bug, deep in "},{"_key":"1a39b2276cb8","_type":"span","marks":["e649dc8fe348"],"text":"SQLite"},{"_key":"f5f2993d1fd4","_type":"span","marks":[],"text":". It took months of intense forensics to track it down."}],"markDefs":[{"_key":"b38d9d5857a2","_type":"link","href":"https://tailscale.com/blog/hypergrowth-isnt-always-easy"},{"_key":"23aae48f6a51","_type":"link","href":"https://status.tailscale.com/history"},{"_key":"e649dc8fe348","_type":"link","href":"https://sqlite.org/index.html"}],"style":"normal"},{"_key":"a8aa95be4c26","_type":"block","children":[{"_key":"99f972268c49","_type":"span","marks":[],"text":"Now we’re in summer, we’re confident that we’ve found the bug, that we understand it—and more importantly, that we’ve fixed it."}],"markDefs":[],"style":"normal"},{"_key":"804d8805e042","_type":"block","children":[{"_key":"ed293a1b1dc4","_type":"span","marks":[],"text":"We know our customers expect Tailscale to be a reliable service, and for several months we didn’t live up to that promise. That’s disruptive, and we’re sorry. We’re publishing this blog post to explain what went wrong, how we responded, and how we ultimately helped to uncover a long-standing bug in the heart of the SQLite database."}],"markDefs":[],"style":"normal"},{"_key":"2b131401530c","_type":"block","children":[{"_key":"1e9f7db12acc","_type":"span","marks":[],"text":"Tailscale’s database architecture"}],"markDefs":[],"style":"h2"},{"_key":"af29ea85868e","_type":"block","children":[{"_key":"e0b4807dffb3","_type":"span","marks":[],"text":"While our clients interact with our "},{"_key":"e1dcde76b526","_type":"span","marks":["49fc5a99adca"],"text":"control plane"},{"_key":"eec22969a307","_type":"span","marks":[],"text":" as a single public endpoint ("},{"_key":"cf48d86ae343","_type":"span","marks":["2b269d757bd6"],"text":"controlplane.tailscale.com"},{"_key":"23090b983fb5","_type":"span","marks":[],"text":"), internally, our control plane is split into a series of coordination servers (or “shards”). Each tailnet lives on one internal shard at a time, but can migrate seamlessly from one to another. These shards are an internal implementation detail: you don’t know what shard your tailnet is on, and you never need to."}],"markDefs":[{"_key":"49fc5a99adca","_type":"link","href":"https://tailscale.com/docs/concepts/control-data-planes"},{"_key":"2b269d757bd6","_type":"link","href":"http://controlplane.tailscale.com"}],"style":"normal"},{"_key":"62c18e16dca8","_type":"block","children":[{"_key":"cbb4bd3bd143","_type":"span","marks":[],"text":"Each shard has an SQLite database that holds all the information about the tailnets on that shard. A single Go process exclusively accesses that database, and serves the control plane for those tailnets. This single-writer design is exactly how SQLite is meant to be used."}],"markDefs":[],"style":"normal"},{"_key":"267b6931c341","_type":"assetBlock","asset":{"_type":"asset","alignment":"center","image":{"_type":"sanityImage","alt":"Architecture diagram illustrating how the Tailscale control plane is made of isolated shards, each of which has an individual SQLite database.","asset":{"_ref":"image-6db766d5b5c004085237e7498faf57a0e8d3205a-1920x1080-svg","_type":"reference"}},"mobileImage":{"_type":"sanityImage","alt":"Architecture diagram illustrating how the Tailscale control plane is made of isolated shards, each of which has an individual SQLite database.","asset":{"_ref":"image-5465abbf3d023ee88d2b3b073b759750758b6f33-3840x2160-png","_type":"reference"}},"placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":true,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"display":"block"},{"_key":"3eb4f3c257c8","_type":"block","children":[{"_key":"e734fe664506","_type":"span","marks":[],"text":"We’ve used SQLite as our primary database "},{"_key":"d29c6c4af58a","_type":"span","marks":["d0561acf50ef"],"text":"since 2022"},{"_key":"4cb4e05afbb4","_type":"span","marks":[],"text":", and we chose it because it's well-known, reliable, and widely used. SQLite is "},{"_key":"eb786379953d","_type":"span","marks":["f4928b0c8808"],"text":"“boring technology”"},{"_key":"c615608ce635","_type":"span","marks":[],"text":"—in a good way. Many companies use SQLite in much larger deployments without issue, and we expected the same stress-free usage."}],"markDefs":[{"_key":"d0561acf50ef","_type":"link","href":"https://tailscale.com/blog/database-for-2022"},{"_key":"f4928b0c8808","_type":"link","href":"https://mcfunley.com/choose-boring-technology"}],"style":"normal"},{"_key":"4c0d874030e0","_type":"block","children":[{"_key":"3aac496c756a","_type":"span","marks":[],"text":"In our current backup pipeline, we take a complete snapshot of the database every few minutes, then upload the entire SQLite file to an S3 bucket. We’d been running this setup without incident since early 2023."}],"markDefs":[],"style":"normal"},{"_key":"d9d261f71e75","_type":"block","children":[{"_key":"95105a19dce0","_type":"span","marks":[],"text":"Fast forward to August last year, when a data pipeline that reads those S3 backups reported an error in one of our databases. We ran SQLite’s "},{"_key":"eb5d5b07178b","_type":"span","marks":["code"],"text":"PRAGMA integrity_check"},{"_key":"2b73950ad785","_type":"span","marks":[],"text":" command against the backup, and found it was indeed corrupted. SQLite corruption "},{"_key":"8cd581c357c8","_type":"span","marks":["4434cb692d57"],"text":"is possible"},{"_key":"670b9b868f8d","_type":"span","marks":[],"text":", but it’s highly unusual and not something you should encounter in normal operation. We repaired the affected database, and investigated the cause, but to no avail."}],"markDefs":[{"_key":"4434cb692d57","_type":"link","href":"https://www.sqlite.org/howtocorrupt.html"}],"style":"normal"},{"_key":"821ab25c6912","_type":"block","children":[{"_key":"d9e2aaad8194","_type":"span","marks":[],"text":"When operating at scale, even rare events can occur with some frequency, so we should have been unsurprised when it happened again—and again, and again, and again. In total, we faced 19 separate instances of database corruption over six months before we finally resolved the underlying bug."}],"markDefs":[],"style":"normal"},{"_key":"54c1c913aa8b","_type":"block","children":[{"_key":"fed3fb3c5f41","_type":"span","marks":[],"text":"When you hear the phrase “database corruption”, it’s natural to worry about data loss. Because our control plane only handles configuration data, these databases contain metadata about your tailnet and devices, but never your private encryption keys or network traffic. In the earliest incidents, the recovery process meant a handful of newly added devices or configuration changes didn’t persist, and a small amount of metadata had to be re-entered."}],"markDefs":[],"style":"normal"},{"_key":"4e2d2163da5d","_type":"assetBlock","asset":{"_type":"asset","alignment":"center","image":{"_type":"sanityImage","alt":"Architecture diagram illustrating how the Tailscale control plane copes with a database issue on a single shard. When the SQLite database has database corruption, it affects traffic on that shard, but other shards are unaffected.","asset":{"_ref":"image-eec436b49f783f30c56edb32b6eeacff62ebfdf4-1920x1080-svg","_type":"reference"}},"mobileImage":{"_type":"sanityImage","alt":"Architecture diagram illustrating how the Tailscale control plane copes with a database issue on a single shard. When the SQLite database has database corruption, it affects traffic on that shard, but other shards are unaffected.","asset":{"_ref":"image-7e753f8c4f4fcc18d8231d73c6474870a33eaf20-3840x2160-png","_type":"reference"}},"placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":true,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"display":"block"},{"_key":"64aa35eab7dd","_type":"block","children":[{"_key":"8ac76f41ceaa","_type":"span","marks":[],"text":"Whenever corruption occurred, we had to stop the control plane process on the shard while we repaired or restored the database. This was painful for tailnets on that shard, because their entire control plane disappeared during that recovery window. In the early incidents, that downtime was over an hour, but we gradually sped up the recovery process over subsequent incidents."}],"markDefs":[],"style":"normal"},{"_key":"458359a008ff","_type":"block","children":[{"_key":"91d676f562be","_type":"span","marks":[],"text":"Each tailnet is a mesh network, where devices make peer-to-peer WireGuard® connections to each other. When a device joins the tailnet, it has to get a list of other devices from the control plane before it can establish new connections—so if a device came online during the SQLite downtime, it couldn’t connect. While the database was being repaired, devices already online remained connected to each other, but they couldn’t learn about changes to the network. Those tailnets also temporarily lost access to the web-based admin console and the Tailscale API."}],"markDefs":[],"style":"normal"},{"_key":"9bda6b608dda","_type":"block","children":[{"_key":"56d93544a593","_type":"span","marks":[],"text":"There’s also a broader impact on trust. We post a global incident on "},{"_key":"402729feb701","_type":"span","marks":["45578d366df6"],"text":"our status page"},{"_key":"7c1faff35683","_type":"span","marks":[],"text":" even when only a small number of tailnets are affected. Many people saw a status page event for an incident that didn’t affect them. Indeed, the majority of shards and tailnets were never involved in a database corruption incident! Nonetheless, repeated downtime erodes trust, whether or not you’re directly affected."}],"markDefs":[{"_key":"45578d366df6","_type":"link","href":"https://status.tailscale.com/"}],"style":"normal"},{"_key":"7f76354d056b","_type":"block","children":[{"_key":"9a25540b8796","_type":"span","marks":[],"text":"From the very first instance of corruption, we knew this was a serious threat to our reliability, and we threw a lot of engineering time at the problem—but the fix wasn’t easy."}],"markDefs":[],"style":"normal"},{"_key":"c7ba3e172bfa","_type":"block","children":[{"_key":"be889e089f14","_type":"span","marks":[],"text":"Trying to find the fault"}],"markDefs":[],"style":"h2"},{"_key":"ed486e6e3cfd","_type":"block","children":[{"_key":"dbd792e12c2f","_type":"span","marks":[],"text":"This bug resisted all our initial attempts to find it."}],"markDefs":[],"style":"normal"},{"_key":"9e6d73e5d2b1","_type":"block","children":[{"_key":"5d638ee917ba","_type":"span","marks":[],"text":"We looked at recent changes, but there weren’t any that seemed relevant. Nobody had been working on our low-level code that interacts with SQLite, because it had all been written years ago and presented no issues up until that point. We re-reviewed all of that code with a fine-toothed comb to look for previously missed bugs, but we didn’t find anything that would cause the corruption we were seeing."}],"markDefs":[],"style":"normal"},{"_key":"a3951d847821","_type":"block","children":[{"_key":"3f7b76206173","_type":"span","marks":[],"text":"We looked for common factors between corruption incidents, but we couldn’t find any. It wasn’t tied to a single shard, or customer, or tailnet feature, or time of day, or load level. We were at a loss for what might be triggering the behaviour."}],"markDefs":[],"style":"normal"},{"_key":"58ce4f1b088c","_type":"block","children":[{"_key":"51284a31d095","_type":"span","marks":[],"text":"This lack of reliable trigger conditions meant we couldn’t reproduce the bug synthetically. Instead, we had to rely on deploying passive, forensic telemetry in our live environment to catch the corruption red-handed. Gathering live diagnostics for a database issue is the last thing we wanted to do, but we had no choice."}],"markDefs":[],"style":"normal"},{"_key":"26f1c454029d","_type":"block","children":[{"_key":"ff73ed62feb6","_type":"span","marks":[],"text":"As an additional complication, the corruption didn’t occur on a regular schedule. Sometimes incidents would be hours apart, other times weeks. This made it difficult to predict progress or plan further work, because we were never sure when we’d get our next diagnostic dump. We had a six-week period between October and December when there were no corruption incidents, before they returned as an unwelcome Christmas present."}],"markDefs":[],"style":"normal"},{"_key":"81c7a5ac9f82","_type":"block","children":[{"_key":"e1311b9f16c0","_type":"span","marks":[],"text":"Because this wouldn’t be a quick or easy fix, we reached out to the SQLite developers for a "},{"_key":"f726127bbc52","_type":"span","marks":["8d059be079f6"],"text":"professional support contract"},{"_key":"f1538b7703f0","_type":"span","marks":[],"text":". This was a great decision. It gave us direct access to their deep expertise and experience, and we had many detailed technical conversations about our architecture and our incidents."}],"markDefs":[{"_key":"8d059be079f6","_type":"link","href":"https://sqlite.org/prosupport.html"}],"style":"normal"},{"_key":"08ab12657084","_type":"block","children":[{"_key":"3e6ca4d04517","_type":"span","marks":[],"text":"Between Tailscale engineering and the SQLite core developers, we mapped out several theories for what might be causing the corruption—including "},{"_key":"c25c9c199b82","_type":"span","marks":["e39b36347cce"],"text":"broken POSIX locks on close()"},{"_key":"7c675c80b0f1","_type":"span","marks":[],"text":", mismanaging memory owned by SQLite, or accidentally using SQLite from multiple threads "},{"_key":"9754dafd695f","_type":"span","marks":["bf0c5160a86a"],"text":"while disabling thread safety"},{"_key":"ca78e8e547ba","_type":"span","marks":[],"text":". After every incident, we gathered more data, added more diagnostics, and systematically ruled out these theories. We were gradually converging on the true bug."}],"markDefs":[{"_key":"e39b36347cce","_type":"link","href":"https://www.sqlite.org/howtocorrupt.html#_posix_advisory_locks_canceled_by_a_separate_thread_doing_close_"},{"_key":"bf0c5160a86a","_type":"link","href":"https://sqlite.org/compile.html#threadsafe"}],"style":"normal"},{"_key":"d12b800cbb0a","_type":"block","children":[{"_key":"99a38593f0b2","_type":"span","marks":[],"text":"The transactions that didn’t bark"}],"markDefs":[],"style":"h2"},{"_key":"a970ac78f758","_type":"block","children":[{"_key":"8e3ef631ee5e","_type":"span","marks":[],"text":"While we were investigating the root cause, we still had a live platform to run. We took aggressive steps to automate recovery and minimize downtime:"}],"markDefs":[],"style":"normal"},{"_key":"deede17449c7","_type":"block","children":[{"_key":"6dcdd4f1b32b","_type":"span","marks":[],"text":"Configuring our control plane shards to hard-stop immediately upon encountering corruption"}],"level":1,"listItem":"bullet","markDefs":[],"style":"normal"},{"_key":"afc68f949c6c","_type":"block","children":[{"_key":"e6b8db8095eb","_type":"span","marks":[],"text":"Deploying an automated backup monitor that continuously ran "},{"_key":"7d7f4fd50cfa","_type":"span","marks":["code"],"text":"PRAGMA integrity_check"},{"_key":"9a7140c417f9","_type":"span","marks":[],"text":" over our backups"}],"level":1,"listItem":"bullet","markDefs":[],"style":"normal"},{"_key":"53c746b6cfb6","_type":"block","children":[{"_key":"2b96773cabd8","_type":"span","marks":[],"text":"Improving our runbooks and on-call training"}],"level":1,"listItem":"bullet","markDefs":[],"style":"normal"},{"_key":"d22a30ed1a3a","_type":"block","children":[{"_key":"6b26217bca77","_type":"span","marks":[],"text":"These efforts cut our response time to under an hour—and then we discovered an unexpected clue."}],"markDefs":[],"style":"normal"},{"_key":"d8c8fff69df1","_type":"block","children":[{"_key":"9aaaaaf3f450","_type":"span","marks":[],"text":"We wanted a way to restore service that didn’t involve rolling back to the last known-good backup (which would lose a lot of data) or repairing the known-corrupted database (which was potentially risky)."}],"markDefs":[],"style":"normal"},{"_key":"1d92acc3b9d8","_type":"block","children":[{"_key":"ee07ab42fdbc","_type":"span","marks":[],"text":"To do this, we built a transaction logging pipeline. We streamed every SQL statement that modified the database to a separate log file. Because SQLite is "},{"_key":"05c29d7c29ce","_type":"span","marks":["4c779748645c"],"text":"a single-writer database"},{"_key":"c07abc03d00f","_type":"span","marks":[],"text":" with "},{"_key":"91cddd866223","_type":"span","marks":["6677cb0d2e98"],"text":"serialisable transactions"},{"_key":"085498fd7c28","_type":"span","marks":[],"text":", our transaction history was completely linear and deterministic. (This wouldn’t be true in a multi-writer database like Postgres or MySQL.) Replaying those transactions against the latest known-good backup should restore the database to its most recent state, safely bypassing the corruption."}],"markDefs":[{"_key":"4c779748645c","_type":"link","href":"https://sqlite.org/faq.html#:~:text=only%20one%20process%20can%20be%20making%20changes%20to%20the%20database%20at%20any%20moment%20in%20time"},{"_key":"6677cb0d2e98","_type":"link","href":"https://www.sqlite.org/isolation.html"}],"style":"normal"},{"_key":"be3229c01011","_type":"assetBlock","asset":{"_type":"asset","alignment":"center","image":{"_type":"sanityImage","alt":"Diagram illustrating how the changes between two backups can be reconstructed by replaying the transactions that occurred between them.","asset":{"_ref":"image-34832281499b7d1388317372478dc0001770b16e-1920x1080-svg","_type":"reference"}},"mobileImage":{"_type":"sanityImage","alt":"Diagram illustrating how the changes between two backups can be reconstructed by replaying the transactions that occurred between them.","asset":{"_ref":"image-cefdf5f9f029f1d61d3da5cf426c487cb010c9f2-3840x2160-png","_type":"reference"}},"placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":true,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"display":"block"},{"_key":"8248be21aec0","_type":"block","children":[{"_key":"56fa6e39796e","_type":"span","marks":[],"text":"This pipeline worked, but then it did something even better: it gave us a clue."}],"markDefs":[],"style":"normal"},{"_key":"a02eb58429f2","_type":"block","children":[{"_key":"2345629ded6c","_type":"span","marks":[],"text":"In two incidents, our transaction logs failed to replay cleanly. Upon closer inspection, we discovered that data written and committed by one transaction was inexplicably invisible to later transactions. A write had vanished into thin air without raising an error. That should be impossible!"}],"markDefs":[],"style":"normal"},{"_key":"c949fe50f6fe","_type":"block","children":[{"_key":"076fbd3c5f72","_type":"span","marks":[],"text":"The writing on the WAL"}],"markDefs":[],"style":"h2"},{"_key":"7a2a95c4394c","_type":"block","children":[{"_key":"43b0e16632dc","_type":"span","marks":[],"text":"As these incidents were ongoing, the SQLite developers had been developing a new debugging tool. For a while, we’d suspected that the bug was somewhere in the checkpoint process. They were building a new tool to give better visibility into what was happening during checkpoints."}],"markDefs":[],"style":"normal"},{"_key":"d4a1961196d5","_type":"block","children":[{"_key":"105a8a9a2fe8","_type":"span","marks":[],"text":"To understand what this tool found, we need to briefly explain how SQLite checkpoints work."}],"markDefs":[],"style":"normal"},{"_key":"6cf84a4114c9","_type":"block","children":[{"_key":"1c53333fb2b4","_type":"span","marks":[],"text":"A SQLite database is made of a series of “pages”, tiny blocks of information. When you update the database, some of those pages need to be replaced with new pages with the updated information."}],"markDefs":[],"style":"normal"},{"_key":"5cc52f2b1462","_type":"block","children":[{"_key":"81d9cdef7256","_type":"span","marks":[],"text":"For better performance and greater concurrency, we run SQLite with Write-Ahead Logging, which means new pages aren't written directly to the database file. Instead, they’re written to the \"write-ahead log\" or \"WAL file\"."}],"markDefs":[],"style":"normal"},{"_key":"7d472a424563","_type":"assetBlock","asset":{"_type":"asset","alignment":"center","image":{"_type":"sanityImage","alt":"Architecture diagram illustrating the difference between the database file and the write-ahead log (also known as the “WAL file”). Both are made of individual “pages”, and new pages are written to the WAL file first.","asset":{"_ref":"image-e80993ba5a6df5095f6cad9649545d52c61bd4c6-1920x1080-svg","_type":"reference"}},"mobileImage":{"_type":"sanityImage","alt":"Architecture diagram illustrating the difference between the database file and the write-ahead log (also known as the “WAL file”). Both are made of individual “pages”, and new pages are written to the WAL file first.","asset":{"_ref":"image-528a8e29dc95d00770df5eb48fdbf2340f7fdc01-3840x2160-png","_type":"reference"}},"placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":true,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"display":"block"},{"_key":"527273594761","_type":"block","children":[{"_key":"77c2c30aab8b","_type":"span","marks":[],"text":"New pages can't be written to the WAL file indefinitely; at some point they have to be copied back to the main database file. This process is called “checkpointing”."}],"markDefs":[],"style":"normal"},{"_key":"dadbc2b39dce","_type":"assetBlock","asset":{"_type":"asset","alignment":"center","image":{"_type":"sanityImage","alt":"Architecture diagram illustrating the SQLite checkpoint procedure. Pages in the WAL file are copied back into the database file. New pages can replace existing pages anywhere in the database file, or be appended to the end of the file.","asset":{"_ref":"image-9499bb50a8a15bcf41faed009237de042cf3969e-1920x1080-svg","_type":"reference"}},"mobileImage":{"_type":"sanityImage","alt":"Architecture diagram illustrating the SQLite checkpoint procedure. Pages in the WAL file are copied back into the database file. New pages can replace existing pages anywhere in the database file, or be appended to the end of the file.","asset":{"_ref":"image-f109e53dc7c5aedda6bcfce13a62454ede7bfbd2-3840x2160-png","_type":"reference"}},"placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":true,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"display":"block"},{"_key":"5a79fb1a1937","_type":"block","children":[{"_key":"1cfca15d1c61","_type":"span","marks":[],"text":"In most deployments, SQLite itself decides when to do a checkpoint, and the process is invisible to the end user and developer. In our control plane, we take manual control of the checkpoint process so we can run fast and consistent backups. This non-standard approach seemed suspicious as we steadily eliminated potential causes."}],"markDefs":[],"style":"normal"},{"_key":"5714a25c6fea","_type":"block","children":[{"_key":"79f2b53c2f6c","_type":"span","marks":[],"text":"One clue was that during corruption incidents, our metrics showed that SQLite would report copying more pages from the WAL file than were actually available. If there are 10 pages in the WAL file and 20 pages get copied to the database, something is clearly wrong."}],"markDefs":[],"style":"normal"},{"_key":"4f6e92a416fe","_type":"block","children":[{"_key":"bc86600de51a","_type":"span","marks":[],"text":"To understand what was happening during these faulty checkpoints, the SQLite developers created a new debugging tool for the virtual filesystem layer."}],"markDefs":[],"style":"normal"},{"_key":"a9642821a7f4","_type":"block","children":[{"_key":"66c7f0c0eb9f","_type":"span","marks":[],"text":"SQLite is split into several layers. The top layer is the parser and code generator, which converts SQL statements into SQLite’s internal data structures. These data structures get passed to the pager, which splits them into the individual pages to be written to disk. Actually writing them to disk is handled by the OS interface, or “virtual filesystem”. Currently SQLite has two mainstream virtual filesystem implementations—Unix and Windows."}],"markDefs":[],"style":"normal"},{"_key":"c38a20392044","_type":"assetBlock","asset":{"_type":"asset","alignment":"center","image":{"_type":"sanityImage","alt":"Architecture diagram illustrating the internals of SQLite. It receives SQL statements as input, which pass through three layers: the parser/code generator, the pager, and the OS interface/virtual filesystem. The filesystem layer writes the changes to disk.","asset":{"_ref":"image-fc1ab3b4825fab7a103b2023b8122f6f6c8f258c-1920x1080-svg","_type":"reference"}},"mobileImage":{"_type":"sanityImage","alt":"Architecture diagram illustrating the internals of SQLite. It receives SQL statements as input, which pass through three layers: the parser/code generator, the pager, and the OS interface/virtual filesystem. The filesystem layer writes the changes to disk.","asset":{"_ref":"image-d7dfb88f44c953799b22c0a55cb820f477e85071-3840x2160-png","_type":"reference"}},"placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":true,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"display":"block"},{"_key":"456fb2efcc3b","_type":"block","children":[{"_key":"69854de211b3","_type":"span","marks":[],"text":"If you're interested in a deeper dive on these internals, I recommend "},{"_key":"7d00c193927a","_type":"span","marks":["2ffa13494ef0"],"text":"this lecture by Richard Hipp"},{"_key":"6a68f173297f","_type":"span","marks":[],"text":", the primary author of SQLite."}],"markDefs":[{"_key":"2ffa13494ef0","_type":"link","href":"https://www.youtube.com/watch?v=ZSKLA81tBis"}],"style":"normal"},{"_key":"b6280570fce8","_type":"block","children":[{"_key":"c83f278cc5d3","_type":"span","marks":[],"text":"This approach allows you to replace different layers with different implementations, or wrap an existing layer to get more information. To help diagnose our problem, the SQLite developers created a wrapper around the virtual filesystem that writes additional tracing information and logs about changes to the database. This wrapper is called the "},{"_key":"24ab4081ebea","_type":"span","marks":["code"],"text":"tmstmpvfs"},{"_key":"5635abaf59c3","_type":"span","marks":[],"text":" shim, and the source code is available in the "},{"_key":"243451896e81","_type":"span","marks":["465059c6be42"],"text":"SQLite public repository"},{"_key":"744087aa3127","_type":"span","marks":[],"text":"."}],"markDefs":[{"_key":"465059c6be42","_type":"link","href":"https://sqlite.org/src/file/ext/misc/tmstmpvfs.c?proof=394548354"}],"style":"normal"},{"_key":"c6c9bbb53545","_type":"assetBlock","asset":{"_type":"asset","alignment":"center","image":{"_type":"sanityImage","alt":"Architecture diagram illustrating the internals of SQLite with our new debugging layer. The OS interface/virtual filesystem layer has now been wrapped in a tmstmpvfs shim.","asset":{"_ref":"image-2c42cf49f1419d6d8c07a56873b7ef1d327d1efe-1920x1080-svg","_type":"reference"}},"mobileImage":{"_type":"sanityImage","alt":"Architecture diagram illustrating the internals of SQLite with our new debugging layer. The OS interface/virtual filesystem layer has now been wrapped in a tmstmpvfs shim.","asset":{"_ref":"image-4daa62707323a152dbfaa67f38ce03f90a812095-3840x2160-png","_type":"reference"}},"placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":true,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"display":"block"},{"_key":"f17ac1cd6499","_type":"block","children":[{"_key":"3b0da56fd896","_type":"span","marks":[],"text":"We deployed the shim into our live environment, and waited for the next corruption to occur. Fortunately, we didn't have to wait long."}],"markDefs":[],"style":"normal"},{"_key":"f92c3124a1cf","_type":"block","children":[{"_key":"a391791f60e1","_type":"span","marks":[],"text":"The WAL-Reset bug"}],"markDefs":[],"style":"h2"},{"_key":"823893d41bea","_type":"block","children":[{"_key":"cf5293e1b676","_type":"span","marks":[],"text":"After our next corruption incident, the additional logs from the new "},{"_key":"cfcf2adbaabc","_type":"span","marks":["code"],"text":"tmstmpvfs"},{"_key":"163adfd93262","_type":"span","marks":[],"text":" shim allowed the SQLite developers to find and fix the bug: a rare data race in the SQLite source code between a checkpoint and a write transaction."}],"markDefs":[],"style":"normal"},{"_key":"b5e82c7eb106","_type":"block","children":[{"_key":"917e921624ee","_type":"span","marks":[],"text":"In particular, if a write occurs at a specific time during a checkpoint, the checkpointing process gets confused—it thinks some of the pages have been copied from the WAL into the main database file, but they haven’t. Those pages never get written to the database file, and that data is permanently lost. The database file becomes corrupt, because other pages which reference those pages—such as an index—are written to the database."}],"markDefs":[],"style":"normal"},{"_key":"99aaf6735c61","_type":"block","children":[{"_key":"832934fa28a5","_type":"span","marks":[],"text":"The SQLite developers named this the "},{"_key":"463adddce01b","_type":"span","marks":["d8e102b2143a"],"text":"“WAL-Reset bug”"},{"_key":"17f3196172a2","_type":"span","marks":[],"text":", and they estimate it was present in SQLite for at least 16 years. It could exist that long because it was rare—so rare, the SQLite developers had to add code to deliberately trigger it in their testing environments. Their fix adds "},{"_key":"d4207f7a0431","_type":"span","marks":["b48bcd81f029"],"text":"an additional check to the checkpointing function"},{"_key":"2a0fb0ea5b15","_type":"span","marks":[],"text":" which detects when the WAL has been reset by another thread."}],"markDefs":[{"_key":"d8e102b2143a","_type":"link","href":"https://sqlite.org/wal.html#the_wal_reset_bug"},{"_key":"b48bcd81f029","_type":"link","href":"https://sqlite.org/src/info/7168988acbec2d8d"}],"style":"normal"},{"_key":"4763e5d51ea6","_type":"block","children":[{"_key":"90d13e367514","_type":"span","marks":[],"text":"They confirmed that this bug caused all of the baffling behaviour we’d seen. It explained the corruption, the transaction logs that wouldn’t apply cleanly, and the inconsistent checkpoint statistics. They also explained why we were more likely to hit the bug than other SQLite users: we take manual control of the checkpointing process, and we checkpoint very aggressively. Even a bug triggered by a rare condition was bound to hit us eventually."}],"markDefs":[],"style":"normal"},{"_key":"54917174af4a","_type":"block","children":[{"_key":"cc3cf5dbf6c2","_type":"span","marks":[],"text":"This was an exciting moment. After months of confusion and uncertainty, we finally had a plausible theory for why the corruption was occurring, and a fix we could deploy to prevent it."}],"markDefs":[],"style":"normal"},{"_key":"11e7564524c2","_type":"block","children":[{"_key":"deee9abdeda8","_type":"span","marks":[],"text":"The SQLite developers released the fix as "},{"_key":"a4fc657ab34d","_type":"span","marks":["667a4ba65cdc"],"text":"SQLite 3.52.0"},{"_key":"9ef0096861e6","_type":"span","marks":[],"text":", and we prepared to deploy it as soon as it was available."}],"markDefs":[{"_key":"667a4ba65cdc","_type":"link","href":"https://sqlite.org/changes.html#version_3_52_0"}],"style":"normal"},{"_key":"fc2fb700d3f4","_type":"block","children":[{"_key":"d077831f17b0","_type":"span","marks":[],"text":"Fixed, with a false alarm"}],"markDefs":[],"style":"h2"},{"_key":"f76c1b924587","_type":"block","children":[{"_key":"f6c730a3ae1b","_type":"span","marks":[],"text":"We rolled out SQLite 3.52.0 carefully—first to a few canary shards, then, when we saw it running smoothly, we deployed it to the rest of the control plane."}],"markDefs":[],"style":"normal"},{"_key":"47c03433c39a","_type":"block","children":[{"_key":"09487dbf3d59","_type":"span","marks":[],"text":"Our backup monitor promptly turned red, and reported corruption in "},{"_key":"fca01ac6347e","_type":"span","marks":["strong"],"text":"13 different databases"},{"_key":"556494bfd015","_type":"span","marks":[],"text":". This was extremely alarming, but we followed our recovery procedures to fix all the supposed corruption, and everything was happy. It turned out these databases had not suffered real corruption, but were subject to a second problem in the version of SQLite."}],"markDefs":[],"style":"normal"},{"_key":"eabb0afd89ba","_type":"block","children":[{"_key":"76cb896521a7","_type":"span","marks":[],"text":"We shared our errors with the SQLite developers, which uncovered a bug in SQLite related to "},{"_key":"fcac215c7b5b","_type":"span","marks":["9e6e561ee3eb"],"text":"stale expression indexes"},{"_key":"05f8639bc3a4","_type":"span","marks":[],"text":". If you create an index on a computed value, and then the computation changes, the index will contain mismatched values, which gets reported as corruption by "},{"_key":"277da16a058a","_type":"span","marks":["code"],"text":"PRAGMA integrity_check"},{"_key":"2b33e9ef4a12","_type":"span","marks":[],"text":"."}],"markDefs":[{"_key":"9e6e561ee3eb","_type":"link","href":"https://sqlite.org/staleexpridx.html"}],"style":"normal"},{"_key":"ea549bec6e80","_type":"block","children":[{"_key":"2337ac84037c","_type":"span","marks":[],"text":"In our case, we were storing some high-precision timestamps as text, converting them to a floating-point number in a VIRTUAL generated column, and the SQLite 3.52.0 release that fixed our data race also made "},{"_key":"ceffd424514c","_type":"span","marks":["bbc01625293d"],"text":"an optimisation"},{"_key":"6f6088804cdc","_type":"span","marks":[],"text":" that subtly changed the rounding behaviour for text-to-floating-point conversions. Our canary shards didn’t have any timestamps that triggered the changed rounding behaviour, so we missed this in our phased rollout."}],"markDefs":[{"_key":"bbc01625293d","_type":"link","href":"https://research.swtch.com/fp"}],"style":"normal"},{"_key":"34a9c965ef96","_type":"block","children":[{"_key":"733766fa5a54","_type":"span","marks":[],"text":"Because this change caused false corruption warnings, the SQLite developers withdrew the 3.52.0 release and instead published 3.51.3, which only contained a fix for the WAL-Reset bug."}],"markDefs":[],"style":"normal"},{"_key":"1c761050267a","_type":"block","children":[{"_key":"b2c3a18b5723","_type":"span","marks":[],"text":"We fixed the issue on our side by reducing the precision of our timestamps to integer seconds; text-to-integer conversions are unambiguous. Meanwhile, the SQLite developers created an automated, "},{"_key":"76a215c01bcf","_type":"span","marks":["5818cfbdcf39"],"text":"self-healing index feature"},{"_key":"dbec10027a53","_type":"span","marks":[],"text":" in 3.53.0, which prevents the stale expression index problem."}],"markDefs":[{"_key":"5818cfbdcf39","_type":"link","href":"https://sqlite.org/staleexpridx.html#selfheal"}],"style":"normal"},{"_key":"5ba357581c20","_type":"block","children":[{"_key":"3d502de69b12","_type":"span","marks":[],"text":"Party time!"}],"markDefs":[],"style":"h2"},{"_key":"855cefbb05a4","_type":"block","children":[{"_key":"6d42e07151eb","_type":"span","marks":[],"text":"With the fix rolled out to our entire control plane, we were ready to declare victory, but we were still cautious. An absence of corruption incidents doesn’t mean things are fixed—we’d already had one six-week period of deceptive calm."}],"markDefs":[],"style":"normal"},{"_key":"7a997576a8d0","_type":"block","children":[{"_key":"bdf1ddf0ccc6","_type":"span","marks":[],"text":"We wanted positive proof that this data race was actively occurring in our production environment. Now that we understood the cause of the bug—a collision between a write transaction and a WAL-reset—we patched our SQLite driver to "},{"_key":"18d71f67a75b","_type":"span","marks":["bdbe1865d55e"],"text":"log a warning"},{"_key":"f72129cc317e","_type":"span","marks":[],"text":" when these two operations overlap. If the warning fired but the database remained uncorrupted, we’d know the fix had saved us from a potential corruption incident."}],"markDefs":[{"_key":"bdbe1865d55e","_type":"link","href":"https://github.com/tailscale/sqlite/commit/070c921dea03f793cbbbd9d39551d0ad26136113#diff-fcff16fa73210758d3a31f4cfa4609ad5b3ca692b66fc7fb7587c08a1ab11048"}],"style":"normal"},{"_key":"8c14fe468c8a","_type":"block","children":[{"_key":"644e14f5cfe3","_type":"span","marks":[],"text":"We deployed the warning, and we waited. And we waited. And waited. And waited. As weeks slipped by, we began to wonder why we didn’t see it. Was the warning broken? Was our theory wrong? Was the true bug still lurking in the darkness?"}],"markDefs":[],"style":"normal"},{"_key":"1747cc0f4d8f","_type":"block","children":[{"_key":"ccd5fb9edefa","_type":"span","marks":[],"text":"Then, two months later, the alert we were waiting for finally fired:"}],"markDefs":[],"style":"normal"},{"_key":"1928a177780c","_type":"assetBlock","asset":{"_type":"asset","alignment":"center","image":{"_type":"sanityImage","alt":"Alert Manager notification showing SQLitePartyMode warning: SQLite attempted corruption on shard2.corp.ts.net:8383 in party mode, but the system prevented it. Details include warning code, host, instance, job, namespace, severity, and shard information. Message advises checking server logs for corruption incident details.","asset":{"_ref":"image-19ae089ae209ad7e395e188b7f6e4a81be29c431-1458x684-png","_type":"reference"}},"placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","mobileStaticAlt":{"_type":"asset","alignment":"center","placeholderColor":{"_type":"colors","color":"none"},"type":"image","useMobileAlternativeImage":false,"video":{"clickType":"watchVideoCaption","playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"playType":"click","useMobileStaticAlt":false}},"display":"block"},{"_key":"3b8c79f45a7f","_type":"block","children":[{"_key":"536bf4b9c6dc","_type":"span","marks":[],"text":"This alert proved that the precise conditions for the WAL-Reset bug "},{"_key":"ace35ca8aef3","_type":"span","marks":["em"],"text":"do"},{"_key":"2e9250162f01","_type":"span","marks":[],"text":" occur in our production environment, which means it was the likely culprit for our six months of shaky uptime."}],"markDefs":[],"style":"normal"},{"_key":"48334f45af25","_type":"block","children":[{"_key":"36589630a26d","_type":"span","marks":[],"text":"Since that weirdly joyous alert fired, we’ve run for another four months without any database incidents, as of this writing. Finally, we could breathe a sigh of relief."}],"markDefs":[],"style":"normal"},{"_key":"5706199d24f5","_type":"block","children":[{"_key":"9ddcca682b05","_type":"span","marks":[],"text":"Off the well-trodden path"}],"markDefs":[],"style":"h2"},{"_key":"7849061b09a6","_type":"block","children":[{"_key":"58e478b1ff74","_type":"span","marks":[],"text":"Nobody wanted us to spend six months looking for bugs in SQLite. This was an immensely frustrating experience for both our customers and staff, and we’re all glad to put this instability behind us."}],"markDefs":[],"style":"normal"},{"_key":"0aa58154d9b8","_type":"block","children":[{"_key":"610915029317","_type":"span","marks":[],"text":"This investigation is a useful reminder: "},{"_key":"5ab5500fe273","_type":"span","marks":["strong"],"text":"running boring technology in a non-standard way is a risk. "},{"_key":"249a54b215f8","_type":"span","marks":[],"text":"The common paths and standard configurations are incredibly well-tested and reliable. Most people use SQLite in a standard configuration and never face this sort of issue. Everything we were doing was a public, documented, supported configuration—but by taking manual control of the checkpointing process and running at our own aggressive pace, we stepped off the well-trodden operational path."}],"markDefs":[],"style":"normal"},{"_key":"eb1975ffda8c","_type":"block","children":[{"_key":"334b821e4eb7","_type":"span","marks":[],"text":"Resolving these incidents was a massive, cross-functional effort involving dozens of people—including Tailscale's engineering and support teams, and the core maintainers of SQLite. It is to all of their credit that the impact of these incidents was not much worse."}],"markDefs":[],"style":"normal"},{"_key":"ea1fdba42ea2","_type":"block","children":[{"_key":"31c7df0e9bfb","_type":"span","marks":[],"text":"We know that repeated downtime erodes trust, no matter how many people are affected, and we’re grateful to our customers for their patience and support while we chased this down."}],"markDefs":[],"style":"normal"},{"_key":"153596ebe842","_type":"block","children":[{"_key":"f57232b4e74d","_type":"span","marks":[],"text":"Frustrating as this period was, we’re left in a stronger position than we were before. The long-standing bug in SQLite has been patched, and we fixed dozens of other incidental issues that we spotted while looking for it. We funded the open-source SQLite VFS shim that helped isolate the race condition almost immediately, and will help track down similar bugs in the future. Finally, we’ve refined our database backup and recovery processes, and live-tested them over a dozen times."}],"markDefs":[],"style":"normal"},{"_key":"cd00b3143eff","_type":"block","children":[{"_key":"8bab1308408f","_type":"span","marks":[],"text":"Hopefully there won’t be another database incident like this—but if there is, we’ll be ready."}],"markDefs":[],"style":"normal"}],"postMarkdown":null,"publishedAt":"August 12, 2026","seo":{"indexable":true,"seoDescription":"Tailscale and SQLite developers traced maddening corruption incidents to find the WAL-Reset data race, then uncovered a second stale expression index bug.","seoTitle":"How Tailscale helped find the SQLite WAL-Reset bug"},"slug":"sqlite-wal-reset-bug","summary":null,"title":"How we tracked down a 16-year-old SQLite bug","publishedAtRaw":"2026-08-12T14:00:00.000Z","readTime":18},"preview":false,"token":"","global":{"announcement":{"link":{"label":"Read the announcement to see what it means for you. →","url":"https://tailscale.com/blog/border0-joins-tailscale/?utm_source=announcement-bar\u0026utm_medium=content\u0026utm_campaign=border0-announcement"},"text":"Border0 has joined Tailscale!"},"footer":{"_createdAt":"2023-10-06T14:44:29Z","_id":"422b4abf-6e3f-4213-ab94-a03dd444be3d","_rev":"Dj7mvL0PPUTDs4i5zHw2yZ","_system":{"base":{"id":"422b4abf-6e3f-4213-ab94-a03dd444be3d","rev":"UhQG1jCdoCqTZDHLXbfgtb"}},"_type":"footer","_updatedAt":"2026-08-17T14:58:08Z","copyrightContent":"Tailscale Inc. All rights reserved. Tailscale is a registered trademark of Tailscale Inc.","cta":{"asset":{"_type":"asset","image":{"_type":"sanityImage","alt":"cta phone","asset":{"_ref":"image-90e2ca60be7a63ed677b56ad293b23f1aae42baa-362x567-svg","_type":"reference"}},"type":"image"},"ctaButton":{"_type":"button","buttonOptions":{"color":"white"},"link":{"title":"Get started","url":"/get-started"}},"darkLogoGrid":[{"_key":"fb360d1bc6c4","logo":{"_type":"sanityImage","alt":"mercury","asset":{"_ref":"image-7dfc2f1ef3cc6eafeee143c3e7118d563f87e744-199x81-svg","_type":"reference"}}},{"_key":"a24139987731","logo":{"_type":"sanityImage","alt":"instacart","asset":{"_ref":"image-f8d8890fb95865d1ffb39df26e827b7e768a7e17-199x81-svg","_type":"reference"}}},{"_key":"0fa57e2eebee","logo":{"_type":"sanityImage","alt":"Retool","asset":{"_ref":"image-29af7cda4ae0ac0698a9de4daac73e0ad4c78f39-199x82-svg","_type":"reference"}}},{"_key":"b47251ca28bd","logo":{"_type":"sanityImage","alt":"duolingo","asset":{"_ref":"image-4a3db2857abb3613ead0308c5ce6ce1f7ba71932-199x81-svg","_type":"reference"}}}],"heading":"Try Tailscale for |free|","logoGrid":[{"_key":"a3a9b2012378","logo":{"_type":"sanityImage","alt":"mercury","asset":{"_ref":"image-4590b93f6cc3b4573e5cd5cefe90e62f935e5aa7-199x81-svg","_type":"reference"}}},{"_key":"993b75d39e13","logo":{"_type":"sanityImage","alt":"instacrt","asset":{"_ref":"image-698d5ead889b4e5a57f60549e74102b82ad7aae8-199x81-svg","_type":"reference"}}},{"_key":"8449f10eb5c7","logo":{"_type":"sanityImage","alt":"Retool","asset":{"_ref":"image-258b299c38fba23800176525d02e7ce713f74820-199x82-svg","_type":"reference"}}},{"_key":"3ab303288a39","logo":{"_type":"sanityImage","alt":"duolingo","asset":{"_ref":"image-b74adcdab85e2515c997140d08827f084bccfa2a-199x81-svg","_type":"reference"}}}],"secondaryCta":{"heading":"Schedule a demo","link":{"title":"Contact sales","url":"/contact/sales"}},"textCard":{"_type":"textCard","heading":"Try Tailscale for |free|","links":[{"_key":"dc04d805f7e0","button":{"_type":"button","buttonOptions":{"color":"white"},"link":{"title":"Get started","url":"https://login.tailscale.com/start"}},"textLink":{"_type":"textLink","textLinkOptions":{"arrowColor":"black","underlineColor":"black"}},"type":"button"}],"options":{"contentFontColor":"white","hasMobileTextAlignment":false,"headingFontColor":"white","headingFontSize":"h3","headingMarginBottom":"30","headingMaxWidth":292,"highlightColor":"blue-3","sectionAlignment":"left"}}},"footerNav":[{"_key":"05f3fa61c972","heading":"Product","links":[{"_key":"30386cf08177","title":"How it works","url":"/blog/how-tailscale-works/"},{"_key":"45dec9531713","title":"Pricing","url":"/pricing"},{"_key":"e6f4d8daff21","title":"Integrations","url":"/integrations"},{"_key":"d4f7875a767f","title":"Features","url":"/features"},{"_key":"64846fcdaf3b","title":"Compare Tailscale","url":"/compare"}]},{"_key":"7870d03d9802","heading":"Use Cases","links":[{"_key":"7b4858603fc7","title":"Business VPN","url":"/use-cases/business-vpn"},{"_key":"06fbf46e9354","title":"CI/CD","url":"/use-cases/ci-cd"},{"_key":"602f747f5b2a","title":"Infra Access","url":"/use-cases/infrastructure-access"},{"_key":"8660f39ec574","title":"Cloud Connectivity","url":"/use-cases/cloud-connectivity"},{"_key":"ab3e69241df2","title":"Zero Trust Networking","url":"/use-cases/zero-trust-networking"},{"_key":"b79f544a8266","title":"Homelab","url":"/use-cases/homelab"}]},{"_key":"2e262725243d","heading":"Resources","links":[{"_key":"b5ad8866742c","title":"Blog","url":"/blog"},{"_key":"21869f26f11b","title":"Events \u0026 Webinars","url":"/events-webinars"},{"_key":"c844ea072844","title":"Partnerships","url":"/partnerships"}]},{"_key":"a1e16018d519","heading":"Company","links":[{"_key":"8cc3fedb5b31","title":"Company","url":"/company"},{"_key":"e69d139c2c7c","title":"Careers","url":"/careers"},{"_key":"ad370d7ab2c1","title":"Press","url":"/press"}]},{"_key":"b25bd2c7203e","heading":"Help \u0026 Support","links":[{"_key":"f7d6ef6a99c6","title":"Support","url":"/contact/support"},{"_key":"18077954da8f455140153a58c74e53ba","title":"Sales","url":"/contact/sales"},{"_key":"3b91a6bb3d6b","title":"Security","url":"/security"},{"_key":"9d3e837341e2","title":"Legal","url":"/legal"},{"_key":"a69304fe5b80","title":"Open Source","url":"/opensource"},{"_key":"a02943ca7fdd","title":"Changelog","url":"/changelog"},{"_key":"94a676559b98","title":"Tailscale Status","url":"https://status.tailscale.com/"}]},{"_key":"0bdaf34fbe61","heading":"Learn","links":[{"_key":"6c45141fcc65","title":"SSH keys","url":"/learn/generate-ssh-keys/"},{"_key":"86c070f995c4","title":"Docker SSH","url":"/learn/ssh-into-docker-container/"},{"_key":"22e6d051e763","title":"NAT Traversal","url":"/blog/how-nat-traversal-works/"},{"_key":"4e51a8a4f0a7","title":"MagicDNS","url":"/blog/2021-09-private-dns-with-magicdns/"},{"_key":"f8f8893085b3","title":"PAM","url":"/learn/privileged-access-management/"},{"_key":"e7fdb19bd312","title":"All articles","url":"/learn"}]}],"legalContent":"WireGuard is a registered trademark of Jason A. Donenfeld.","legalNav":[{"_key":"b3b1d8dfddea","title":"Terms of Service","url":"/terms"},{"_key":"3e22b7802445","title":"Privacy Policy","url":"/privacy-policy"},{"_key":"78a1108d49e9","title":"California Notice","url":"https://tailscale.com/privacy-policy#california-notice"},{"_key":"9d3370545d40","title":"Cookie Notice","url":"https://tailscale.com/cookie-notice"}],"title":"Production Footer"},"globalOptions":null,"header":{"_createdAt":"2023-10-06T12:21:23Z","_id":"7797109d-2dc4-4a75-b5a3-b1019c33212f","_rev":"JgNZwVoOE3fVRWVQtDlthq","_system":{"base":{"id":"7797109d-2dc4-4a75-b5a3-b1019c33212f","rev":"5FsRtrxmPymeiFerrOYlEk"}},"_type":"header","_updatedAt":"2026-04-28T20:49:26Z","button":{"_type":"button","buttonOptions":{"color":"black"},"link":{"title":"Get started - it's free!","url":"https://login.tailscale.com/start"}},"links":[{"_key":"157b4ad1150d","title":"Download","url":"/download"},{"_key":"f00209e74f6b","title":"Log in","url":"https://login.tailscale.com/welcome"}],"menu":[{"_key":"95381f81d527","hasSubmenu":true,"submenu":{"product":{"leftCol":{"topNav":{"heading":"Meet Tailscale","links":[{"_key":"5495d201056a","icon":{"_type":"sanityImage","alt":"icon"},"link":"/blog/how-tailscale-works/","title":"How it works"},{"_key":"dc9cde7ff83cb94cfc98ff29bdcd0997","icon":{"_type":"sanityImage","alt":"icon"},"link":"/why-tailscale","title":"Why Tailscale"},{"_key":"5d88e3ffcc6b","icon":{"_type":"sanityImage","alt":"WireGuard®"},"link":"/wireguard-vpn","title":"WireGuard® for Enterprises"},{"_key":"435de37ddd5f","icon":{"_type":"sanityImage","alt":"Bring Tailscale to Work"},"link":"/bring-tailscale-to-work","title":"Bring Tailscale to Work"}]}},"rightCol":{"nav":{"heading":"Explore","links":[{"_key":"c653da519dfb","link":"/integrations","title":"Integrations"},{"_key":"a878da5fa54c","link":"/features","title":"Features"},{"_key":"adda698ed879","link":"/compare","title":"Compare Tailscale"},{"_key":"3341bd9805df","link":"/community/community-projects","title":"Community Projects"},{"_key":"b57369965809","link":"/partnerships","title":"Partnerships"}]}}},"submenuType":"product"},"title":"Product"},{"_key":"a7062f1924df","hasSubmenu":true,"submenu":{"product":{"leftCol":{"topNav":{"heading":"By use-case","links":[{"_key":"6fc65e9fe1c6","link":"/use-cases/business-vpn","title":"Business VPN"},{"_key":"193eaaa0cef8","link":"/use-cases/ci-cd","title":"CI/CD"},{"_key":"c59997229dde","link":"/use-cases/infrastructure-access","title":"Infra Access"},{"_key":"05cadfcf3e65b04708a9d88060f68f9e","link":"/use-cases/cloud-connectivity","title":"Cloud Connectivity"},{"_key":"6a363d694952","link":"/use-cases/zero-trust-networking","title":"Zero Trust Networking"},{"_key":"d99d14013ab3","link":"/use-cases/homelab","title":"Homelab"},{"_key":"b049143e7a1c","link":"/use-cases/securing-ai","title":"Securing AI"}]}},"rightCol":{"nav":{"heading":"By role","links":[{"_key":"502a00f49baf","link":"/solutions/devops","title":"DevOps"},{"_key":"0fe4c0d6fa83","link":"/solutions/it","title":"IT"},{"_key":"026f30b876a7","link":"/solutions/security","title":"Security"}]}}},"submenuType":"product"},"title":"Solutions"},{"_key":"fd055b16290df04c6012d0d33c2fad13","hasSubmenu":false,"link":"/enterprise","submenu":{"submenuType":"product"},"title":"Enterprise"},{"_key":"b595975539c7407a7ed4510edd549223","hasSubmenu":false,"link":"/customers","submenu":{"product":{"leftCol":{"topNav":{"heading":"Nav heading here","links":[{"_key":"2d22491d8262","description":"How Cribl Enables Secure Work From Anywhere with Tailscale","icon":{"_type":"sanityImage","alt":"Alt text ","asset":{"_ref":"image-a06dc612b1e3e4f4df53a72030002600639a8738-300x120-png","_type":"reference"}},"link":"https://tailscale.com/customers","title":"Title here"}]}}},"resources":{"topNav":[{"_key":"61d0f0cb130e","description":"How Cribl Enables Secure Work From Anywhere with Tailscale","heading":"Cribl"},{"_key":"712684d509a6a6ea07ab9401bdb23f8f","description":"How Cribl Enables Secure Work From Anywhere with Tailscale","heading":"Cribl"},{"_key":"ceac3f234a3a6923a671af91772b7e8b","description":"How Cribl Enables Secure Work From Anywhere with Tailscale","heading":"Cribl"}]},"submenuType":"product"},"title":"Customers"},{"_key":"f06fabeb084c","hasSubmenu":false,"link":"/docs","submenu":{"submenuType":"product"},"title":"Docs"},{"_key":"f2537b6fa068","hasSubmenu":false,"link":"/blog","submenu":{"submenuType":"product"},"title":"Blog"},{"_key":"e1b7b44dc091","hasSubmenu":false,"link":"/pricing","submenu":{"submenuType":"product"},"title":"Pricing"}],"title":"Production Header"},"legal":null,"newsBar":null,"redirects":[{"_key":"8b0a3ebcf822","destination":"/wireguard-vpn","source":"/wireguard"}],"seo":{"ogImage":{"_type":"image","asset":{"_ref":"image-8e0455b2d9b33c6151016afdf2ea81d7623c2f04-1200x628-png","_type":"reference"}}},"socials":null,"newHeader":{"links":[{"feature":{"buttons":[{"_key":"eda43f90569a","_type":"buttonsMember","openInNewTab":false,"text":"Click here","url":"https://tailscale.com/use-cases/securing-ai"}],"contentText":"Unified AI governance for AI agents and users. ","image":{"alt":"aperture dashboard","asset":{"_createdAt":"2026-04-10T14:17:26Z","_id":"image-a88cf9aad8777f4b0641202a5ed6c02bbb005d8c-266x342-svg","_rev":"v1HLz1pJ8t2rE6sWIKe7CE","_type":"sanity.imageAsset","_updatedAt":"2026-04-10T14:17:26Z","assetId":"a88cf9aad8777f4b0641202a5ed6c02bbb005d8c","extension":"svg","metadata":{"_type":"sanity.imageMetadata","blurHash":"d,Ju7Txuxu%2t7RkWBay~qRjRjWBM{s:ofj[xuocWBWB","dimensions":{"_type":"sanity.imageDimensions","aspectRatio":0.7777777777777778,"height":342,"width":266},"hasAlpha":true,"isOpaque":true,"lqip":"data:image/jpeg;base64,/9j/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQYGBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYaKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wAARCAAaABQDASIAAhEBAxEB/8QAGQABAQADAQAAAAAAAAAAAAAABwACBQgG/8QAKBAAAQIEBAYDAQAAAAAAAAAAAQIDAAQFBhESIVQHFRdxk9FBQnOS/8QAFgEBAQEAAAAAAAAAAAAAAAAAAAEC/8QAGBEAAwEBAAAAAAAAAAAAAAAAAAERAjH/2gAMAwEAAhEDEQA/AECsVe1KS661NtS6XGyApOTXWNjILtWZYC3HqWyT9VLTAXxvxavibSsHDKnQHD4g8ClBOYZ8O8a08viCq6dflu0t7SP7TFHIALhGIz4d4ozC0eOMnDW57ivKZnqPJIdllpSEqKwNQI8L0YvoN5eWt4foIXRUZ3eTPlV7jI1Gd3kz5Ve4gB7ozfQ05c35BFC8ajO7yZ8qvcUKD//Z","palette":{"_type":"sanity.imagePalette","darkMuted":{"_type":"sanity.imagePaletteSwatch","background":"#2f3550","foreground":"#fff","population":0.3,"title":"#fff"},"darkVibrant":{"_type":"sanity.imagePaletteSwatch","background":"#2c3a6f","foreground":"#fff","population":0.59,"title":"#fff"},"dominant":{"_type":"sanity.imagePaletteSwatch","background":"#7d7d7d","foreground":"#fff","population":1.59,"title":"#fff"},"lightMuted":{"_type":"sanity.imagePaletteSwatch","background":"#a9b4d4","foreground":"#000","population":0.1,"title":"#fff"},"lightVibrant":{"_type":"sanity.imagePaletteSwatch","background":"#7b9dcc","foreground":"#000","population":0.38,"title":"#fff"},"muted":{"_type":"sanity.imagePaletteSwatch","background":"#7d7d7d","foreground":"#fff","population":1.59,"title":"#fff"},"vibrant":{"_type":"sanity.imagePaletteSwatch","background":"#6591ea","foreground":"#fff","population":1.15,"title":"#fff"}},"thumbHash":"4geaBAAPmZl393eGCRaicBZ+cHB4eHiHBw=="},"mimeType":"image/svg+xml","originalFilename":"nav-platform-cta-aperture.svg","path":"images/w77i7m8x/production-v2/a88cf9aad8777f4b0641202a5ed6c02bbb005d8c-266x342.svg","sha1hash":"a88cf9aad8777f4b0641202a5ed6c02bbb005d8c","size":1123481,"uploadId":"hcqthI445Uju66Y2TYlMRwwG2aXIeXYC","url":"https://cdn.sanity.io/images/w77i7m8x/production-v2/a88cf9aad8777f4b0641202a5ed6c02bbb005d8c-266x342.svg"}},"title":"Aperture by Tailscale"},"label":"Platform","tray":{"buttons":[{"_key":"3388b94d369d","_type":"buttonsMember","openInNewTab":true,"text":"Make the case","url":"https://tailscale.com/bring-tailscale-to-work"}],"contentText":"See how Tailscale fits the systems your team already runs.","title":"Bring Tailscale to work"}},{"feature":null,"label":"Customer Stories","tray":null},{"feature":null,"label":"Community","tray":{"buttons":[{"_key":"06c200585c0f","_type":"buttonsMember","openInNewTab":true,"text":"Make the case","url":"https://tailscale.com/bring-tailscale-to-work"}],"contentText":"Start with your own devices. Bring the same simplicity to your team.","title":"Bring Tailscale to work"}},{"feature":null,"label":"Partnerships","tray":null}],"title":"Production Header"},"newAnnouncement":{"icon":"sparkles","link":{"label":"Read the recap →","url":"https://tailscale.com/tailscaleup-date-week-26"},"text":"See everything you missed at TailscaleUp.","themeColor":"purple","title":"Production Announcement"}},"slug":"blog/sqlite-wal-reset-bug"},"__N_SSG":true},"page":"/blog/[slug]","query":{"slug":"sqlite-wal-reset-bug"},"buildId":"YcO3-STY7pBVXk7rkvXgF","isFallback":false,"isExperimentalCompile":false,"gsp":true,"scriptLoader":[]}</script></body></html>