Files
nexus/sreweekly/articles/415/02-investigating-and-optimizing-over-querying.html
2026-09-12 17:23:01 +08:00

232 lines
104 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8" nonce="goa3e761xWwsor0s+F8w/g=="/><meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="preload" as="image" href="https://cms.readyset.io/files/articles/65cb6bb6fbd2870001265837/c710994e211d520d8fe0dd35.png"/><link rel="preload" as="image" href="https://cms.readyset.io/files/articles/65cb6bb6fbd2870001265837/1fe58061ad314ed7a931938d.bin"/><link rel="preload" as="image" href="https://cms.readyset.io/files/articles/65cb6bb6fbd2870001265837/2734aa9275276207973d0dba.bin"/><link rel="stylesheet" href="/assets/styles-Dkt3QWRo.css" nonce="goa3e761xWwsor0s+F8w/g==" data-precedence="default"/><meta property="csp-nonce" content="goa3e761xWwsor0s+F8w/g=="/><title>How to Solve for the N+1 Query Problem: Investigating and Optimizing Over-Querying | Readyset</title><meta name="theme-color" content="#ffffff" nonce="goa3e761xWwsor0s+F8w/g=="/><meta name="msapplication-TileColor" content="#161615" nonce="goa3e761xWwsor0s+F8w/g=="/><meta name="apple-mobile-web-app-capable" content="yes" nonce="goa3e761xWwsor0s+F8w/g=="/><meta name="mobile-web-app-capable" content="yes" nonce="goa3e761xWwsor0s+F8w/g=="/><meta name="apple-mobile-web-app-status-bar-style" content="black" nonce="goa3e761xWwsor0s+F8w/g=="/><meta name="description" content="Imagine you&#x27;re running a popular e-commerce online bookstore that offers a vast collection of titles and authors to a growing user base. However, you&#x27;ve noticed a troubling trend over the past few months: the website..." nonce="goa3e761xWwsor0s+F8w/g=="/><meta name="application-name" content="Readyset | Drop-in SQL Caching for PostgreSQL and MySQL" nonce="goa3e761xWwsor0s+F8w/g=="/><meta name="apple-mobile-web-app-title" content="Readyset | Drop-in SQL Caching for PostgreSQL and MySQL" nonce="goa3e761xWwsor0s+F8w/g=="/><meta name="keywords" content="SQL caching, real-time cache, database cache, PostgreSQL cache, MySQL cache" nonce="goa3e761xWwsor0s+F8w/g=="/><meta property="og:type" content="article" nonce="goa3e761xWwsor0s+F8w/g=="/><meta property="og:site_name" content="Readyset | Drop-in SQL Caching for PostgreSQL and MySQL" nonce="goa3e761xWwsor0s+F8w/g=="/><meta property="og:title" content="How to Solve for the N+1 Query Problem: Investigating and Optimizing Over-Querying | Readyset" nonce="goa3e761xWwsor0s+F8w/g=="/><meta property="og:description" content="Imagine you&#x27;re running a popular e-commerce online bookstore that offers a vast collection of titles and authors to a growing user base. However, you&#x27;ve noticed a troubling trend over the past few months: the website..." nonce="goa3e761xWwsor0s+F8w/g=="/><meta property="og:image" content="https://readyset.io/api/og/blog/investigating-and-optimizing-over-querying?v=1777903709106" nonce="goa3e761xWwsor0s+F8w/g=="/><meta property="og:image:secure_url" content="https://readyset.io/api/og/blog/investigating-and-optimizing-over-querying?v=1777903709106" nonce="goa3e761xWwsor0s+F8w/g=="/><meta property="og:image:type" content="image/png" nonce="goa3e761xWwsor0s+F8w/g=="/><meta property="og:image:alt" content="How to Solve for the N+1 Query Problem: Investigating and Optimizing Over-Querying | Readyset" nonce="goa3e761xWwsor0s+F8w/g=="/><meta name="twitter:card" content="summary_large_image" nonce="goa3e761xWwsor0s+F8w/g=="/><meta name="twitter:site" content="@readysetio" nonce="goa3e761xWwsor0s+F8w/g=="/><meta name="twitter:creator" content="@readysetio" nonce="goa3e761xWwsor0s+F8w/g=="/><meta name="twitter:title" content="How to Solve for the N+1 Query Problem: Investigating and Optimizing Over-Querying | Readyset" nonce="goa3e761xWwsor0s+F8w/g=="/><meta name="twitter:description" content="Imagine you&#x27;re running a popular e-commerce online bookstore that offers a vast collection of titles and authors to a growing user base. However, you&#x27;ve noticed a troubling trend over the past few months: the website..." nonce="goa3e761xWwsor0s+F8w/g=="/><meta name="twitter:image" content="https://readyset.io/api/og/blog/investigating-and-optimizing-over-querying?v=1777903709106" nonce="goa3e761xWwsor0s+F8w/g=="/><meta property="og:image:width" content="1200" nonce="goa3e761xWwsor0s+F8w/g=="/><meta property="og:image:height" content="630" nonce="goa3e761xWwsor0s+F8w/g=="/><meta property="og:url" content="https://readyset.io/blog/investigating-and-optimizing-over-querying" nonce="goa3e761xWwsor0s+F8w/g=="/><meta name="twitter:url" content="https://readyset.io/blog/investigating-and-optimizing-over-querying" nonce="goa3e761xWwsor0s+F8w/g=="/><meta property="article:author" content="Readyset" nonce="goa3e761xWwsor0s+F8w/g=="/><meta property="article:published_time" content="2024-02-14" nonce="goa3e761xWwsor0s+F8w/g=="/><meta property="article:section" content="PostgreSQL" nonce="goa3e761xWwsor0s+F8w/g=="/><meta property="article:tag" content="PostgreSQL" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="modulepreload" href="/assets/index-CgozBq9o.js" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="modulepreload" href="/assets/_slug-feOgb-yl.js" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="modulepreload" href="/assets/article-card-BslrmY68.js" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="modulepreload" href="/assets/newsletter-popup-C0xZnHQP.js" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="modulepreload" href="/assets/rich-markdown-DYApPWH1.js" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="modulepreload" href="/assets/index-rMShz_NE.js" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="modulepreload" href="/assets/use-disclosure-DaTsKJMJ.js" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="icon" href="/favicon.ico" sizes="any" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="icon" type="image/png" sizes="16x16" href="/manifest/favicon-16x16.png" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="icon" type="image/png" sizes="32x32" href="/manifest/favicon-32x32.png" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="apple-touch-icon" sizes="180x180" href="/manifest/apple-touch-icon.png" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="mask-icon" href="/manifest/safari-pinned-tab.svg" color="#161615" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="manifest" href="/manifest.webmanifest" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="prefetch" href="/icons/sprite-stroke.svg" as="image" type="image/svg+xml" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="prefetch" href="/icons/sprite-bulk.svg" as="image" type="image/svg+xml" nonce="goa3e761xWwsor0s+F8w/g=="/><link rel="canonical" href="https://readyset.io/blog/investigating-and-optimizing-over-querying" nonce="goa3e761xWwsor0s+F8w/g=="/><script type="application/ld+json" nonce="goa3e761xWwsor0s+F8w/g==">{"@context":"https://schema.org","@type":"Organization","@id":"https://readyset.io/#organization","name":"Readyset","url":"https://readyset.io","logo":"https://readyset.io/manifest/apple-touch-icon.png","description":"Increase the scale of your PostgreSQL and MySQL deployment by up to 100x with Readyset - all without modifying your application code or database. Start using Readyset today for free!","sameAs":["https://x.com/readysetio"]}</script><script type="application/ld+json" nonce="goa3e761xWwsor0s+F8w/g==">{"@context":"https://schema.org","@type":"WebSite","@id":"https://readyset.io/#website","url":"https://readyset.io","name":"Readyset","publisher":{"@id":"https://readyset.io/#organization"}}</script><script type="application/ld+json" nonce="goa3e761xWwsor0s+F8w/g==">{"@context":"https://schema.org","@type":"Article","headline":"How to Solve for the N+1 Query Problem: Investigating and Optimizing Over-Querying","description":"Imagine you're running a popular e-commerce online bookstore that offers a vast collection of titles and authors to a growing user base. However, you've noticed a troubling trend over the past few months: the website...","url":"https://readyset.io/blog/investigating-and-optimizing-over-querying","image":"https://readyset.io/images/blog-placeholder.svg","datePublished":"2024-02-14T00:00:00.000Z","dateModified":"2026-05-04T14:08:29.106Z","author":{"@type":"Person","name":"Readyset"},"publisher":{"@id":"https://readyset.io/#organization"}}</script></head><body class="min-h-screen bg-surface-layout-2"><a href="#main-content" class="sr-only focus:not-sr-only focus:absolute focus:left-4 focus:top-4 focus:z-[100] focus:rounded-lg focus:bg-content-layout-1 focus:px-4 focus:py-2 focus:text-sm focus:font-semibold focus:text-surface-layout-1">Skip to content</a><header class="sticky top-0 z-50 bg-surface-layout-2/80 backdrop-blur-lg border-b border-border-layout-soft"><div class="relative max-w-7xl mx-auto px-6 py-4 flex items-center justify-between"><div class="flex items-center gap-8"><a href="/" class="flex items-center"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 145 33" class="fill-current text-content-layout-1 h-6"><path d="M16 31.328c0 .736-.597 1.333-1.333 1.333H4a4 4 0 0 1-4-4V17.994c0-.736.597-1.333 1.333-1.333H16v14.667Z"></path><path d="M21.333 11.328v5.333h-4c-.736 0-1.333.597-1.333 1.333v4h-5.333v-5.333h4c.736 0 1.333-.597 1.333-1.333v-4h5.333Z"></path><path d="M32 15.328c0 .736-.597 1.333-1.333 1.333H16V1.994c0-.736.597-1.333 1.333-1.333H28a4 4 0 0 1 4 4v10.667Z"></path><path fill-rule="evenodd" d="M91.272 5.367h2.864V23.25h-2.864v-1.99c-1.145 1.45-2.665 2.175-4.557 2.175-1.204 0-2.296-.294-3.275-.882a6.299 6.299 0 0 1-2.304-2.45c-.548-1.036-.822-2.193-.822-3.47 0-1.276.274-2.433.822-3.469a6.209 6.209 0 0 1 2.304-2.437c.98-.597 2.071-.896 3.275-.896 1.9 0 3.42.722 4.557 2.164V5.367Zm-3.997 15.308c1.146 0 2.096-.386 2.852-1.157.763-.779 1.145-1.74 1.145-2.885 0-1.152-.382-2.114-1.145-2.884-.756-.78-1.706-1.17-2.852-1.17-1.154 0-2.117.39-2.889 1.17-.764.77-1.145 1.732-1.145 2.884 0 1.145.381 2.106 1.145 2.885.772.771 1.735 1.157 2.89 1.157Zm-24.258-2.898c.05-.364.075-.737.075-1.119 0-1.956-.652-3.581-1.955-4.874-1.295-1.302-2.918-1.953-4.87-1.953-1.278 0-2.444.295-3.498.883a6.442 6.442 0 0 0-2.466 2.438c-.597 1.028-.896 2.172-.896 3.432 0 1.948.68 3.577 2.042 4.887 1.37 1.31 3.084 1.965 5.143 1.965 1.203 0 2.32-.24 3.35-.722 1.037-.48 1.892-1.144 2.564-1.99l-1.93-1.653c-.39.514-.938.94-1.643 1.28-.698.34-1.453.51-2.267.51-1.054 0-1.959-.277-2.714-.833-.747-.563-1.237-1.314-1.47-2.25h10.535Zm-9.19-4.538c.69-.556 1.494-.834 2.416-.834.93 0 1.74.278 2.428.834.697.555 1.154 1.289 1.37 2.2h-7.584c.233-.911.69-1.645 1.37-2.2Zm21.469-3.166h2.863V23.25h-2.864v-1.99c-1.145 1.45-2.664 2.175-4.557 2.175-1.204 0-2.296-.294-3.275-.882a6.299 6.299 0 0 1-2.304-2.45c-.548-1.036-.822-2.193-.822-3.47 0-1.276.274-2.433.822-3.469a6.209 6.209 0 0 1 2.304-2.437c.98-.597 2.071-.896 3.275-.896 1.901 0 3.42.722 4.558 2.164v-1.922Zm-3.998 10.602c1.146 0 2.096-.386 2.852-1.157.764-.779 1.145-1.74 1.145-2.885 0-1.152-.381-2.114-1.145-2.884-.756-.78-1.706-1.17-2.852-1.17-1.154 0-2.117.39-2.889 1.17-.763.77-1.145 1.732-1.145 2.884 0 1.145.382 2.106 1.145 2.885.772.771 1.735 1.157 2.89 1.157Zm35.186-10.602h3.188L102.941 25.6l-1.027 2.356h-3.319c.461-.939 1.313-2.68 1.718-3.537l.697-1.443-5.51-12.902h3.275l3.842 9.532 3.867-9.532Zm8.586 13.363c-2.092 0-3.894-.664-5.405-1.99l1.482-2.015c1.071 1.02 2.362 1.53 3.873 1.53.755 0 1.374-.124 1.855-.373.482-.257.722-.61.722-1.057a.93.93 0 0 0-.161-.535c-.1-.157-.225-.29-.374-.398-.141-.108-.353-.211-.635-.31-.282-.1-.531-.179-.747-.237a25.515 25.515 0 0 0-.884-.224l-.934-.224a16.527 16.527 0 0 1-.909-.286 6.683 6.683 0 0 1-.909-.373 5.835 5.835 0 0 1-.76-.497 2.716 2.716 0 0 1-.647-.672 3.57 3.57 0 0 1-.561-1.952c0-.862.241-1.6.722-2.213a4.153 4.153 0 0 1 1.868-1.343c.764-.29 1.627-.436 2.59-.436 1.901 0 3.549.577 4.944 1.729l-1.507 2.014c-.996-.845-2.125-1.268-3.387-1.268-.672 0-1.228.112-1.668.336-.432.224-.648.551-.648.982 0 .166.029.315.087.448a.93.93 0 0 0 .299.36c.15.108.291.2.423.274.133.075.328.153.586.236l.66.187.772.174c.373.083.68.157.921.224.249.058.552.149.909.273a4.943 4.943 0 0 1 1.669.87c.266.2.477.42.635.66.166.232.299.514.398.845.108.332.162.693.162 1.082 0 .87-.245 1.63-.734 2.276-.49.638-1.142 1.115-1.955 1.43-.806.315-1.723.473-2.752.473Zm20.26-6.778c0 .382-.025.755-.075 1.12h-10.534c.232.936.722 1.687 1.469 2.25.755.556 1.66.833 2.715.833.813 0 1.569-.17 2.266-.51.706-.34 1.254-.766 1.644-1.28l1.93 1.654c-.673.845-1.528 1.508-2.565 1.99-1.03.48-2.146.72-3.35.72-2.059 0-3.773-.654-5.143-1.964-1.361-1.31-2.042-2.94-2.042-4.887 0-1.26.299-2.404.897-3.432a6.436 6.436 0 0 1 2.465-2.438c1.054-.588 2.221-.883 3.499-.883 1.951 0 3.574.651 4.869 1.953 1.303 1.293 1.955 2.918 1.955 4.874Zm-6.849-4.253c-.921 0-1.726.278-2.415.834-.681.555-1.138 1.289-1.37 2.2h7.583c-.216-.911-.672-1.645-1.37-2.2-.689-.556-1.498-.834-2.428-.834Zm15.99.255h-3.674v6.212c0 .597.162 1.049.486 1.355.332.299.801.448 1.407.448.573 0 1.166-.15 1.781-.448v2.674c-.706.356-1.499.535-2.379.535-1.37 0-2.403-.361-3.1-1.082-.698-.722-1.046-1.729-1.046-3.022V12.66h-2.205v-2.587h2.205V5.367h2.851v4.706h3.674v2.587ZM48.869 9.873h-.49c-1.673 0-3.033.712-4.08 2.135v-1.935h-2.887V23.25h2.887v-6.434c0-1.222.36-2.189 1.08-2.9.728-.72 1.729-1.08 3-1.08h.49V9.873Z" clip-rule="evenodd"></path></svg></a><nav class="hidden laptop:flex items-center"><button type="button" class="flex items-center gap-1 text-label-medium transition-colors px-3 py-2 cursor-pointer text-content-layout-2 hover:text-content-layout-1" aria-haspopup="true" aria-expanded="false">Products<svg class="inline justify-self-center stroke-current h-4 w-4 min-w-4 stroke-2 transition-transform" aria-hidden="true" focusable="false"><use href="/icons/sprite-stroke.svg#chevron-down"></use></svg><span style="position:absolute;border:0;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;word-wrap:normal">dropdown</span></button><button type="button" class="flex items-center gap-1 text-label-medium transition-colors px-3 py-2 cursor-pointer text-content-layout-2 hover:text-content-layout-1" aria-haspopup="true" aria-expanded="false">Solutions<svg class="inline justify-self-center stroke-current h-4 w-4 min-w-4 stroke-2 transition-transform" aria-hidden="true" focusable="false"><use href="/icons/sprite-stroke.svg#chevron-down"></use></svg><span style="position:absolute;border:0;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;word-wrap:normal">dropdown</span></button><button type="button" class="flex items-center gap-1 text-label-medium transition-colors px-3 py-2 cursor-pointer text-content-layout-2 hover:text-content-layout-1" aria-haspopup="true" aria-expanded="false">Resources<svg class="inline justify-self-center stroke-current h-4 w-4 min-w-4 stroke-2 transition-transform" aria-hidden="true" focusable="false"><use href="/icons/sprite-stroke.svg#chevron-down"></use></svg><span style="position:absolute;border:0;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;word-wrap:normal">dropdown</span></button></nav></div><div class="hidden laptop:flex items-center"><button type="button" class="relative flex items-center justify-center min-w-max cursor-pointer select-none transition duration-fast ease-base transform focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-border-primary-soft focus-visible:ring-offset-2 focus-visible:ring-offset-surface-layout-1 active:scale-[0.98] active:origin-center text-button-medium py-3 px-4 rounded-2xl h-10 text-content-primary-solid bg-surface-primary-solid hover:bg-surface-primary-solid-hover active:bg-surface-primary-solid-active [&amp;_#loader]:border-t-content-primary-solid"><div class="flex items-center">Try locally<div class="h-1 w-1"></div><svg aria-hidden="true" class="inline justify-self-center stroke-current h-5 w-5 min-w-5 stroke-[1.5px]" focusable="false"><use href="/icons/sprite-stroke.svg#arrow-right"></use></svg><span style="position:absolute;border:0;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;word-wrap:normal"></span></div></button></div><button type="button" class="laptop:hidden flex items-center justify-center w-10 h-10 rounded-2xl text-content-layout-1 hover:bg-surface-layout-2 transition-colors cursor-pointer" aria-label="Open menu" aria-expanded="false"><svg class="inline justify-self-center stroke-current h-5 w-5 min-w-5 stroke-[1.5px]" aria-hidden="true" focusable="false"><use href="/icons/sprite-stroke.svg#menu"></use></svg><span style="position:absolute;border:0;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;word-wrap:normal">menu</span></button></div></header><main id="main-content"><!--$--><!--$--><div class="relative mx-auto container px-4 tablet:px-6 laptop:px-8 max-w-[1456px] flex flex-col gap-2 mt-12" id="blog-post-content"><div class="w-full flex flex-col gap-8"><div class="max-w-3xl mx-auto w-full flex flex-col gap-8"><div class="flex flex-col gap-6"><a class="text-body-small text-content-layout-3 hover:text-content-layout-1 transition-colors w-fit active" href="/blog" data-status="active" aria-current="page">← Back to blog</a><span class="inline-block w-fit px-3 py-1 rounded-full bg-surface-layout-1 text-body-small text-content-layout-3">PostgreSQL</span><h1 class="text-display-small">How to Solve for the N+1 Query Problem: Investigating and Optimizing Over-Querying</h1><p class="text-body-large text-content-layout-3">Imagine you&#x27;re running a popular e-commerce online bookstore that offers a vast collection of titles and authors to a growing user base. However, you&#x27;ve noticed a troubling trend over the past few months: the website is gradually slowing down, especially during peak hours when users browse various book categories. After an initial investigation, you find that the cause of the slowdown isn&#x27;t an increase in user traffic or a lack of server resources. Instead, it&#x27;s rooted in the very foundation of </p><div class="flex items-center gap-4"><a href="/blog/authors/neda" class="shrink-0"><img src="https://cms.readyset.io/files/authors/657385d2b4c219000125f4cd/c0376b666550a188861d4a2d.bin" alt="Readyset" loading="lazy" decoding="async" class="w-10 h-10 rounded-full object-cover"/></a><div class="flex flex-col"><a href="/blog/authors/neda" class="hover:underline"><p class="text-body-medium">Readyset</p></a><p class="text-body-small text-content-layout-3">2024-02-14<!-- --> · <!-- -->14 min read</p></div></div><div class="flex flex-wrap gap-2"><a href="/blog/tags/postgres" class="px-3 py-1 rounded-full bg-surface-layout-1 text-body-small text-content-layout-3 hover:bg-surface-layout-2 transition-colors">PostgreSQL</a></div></div><div role="separator" class="bg-border-layout-soft h-(--divider-thickness) w-full" style="--divider-thickness:1.5px"></div><div class="prose prose-neutral max-w-none"><!--$--><p>Imagine you&#x27;re running a popular e-commerce online bookstore that offers a vast collection of titles and authors to a growing user base. However, you&#x27;ve noticed a troubling trend over the past few months: the website is gradually slowing down, especially during peak hours when users browse various book categories. After an initial investigation, you find that the cause of the slowdown isn&#x27;t an increase in user traffic or a lack of server resources. Instead, it&#x27;s rooted in the very foundation of how your application interacts with your Postgres database.</p>
<p>The culprit? N+1 query problems. As more users navigate your site, more requests are made to the database to fetch information. Instead of being efficiently retrieved in grouped queries, each request individually pulls associated data like author details, reviews, and related books. What should have been a streamlined operation has turned into a burdensome load on your database, leading to longer load times and a compromised user experience.</p>
<p>This scenario is not unique to your online bookstore. Regardless of size or domain, many applications encounter similar performance bottlenecks due to N+1 queries. Understanding the nature of these queries, their impact on database performance, and how to optimize them is crucial for developers and database administrators. Here, we’re going into N+1 queries in a Postgres environment, providing insights and strategies to turn a potential database nightmare into a well-optimized, efficient system.</p>
<h2 id="what-are-n1-queries"><a href="#what-are-n1-queries">What Are N+1 Queries?</a></h2>
<p>N+1 queries are a common performance bottleneck in databases. This issue occurs when an application performs an initial query to retrieve a set of records, followed by additional queries for each individual record. The name &#x27;N+1&#x27; stems from making one (1) initial query and then N additional queries, resulting in N+1 total queries for N records.</p>
<p>Let’s say you have an application that displays user profiles and their respective posts. The application first executes a query to fetch all users. This is the &quot;1&quot; in N+1. The application performs another query for each user retrieved to fetch their posts. If there are ten users, this results in 10 additional queries (the &quot;N&quot; in N+1), totaling 11 queries. While this approach may seem straightforward, it&#x27;s highly inefficient, especially as the number of users grows.</p>
<p>Let’s look at what this looks like. Assume you have two tables: <code>users</code> and <code>posts</code>. Each user has multiple posts. The <code>posts</code> table has a foreign key that references the <code>users</code> table. You want to display each user along with their posts.</p>
<ul>
<li><strong>Users Table:</strong>
<ul>
<li><code>id</code> (Primary Key)</li>
<li><code>name</code></li>
</ul>
</li>
<li><strong>Posts Table:</strong>
<ul>
<li><code>id</code> (Primary Key)</li>
<li><code>content</code></li>
<li><code>user_id</code> (Foreign Key to Users)</li>
</ul>
</li>
</ul>
<p>A naive N+1 query to get these posts might look like this:</p>
<pre><code class="hljs language-unset">-- Query 1: Fetch all users
SELECT id, name FROM users;
-- For each user obtained from the above query, execute the following query:
SELECT content FROM posts WHERE user_id = [user_id];
</code></pre>
<p>Here, Query 1 is the &quot;1&quot; query. Query 2 will be the “N” queries, where you iterate through the <code>user_id</code>s.</p>
<p>As a result, if there are 100 users, the total number of queries executed will be 101: 1 for fetching all users and 100 for fetching the posts for each user. This is a classic example of an N+1 query problem and can lead to performance issues, especially with a large number of users and posts.</p>
<h3 id="n1-queries-in-orms"><a href="#n1-queries-in-orms">N+1 Queries in ORMs</a></h3>
<p>It’s common to find N+1 queries in frameworks using Object-Relational Mappings (ORMs). These are designed to convert models in an application into SQL statements to query data in relational databases. However, they often exacerbate the N+1 query issue due to their internal mechanisms.</p>
<p>Let’s use the Python web framework Django as our example. In Django, an N+1 query problem can quickly occur when you have related models and access related data without properly optimizing your queries. Let&#x27;s consider a scenario where each <code>User</code> has a foreign key to a <code>Profile</code> model:</p>
<pre><code class="hljs language-python"><span class="hljs-comment"># models.py</span>
<span class="hljs-keyword">from</span> django.db <span class="hljs-keyword">import</span> models
<span class="hljs-keyword">class</span> <span class="hljs-title class_">Profile</span>(models.Model):
bio = models.TextField()
<span class="hljs-comment"># other fields like date_of_birth, location, etc.</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">User</span>(models.Model):
name = models.CharField(max_length=<span class="hljs-number">100</span>)
profile = models.OneToOneField(Profile, on_delete=models.CASCADE)
<span class="hljs-comment"># other fields like email, etc.</span>
<span class="hljs-comment"># views.py</span>
<span class="hljs-keyword">from</span> django.shortcuts <span class="hljs-keyword">import</span> render
<span class="hljs-keyword">from</span> .models <span class="hljs-keyword">import</span> User
<span class="hljs-keyword">def</span> <span class="hljs-title function_">user_list</span>(<span class="hljs-params">request</span>):
users = User.objects.<span class="hljs-built_in">all</span>()
user_profiles = []
<span class="hljs-keyword">for</span> user <span class="hljs-keyword">in</span> users:
profile = user.profile <span class="hljs-comment"># This creates the N+1 problem</span>
user_profiles.append((user, profile))
context = {<span class="hljs-string">&#x27;user_profiles&#x27;</span>: user_profiles}
<span class="hljs-keyword">return</span> render(request, <span class="hljs-string">&#x27;user_list.html&#x27;</span>, context)
</code></pre>
<p>Here, the <code>User.objects.all()</code> query retrieves all users. This is the &quot;1&quot; in N+1. Inside the loop, accessing each user&#x27;s profile may result in a separate database query to fetch the corresponding <code>Profile</code> instance. This is the &quot;N&quot; part of N+1, where N is the number of users.</p>
<p>Because ORMs abstract away the underlying SQL queries, developers might not immediately realize that their code generates these inefficient N+1 queries.</p>
<h2 id="the-impact-of-n1-queries-on-performance"><a href="#the-impact-of-n1-queries-on-performance">The Impact of N+1 Queries on Performance</a></h2>
<p>N+1 queries impact performance, especially in large-scale systems. These queries lead to inefficient data retrieval, increased load on the database, and, ultimately, a poor user experience. Understanding the consequences of N+1 queries is crucial for database optimization.</p>
<h3 id="increased-database-load"><a href="#increased-database-load">Increased Database Load</a></h3>
<p>Each additional query in an N+1 problem adds to the load on the database. This is particularly problematic with large datasets. For instance, if an application retrieves 1000 users and makes a separate query for each user&#x27;s posts, it results in 1001 queries hitting the database instead of potentially just one. This extra load can slow down the database response times, affecting all application users.</p>
<p>Imagine a web application displaying a list of users and their recent activities. Without optimization, fetching 1,000 users might result in 1,001 queries (one to fetch all users and 1,000 to fetch activities for each). This heavy load can lead to longer wait times for the data to load, affecting user experience.</p>
<p>A high volume of queries also consumes more CPU and memory resources on the database server. In our previous example, instead of a single, efficient query, the server must process 1,001 queries, consuming more resources. This impacts the query in question and affects the overall efficiency of the database server, hindering its ability to handle other requests.</p>
<h3 id="scaling-challenges"><a href="#scaling-challenges">Scaling Challenges</a></h3>
<p>Applications with N+1 query problems often struggle to scale. As the data grows, so does the number of queries. In a social media app, for example, as more users join and create posts, the N+1 issue exacerbates, leading to an exponential increase in queries. This scaling challenge can cause significant performance degradation over time, requiring more hardware resources to maintain the same level of performance.</p>
<p>The total time complexity for the N+1 queries pattern is O(N). As the number of records (N) increases, the total number of queries increases linearly. This linear growth can lead to significant performance degradation, especially with large datasets. Each query incurs a certain amount of overhead due to network latency, query parsing, execution planning, and data retrieval. This overhead, multiplied by the number of queries, can substantially affect performance.</p>
<h3 id="poor-user-experience"><a href="#poor-user-experience">Poor User Experience</a></h3>
<p>The cumulative effect of these performance issues can result in longer loading times for users, negatively impacting the user experience. In an e-commerce site, if product details are fetched with N+1 queries, each additional millisecond in load time can potentially lead to lost sales as customers grow impatient and leave the site.</p>
<p>Consider a real-time data dashboard that monitors and displays various metrics. If each metric&#x27;s data is fetched using separate queries for each element, the dashboard will experience noticeable delays, failing to deliver the real-time experience expected by users.</p>
<h2 id="detecting-and-investigating-n1-queries"><a href="#detecting-and-investigating-n1-queries">Detecting and Investigating N+1 Queries</a></h2>
<p>Sometimes, the simplest way to detect N+1 queries is through careful code review. Reviewers can look for loops or iterative processes that make database calls, particularly in the context of ORMs or when accessing related data.</p>
<p>But N+1 queries can often be subtle and not immediately evident, especially in complex applications. Here are some effective strategies and tools that can be employed to detect and investigate N+1 queries:</p>
<h3 id="query-logging"><a href="#query-logging">Query Logging</a></h3>
<p>Enabling query logging in Postgres is one of the first steps in detecting N+1 query problems. Logging all executed queries allows you to analyze the logs for patterns that indicate N+1 issues.</p>
<pre><code>-- Set the logging level to log all statements
ALTER DATABASE your_database_name SET log_statement = &#x27;all&#x27;;
</code></pre>
<p>After enabling logging, look for sequences of similar queries that differ only in a parameter, such as multiple queries fetching details for different user IDs.</p>
<pre><code>2024-01-23 10:00:01 UTC LOG: statement: SELECT * FROM users
2024-01-23 10:00:02 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 1
2024-01-23 10:00:02 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 2
2024-01-23 10:00:02 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 3
2024-01-23 10:00:02 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 4
2024-01-23 10:00:02 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 5
2024-01-23 10:00:03 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 6
2024-01-23 10:00:03 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 7
2024-01-23 10:00:03 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 8
2024-01-23 10:00:03 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 9
2024-01-23 10:00:03 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 10
...
</code></pre>
<p>This is a strong indicator of N+1 queries.</p>
<h3 id="performance-monitoring-tools"><a href="#performance-monitoring-tools">Performance Monitoring Tools</a></h3>
<p>Tools like<a href="https://github.com/darold/pgbadger?ref=blog.readyset.io" rel="noopener noreferrer" target="_blank"> pgBadger</a>,<a href="https://www.datadoghq.com/blog/database-performance-monitoring-datadog/?ref=blog.readyset.io" rel="noopener noreferrer" target="_blank"> DataDog</a>, or<a href="https://scoutapm.com/blog/understanding-n1-database-queries?ref=blog.readyset.io" rel="noopener noreferrer" target="_blank"> ScoutAPM</a> offer detailed insights into database performance. They can help identify inefficient query patterns that may suggest N+1 issues.</p>
<p>With pgBadger, you can analyze your Postgres logs to get a report highlighting frequently executed queries. A high frequency of similar queries can be a sign of N+1 problems.</p>
<p><img src="https://cms.readyset.io/files/articles/65cb6bb6fbd2870001265837/c710994e211d520d8fe0dd35.png" alt=""/></p>
<p>The <code>SELECT * FROM posts WHERE user_id = ?</code> query stands out due to its high frequency of execution (850 times). This pattern is indicative of an N+1 query problem. It suggests that for each user fetched by the <code>SELECT * FROM users</code> query (executed ten times), there are multiple subsequent queries to fetch posts. The average duration of each <code>posts</code> query is low (5ms), but the cumulative impact (total duration of 4250ms) is significant, pointing to a potential performance issue.</p>
<p>The disparity between the number of executions of the <code>users</code> query and the <code>posts</code> query is a classic sign of N+1 queries. From this, the recommendations might be to investigate the application code following the execution of <code>SELECT * FROM users</code>, especially the parts where posts for each user are accessed or displayed. Then, developers can optimize the query pattern to use JOIN operations or batch processing to reduce the total number of queries.</p>
<p>An application performance monitoring tool like DataDog or Scout APM will often automatically highlight inefficiencies in an application, such as N+1 queries.</p>
<p><img src="https://cms.readyset.io/files/articles/65cb6bb6fbd2870001265837/1fe58061ad314ed7a931938d.bin" alt=""/></p>
<p>They can show the response times for these queries, the number of calls, and the query itself. This kind of report helps pinpoint where the application might be inefficiently using the database, guiding developers toward specific areas of the code that may need optimization to address the N+1 query problem.</p>
<p>Languages also have profiling tools built in, such as Python&#x27;s <code>cProfile</code>. These can help detect N+1 queries by showing where your application spends most of its time.</p>
<pre><code class="hljs language-python"><span class="hljs-keyword">import</span> cProfile
cProfile.run(<span class="hljs-string">&#x27;function_that_loads_data()&#x27;</span>)
</code></pre>
<p>This profiling can help identify functions that are making excessive database calls. A large amount of time spent in database-related functions could indicate N+1 queries.</p>
<p>Beyond language tools, most ORMs allow enabling SQL debug logging. This feature logs every SQL query the ORM executes, making spotting repetitive query patterns indicative of N+1 problems easier. Keeping with the Python theme, in Django, SQL debug logging can be activated by setting <code>DEBUG = True</code> in your <code>settings.py</code>, which causes all SQL queries to be printed to the console during development. You can also use Django&#x27;s <code>django.db.connection.queries</code> for query insights.</p>
<h3 id="explain-command"><a href="#explain-command">EXPLAIN Command</a></h3>
<p>The <code>EXPLAIN</code> command in Postgres is an invaluable tool for understanding how your queries are being executed. It can help you identify queries that do not efficiently use indexes or perform full table scans, which could be part of an N+1 problem.</p>
<pre><code>EXPLAIN SELECT * FROM posts WHERE user_id = 1;
</code></pre>
<p>The output provides insights into the query execution plan, which can help identify inefficiencies.</p>
<pre><code>QUERY PLAN
-----------------------------------------------------------
Seq Scan on posts (cost=0.00..35.50 rows=1560 width=2048)
Filter: (user_id = 1)
</code></pre>
<p>Here, <code>EXPLAIN</code> tells us we perform a “Seq Scan on posts.” This indicates that a sequential scan is being performed on the <code>posts</code> table. Sequential scans are generally less efficient than index scans, especially for larger tables, as they involve scanning each row.</p>
<p>The sequential scan (Seq Scan) on the <code>posts</code> table might not be a problem for a single query. However, suppose similar queries are being executed repeatedly for different <code>user_id</code> values (as in an N+1 scenario). In that case, it indicates that the database is performing numerous full table scans, which can be highly inefficient.</p>
<p>The absence of an index scan suggests that there might not be an index on the <code>user_id</code> column or the query planner did not find it efficient to use the index. For an N+1 query pattern, the repeated execution of such full table scans can significantly degrade performance.</p>
<h2 id="optimization-and-query-design"><a href="#optimization-and-query-design">Optimization and Query Design</a></h2>
<p>Optimization and better query design are the only strategies to significantly reduce the impact of N+1 query problems in a Postgres environment. Doing so naturally leads to better application performance and scalability.</p>
<h3 id="eager-loading"><a href="#eager-loading">Eager loading</a></h3>
<p>When using ORMs, the best solution (given you can’t optimize the query yourself) is eager loading. Eager loading involves modifying the query to fetch all related data in a single query instead of separate queries for each record. This can be achieved in ORMs with specific methods or query options (like <code>.include</code> in Rails or <code>select_related</code> in Django). These reduce the number of queries to the database, improving performance.</p>
<p>If we want to optimize our Django example above, this can be achieved through techniques like <code>select_related</code> or <code>prefetch_related</code>, designed to handle database queries more efficiently for related objects.</p>
<p>Assuming the <code>User</code> model has a ForeignKey to another model, let&#x27;s say a <code>Profile</code> model, here&#x27;s an optimized version of the Django code that avoids the N+1 query problem:</p>
<pre><code class="hljs language-python"><span class="hljs-comment"># models.py</span>
<span class="hljs-keyword">from</span> django.db <span class="hljs-keyword">import</span> models
<span class="hljs-keyword">class</span> <span class="hljs-title class_">Profile</span>(models.Model):
bio = models.TextField()
<span class="hljs-comment"># other fields</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">User</span>(models.Model):
name = models.CharField(max_length=<span class="hljs-number">100</span>)
profile = models.OneToOneField(Profile, on_delete=models.CASCADE)
<span class="hljs-comment"># other fields</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">Post</span>(models.Model):
user = models.ForeignKey(User, related_name=<span class="hljs-string">&#x27;posts&#x27;</span>, on_delete=models.CASCADE)
content = models.TextField()
<span class="hljs-comment"># other fields</span>
<span class="hljs-comment"># views.py</span>
<span class="hljs-keyword">from</span> django.shortcuts <span class="hljs-keyword">import</span> render
<span class="hljs-keyword">from</span> .models <span class="hljs-keyword">import</span> User
<span class="hljs-keyword">def</span> <span class="hljs-title function_">user_profiles</span>(<span class="hljs-params">request</span>):
<span class="hljs-comment"># Use &#x27;select_related&#x27; to fetch the related Profile in the same query</span>
users = User.objects.select_related(<span class="hljs-string">&#x27;profile&#x27;</span>).<span class="hljs-built_in">all</span>()
<span class="hljs-comment"># &#x27;prefetch_related&#x27; is used for reverse ForeignKey relationships</span>
<span class="hljs-comment"># This fetches all related posts in a separate query, reducing the overall number of queries</span>
users = users.prefetch_related(<span class="hljs-string">&#x27;posts&#x27;</span>)
<span class="hljs-keyword">return</span> render(request, <span class="hljs-string">&#x27;user_profiles.html&#x27;</span>, {<span class="hljs-string">&#x27;users&#x27;</span>: users})
</code></pre>
<p>The <code>select_related(&#x27;profile&#x27;)</code> method is used with the <code>User.objects.all()</code> query. This fetches the associated <code>Profile</code> for each <code>User</code> in the same database query, thus avoiding separate queries for each user&#x27;s profile.</p>
<p>The <code>prefetch_related(&#x27;posts&#x27;)</code> method handles the reverse ForeignKey relationship from <code>User</code> to <code>Post</code>. It performs a separate query to fetch all related posts and then efficiently pairs them with the corresponding users, which is more efficient than doing individual queries for each user&#x27;s posts.</p>
<p>This approach significantly reduces the number of queries, particularly when you have a large number of users and posts, thus improving the performance of your Django application.</p>
<h3 id="caching"><a href="#caching">Caching</a></h3>
<p>If you have particular queries you want to return fast, you can implement caching to reduce the need to query the database each time.</p>
<p>Readyset allows you to cache SQL queries without changes to your application code. The only change needed is to swap out your primary Postgres database connection string with a Readyset connection string. Readset will connect to your primary database and register with the replication stream. After snapshotting your database, every query will be proxied through Readyset. Along with your monitoring tools from above, you can then also use Readyset to understand your query performance:</p>
<p><img src="https://cms.readyset.io/files/articles/65cb6bb6fbd2870001265837/2734aa9275276207973d0dba.bin" alt=""/></p>
<p>When you have detected N+1 queries that can be cached (usually, ready-heavy queries are optimal candidates), you can start caching. In this case, we want to cache our posts queries. To do so, we just prepend <code>CREATE CACHE FROM</code> to our query:</p>
<pre><code class="hljs language-python">CREATE CACHE FROM SELECT * FROM posts WHERE user_id = ?;
</code></pre>
<p>The results from this query will now be served from Readyset with sub-millisecond latencies. Readyset monitors the replication stream from the primary database, looking for changes to the data, so the cache will be automatically updated whenever the underlying data is updated in the primary database.</p>
<p>Readyset also works with ORMs to help increase performance and optimize those queries under the hood.</p>
<h3 id="query-design"><a href="#query-design">Query design</a></h3>
<p>Just as you can optimize your system around your queries, if you aren’t using an ORM, you can also optimize your queries for better performance. Here are a few options:</p>
<ul>
<li><strong>Use Joins Appropriately</strong>: Utilize SQL joins to combine data from multiple tables in a single query. Understand the differences between <code>INNER JOIN</code>, <code>LEFT JOIN</code>, etc., and choose the most appropriate use case to avoid unnecessary data retrieval.</li>
<li><strong>Select Only Required Columns</strong>: Specify the columns you need in your <code>SELECT</code> statements rather than using <code>SELECT *</code>. This reduces the amount of data transferred and processed, making the query more efficient.</li>
<li><strong>Understand and Use Indices</strong>: Ensure your queries effectively leverage indices, especially for columns in <code>JOIN</code>, <code>WHERE</code>, or <code>ORDER BY</code> clauses. Regularly review and optimize indices based on query patterns and data changes.</li>
<li><strong>Avoid Looping Queries</strong>: Identify scenarios where your code iteratively executes queries (like in a loop) and refactor them to use bulk data retrieval techniques. Replace multiple small queries with fewer, more comprehensive queries.</li>
<li><strong>Limit Data with Pagination</strong>: When dealing with large datasets, use pagination to limit the data retrieved and processed in a single query. This improves both database and application performance.</li>
</ul>
<p>Thus, to optimize the basic queries from above, you could add a join into your query to fetch all the data at once:</p>
<pre><code>SELECT u.id, u.name, p.content
FROM users u
LEFT JOIN posts p ON u.id = p.user_id
</code></pre>
<h2 id="solving-the-n1-puzzle"><a href="#solving-the-n1-puzzle">Solving the N+1 Puzzle</a></h2>
<p>N+1 queries can destroy your application’s performance as you scale. What’s worse is that this issue can be abstracted from you by ORMs and other high-level frameworks, making them harder to detect and resolve. But with monitoring tools and a better understanding of the problem, you can detect these troublesome queries and work to improve your query design and optimize your performance.</p>
<p>At Readyset, we are building ways to help developers and platform engineers do just that. By caching queries, you can significantly improve your response latencies while still serving your users&#x27; fresh data. If this sounds like something that will help you, sign up for <a href="https://readyset.io/?utm%5Fcampaign=eg&amp;utm%5Fmedium=internal&amp;utm%5Fsource=blog" rel="noopener noreferrer" target="_blank">Readyset Cloud</a> or<a href="https://go.readyset.io/community?ref=blog.readyset.io" rel="noopener noreferrer" target="_blank"> reach out to us</a> if you have any questions.</p><!--/$--></div><div role="separator" class="bg-border-layout-soft h-(--divider-thickness) w-full" style="--divider-thickness:1.5px"></div><div class="flex flex-col laptop:flex-row items-start laptop:items-center justify-between gap-4 rounded-4xl bg-surface-layout-1 p-8"><div class="flex flex-col gap-1"><p class="text-headline-5">Want to see Readyset in action?</p><p class="text-body-small text-content-layout-3">Book a demo and see how Readyset can accelerate your database.</p></div><a href="/book-a-demo"><button type="button" class="relative flex items-center justify-center min-w-max cursor-pointer select-none transition duration-fast ease-base transform focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-border-primary-soft focus-visible:ring-offset-2 focus-visible:ring-offset-surface-layout-1 active:scale-[0.98] active:origin-center text-button-medium py-3 px-4 rounded-2xl h-10 text-content-primary-solid bg-surface-primary-solid hover:bg-surface-primary-solid-hover active:bg-surface-primary-solid-active [&amp;_#loader]:border-t-content-primary-solid"><div class="flex items-center">Book a demo<div class="h-1 w-1"></div><svg aria-hidden="true" class="inline justify-self-center stroke-current h-5 w-5 min-w-5 stroke-[1.5px]" focusable="false"><use href="/icons/sprite-stroke.svg#arrow-right"></use></svg><span style="position:absolute;border:0;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;word-wrap:normal"></span></div></button></a></div></div></div></div><div class="relative mx-auto container px-4 tablet:px-6 laptop:px-8 max-w-[1456px] flex flex-col gap-2 mt-40" id="related-articles"><div class="w-full flex flex-col gap-8"><div class="flex flex-col items-center gap-2 max-w-2xl mx-auto"><p class="text-headline-2 text-center">Related articles</p><p class="text-body-large text-content-layout-3 text-center">Continue reading more about <!-- -->postgresql<!-- -->.</p></div><div class="grid grid-cols-1 laptop:grid-cols-3 gap-6"><a href="/blog/getting-started-with-django-and-readyset" class="group flex flex-col rounded-4xl bg-surface-layout-1 overflow-hidden transition-shadow hover:shadow-lg"><div class="aspect-[1200/630] w-full overflow-hidden"><img src="/images/blog-placeholder.svg" alt="Getting started with Django, PostgreSQL, and Readyset" loading="lazy" decoding="async" class="w-full h-full object-cover transition-transform duration-300 group-hover:scale-105"/></div><div class="flex flex-col gap-3 p-6 flex-1"><span class="inline-block w-fit px-3 py-1 rounded-full bg-surface-layout-2 text-body-small text-content-layout-3">PostgreSQL</span><h3 class="text-headline-5 line-clamp-2">Getting started with Django, PostgreSQL, and Readyset</h3><p class="text-body-small text-content-layout-3 line-clamp-3 flex-1">Real-world applications face performance challenges with growing data and user traffic. Caching mitigates these challenges by storing frequently accessed data, reducing database fetches, improving response times, and enhancing application scalability, reliability, and user experience.
Readyset is a caching engine for Postgres and MySQL databases, requiring minimal adjustments to existing setups. Its wire-compatible design means that the only modification necessary is to adjust the connection st</p><div class="flex items-center justify-between pt-2"><div class="flex items-center gap-2"><img src="https://cms.readyset.io/files/authors/6644e32196ffac000112bdee/d3c2963169bf4bcdcbc43d2c.jpeg" alt="Rishi Raj Jain" loading="lazy" decoding="async" class="w-5 h-5 rounded-full object-cover"/><span class="text-body-small text-content-layout-2">Rishi Raj Jain</span></div><div class="flex items-center gap-2 text-body-small text-content-layout-3"><span>2024-05-16</span><span>·</span><span>11 min read</span></div></div></div></a><a href="/blog/optimizing-sql-pagination-in-postgres" class="group flex flex-col rounded-4xl bg-surface-layout-1 overflow-hidden transition-shadow hover:shadow-lg"><div class="aspect-[1200/630] w-full overflow-hidden"><img src="/images/blog-placeholder.svg" alt="Optimizing SQL Pagination in Postgres" loading="lazy" decoding="async" class="w-full h-full object-cover transition-transform duration-300 group-hover:scale-105"/></div><div class="flex flex-col gap-3 p-6 flex-1"><span class="inline-block w-fit px-3 py-1 rounded-full bg-surface-layout-2 text-body-small text-content-layout-3">PostgreSQL</span><h3 class="text-headline-5 line-clamp-2">Optimizing SQL Pagination in Postgres</h3><p class="text-body-small text-content-layout-3 line-clamp-3 flex-1">SQL pagination is a technique to retrieve a subset of records from a Postgres table or result set. It allows you to divide large datasets into smaller, manageable chunks called &quot;pages&quot; and retrieve them one page at a time.
Understanding Pagination in Postgres
Pagination is commonly used in applications that display data in a paginated format, such as search results, product listings, or user records. It helps improve performance and user experience by loading and displaying only a portion o</p><div class="flex items-center justify-between pt-2"><div class="flex items-center gap-2"><img src="https://cms.readyset.io/files/authors/657385d2b4c219000125f4cd/c0376b666550a188861d4a2d.bin" alt="Readyset" loading="lazy" decoding="async" class="w-5 h-5 rounded-full object-cover"/><span class="text-body-small text-content-layout-2">Readyset</span></div><div class="flex items-center gap-2 text-body-small text-content-layout-3"><span>2024-04-24</span><span>·</span><span>13 min read</span></div></div></div></a><a href="/blog/mastering-query-rewriting-for-faster-postgresql-performance" class="group flex flex-col rounded-4xl bg-surface-layout-1 overflow-hidden transition-shadow hover:shadow-lg"><div class="aspect-[1200/630] w-full overflow-hidden"><img src="/images/blog-placeholder.svg" alt="Mastering Query Rewriting for Faster PostgreSQL Performance" loading="lazy" decoding="async" class="w-full h-full object-cover transition-transform duration-300 group-hover:scale-105"/></div><div class="flex flex-col gap-3 p-6 flex-1"><span class="inline-block w-fit px-3 py-1 rounded-full bg-surface-layout-2 text-body-small text-content-layout-3">PostgreSQL</span><h3 class="text-headline-5 line-clamp-2">Mastering Query Rewriting for Faster PostgreSQL Performance</h3><p class="text-body-small text-content-layout-3 line-clamp-3 flex-1">When you first spin up your app, the emphasis is on getting started and getting data to your clients. But when you don’t have throughput, you are also not going to have enough concurrency to unveil bad queries.
But then you have success. And success means data. More users, more interactions, more everything. Suddenly, queries that performed fine are struggling under the load, hurting performance and scalability. This is all going to mean a far worse user experience when much higher costs becaus</p><div class="flex items-center justify-between pt-2"><div class="flex items-center gap-2"><img src="https://cms.readyset.io/files/authors/65b15d7c857d140001d2f6de/20ce2d4a31bccdca053a0d7a.jpg" alt="Marcelo Altmann" loading="lazy" decoding="async" class="w-5 h-5 rounded-full object-cover"/><span class="text-body-small text-content-layout-2">Marcelo Altmann</span></div><div class="flex items-center gap-2 text-body-small text-content-layout-3"><span>2024-04-11</span><span>·</span><span>9 min read</span></div></div></div></a></div></div></div><div class="relative mx-auto container px-4 tablet:px-6 laptop:px-8 max-w-[1456px] flex flex-col gap-2 mt-20 mb-12" id="hero"><div class="w-full gap-4 flex flex-col"><div class="rounded-[80px] bg-surface-layout-1 w-full"><div class="flex flex-col justify-center items-center gap-8 w-full max-w-3xl mx-auto py-44"><h2 class="text-display-medium text-center">Still scaling the hard way?</h2><p class="text-body-large text-content-layout-3 text-center">Modern applications demand instant performance, even under unpredictable load. Readyset helps you eliminate slow queries, stabilize latency, and scale confidently.</p><div class="flex justify-center items-center gap-4"><a href="/book-a-demo"><button type="button" class="relative flex items-center justify-center min-w-max cursor-pointer select-none transition duration-fast ease-base transform focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-border-primary-soft focus-visible:ring-offset-2 focus-visible:ring-offset-surface-layout-1 active:scale-[0.98] active:origin-center text-button-medium py-3 px-4 rounded-3xl h-12 text-content-primary-soft border border-border-primary-soft bg-transparent hover:bg-surface-primary-soft-hover active:bg-surface-primary-soft-active [&amp;_#loader]:border-t-content-primary-soft"><div class="flex items-center">Contact Sales<div class="h-1 w-1"></div><svg aria-hidden="true" class="inline justify-self-center stroke-current h-5 w-5 min-w-5 stroke-[1.5px]" focusable="false"><use href="/icons/sprite-stroke.svg#arrow-right"></use></svg><span style="position:absolute;border:0;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;word-wrap:normal"></span></div></button></a><a href="/book-a-demo"><button type="button" class="relative flex items-center justify-center min-w-max cursor-pointer select-none transition duration-fast ease-base transform focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-border-primary-soft focus-visible:ring-offset-2 focus-visible:ring-offset-surface-layout-1 active:scale-[0.98] active:origin-center text-button-medium py-3 px-4 rounded-3xl h-12 text-content-primary-solid bg-surface-primary-solid hover:bg-surface-primary-solid-hover active:bg-surface-primary-solid-active [&amp;_#loader]:border-t-content-primary-solid"><div class="flex items-center">Start for free<div class="h-1 w-1"></div><svg aria-hidden="true" class="inline justify-self-center stroke-current h-5 w-5 min-w-5 stroke-[1.5px]" focusable="false"><use href="/icons/sprite-stroke.svg#arrow-right"></use></svg><span style="position:absolute;border:0;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;word-wrap:normal"></span></div></button></a></div></div><div role="separator" class="bg-border-layout-soft h-(--divider-thickness) w-full" style="--divider-thickness:1.5px"></div><div class="flex justify-between items-center w-full py-12 px-12"><div class="flex flex-col gap-2"><p class="text-headline-2 text-content-layout-1 text-left">Revolutionize your database performance with Readyset</p><p class="text-body-large text-content-layout-2">Serve requests at sub-millisecond latencies with the modern database scaling and query caching system for MySQL and PostgreSQL.</p></div><div class="hidden laptop:block"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 145 33" class="fill-current text-content-layout-1 h-6"><path d="M16 31.328c0 .736-.597 1.333-1.333 1.333H4a4 4 0 0 1-4-4V17.994c0-.736.597-1.333 1.333-1.333H16v14.667Z"></path><path d="M21.333 11.328v5.333h-4c-.736 0-1.333.597-1.333 1.333v4h-5.333v-5.333h4c.736 0 1.333-.597 1.333-1.333v-4h5.333Z"></path><path d="M32 15.328c0 .736-.597 1.333-1.333 1.333H16V1.994c0-.736.597-1.333 1.333-1.333H28a4 4 0 0 1 4 4v10.667Z"></path><path fill-rule="evenodd" d="M91.272 5.367h2.864V23.25h-2.864v-1.99c-1.145 1.45-2.665 2.175-4.557 2.175-1.204 0-2.296-.294-3.275-.882a6.299 6.299 0 0 1-2.304-2.45c-.548-1.036-.822-2.193-.822-3.47 0-1.276.274-2.433.822-3.469a6.209 6.209 0 0 1 2.304-2.437c.98-.597 2.071-.896 3.275-.896 1.9 0 3.42.722 4.557 2.164V5.367Zm-3.997 15.308c1.146 0 2.096-.386 2.852-1.157.763-.779 1.145-1.74 1.145-2.885 0-1.152-.382-2.114-1.145-2.884-.756-.78-1.706-1.17-2.852-1.17-1.154 0-2.117.39-2.889 1.17-.764.77-1.145 1.732-1.145 2.884 0 1.145.381 2.106 1.145 2.885.772.771 1.735 1.157 2.89 1.157Zm-24.258-2.898c.05-.364.075-.737.075-1.119 0-1.956-.652-3.581-1.955-4.874-1.295-1.302-2.918-1.953-4.87-1.953-1.278 0-2.444.295-3.498.883a6.442 6.442 0 0 0-2.466 2.438c-.597 1.028-.896 2.172-.896 3.432 0 1.948.68 3.577 2.042 4.887 1.37 1.31 3.084 1.965 5.143 1.965 1.203 0 2.32-.24 3.35-.722 1.037-.48 1.892-1.144 2.564-1.99l-1.93-1.653c-.39.514-.938.94-1.643 1.28-.698.34-1.453.51-2.267.51-1.054 0-1.959-.277-2.714-.833-.747-.563-1.237-1.314-1.47-2.25h10.535Zm-9.19-4.538c.69-.556 1.494-.834 2.416-.834.93 0 1.74.278 2.428.834.697.555 1.154 1.289 1.37 2.2h-7.584c.233-.911.69-1.645 1.37-2.2Zm21.469-3.166h2.863V23.25h-2.864v-1.99c-1.145 1.45-2.664 2.175-4.557 2.175-1.204 0-2.296-.294-3.275-.882a6.299 6.299 0 0 1-2.304-2.45c-.548-1.036-.822-2.193-.822-3.47 0-1.276.274-2.433.822-3.469a6.209 6.209 0 0 1 2.304-2.437c.98-.597 2.071-.896 3.275-.896 1.901 0 3.42.722 4.558 2.164v-1.922Zm-3.998 10.602c1.146 0 2.096-.386 2.852-1.157.764-.779 1.145-1.74 1.145-2.885 0-1.152-.381-2.114-1.145-2.884-.756-.78-1.706-1.17-2.852-1.17-1.154 0-2.117.39-2.889 1.17-.763.77-1.145 1.732-1.145 2.884 0 1.145.382 2.106 1.145 2.885.772.771 1.735 1.157 2.89 1.157Zm35.186-10.602h3.188L102.941 25.6l-1.027 2.356h-3.319c.461-.939 1.313-2.68 1.718-3.537l.697-1.443-5.51-12.902h3.275l3.842 9.532 3.867-9.532Zm8.586 13.363c-2.092 0-3.894-.664-5.405-1.99l1.482-2.015c1.071 1.02 2.362 1.53 3.873 1.53.755 0 1.374-.124 1.855-.373.482-.257.722-.61.722-1.057a.93.93 0 0 0-.161-.535c-.1-.157-.225-.29-.374-.398-.141-.108-.353-.211-.635-.31-.282-.1-.531-.179-.747-.237a25.515 25.515 0 0 0-.884-.224l-.934-.224a16.527 16.527 0 0 1-.909-.286 6.683 6.683 0 0 1-.909-.373 5.835 5.835 0 0 1-.76-.497 2.716 2.716 0 0 1-.647-.672 3.57 3.57 0 0 1-.561-1.952c0-.862.241-1.6.722-2.213a4.153 4.153 0 0 1 1.868-1.343c.764-.29 1.627-.436 2.59-.436 1.901 0 3.549.577 4.944 1.729l-1.507 2.014c-.996-.845-2.125-1.268-3.387-1.268-.672 0-1.228.112-1.668.336-.432.224-.648.551-.648.982 0 .166.029.315.087.448a.93.93 0 0 0 .299.36c.15.108.291.2.423.274.133.075.328.153.586.236l.66.187.772.174c.373.083.68.157.921.224.249.058.552.149.909.273a4.943 4.943 0 0 1 1.669.87c.266.2.477.42.635.66.166.232.299.514.398.845.108.332.162.693.162 1.082 0 .87-.245 1.63-.734 2.276-.49.638-1.142 1.115-1.955 1.43-.806.315-1.723.473-2.752.473Zm20.26-6.778c0 .382-.025.755-.075 1.12h-10.534c.232.936.722 1.687 1.469 2.25.755.556 1.66.833 2.715.833.813 0 1.569-.17 2.266-.51.706-.34 1.254-.766 1.644-1.28l1.93 1.654c-.673.845-1.528 1.508-2.565 1.99-1.03.48-2.146.72-3.35.72-2.059 0-3.773-.654-5.143-1.964-1.361-1.31-2.042-2.94-2.042-4.887 0-1.26.299-2.404.897-3.432a6.436 6.436 0 0 1 2.465-2.438c1.054-.588 2.221-.883 3.499-.883 1.951 0 3.574.651 4.869 1.953 1.303 1.293 1.955 2.918 1.955 4.874Zm-6.849-4.253c-.921 0-1.726.278-2.415.834-.681.555-1.138 1.289-1.37 2.2h7.583c-.216-.911-.672-1.645-1.37-2.2-.689-.556-1.498-.834-2.428-.834Zm15.99.255h-3.674v6.212c0 .597.162 1.049.486 1.355.332.299.801.448 1.407.448.573 0 1.166-.15 1.781-.448v2.674c-.706.356-1.499.535-2.379.535-1.37 0-2.403-.361-3.1-1.082-.698-.722-1.046-1.729-1.046-3.022V12.66h-2.205v-2.587h2.205V5.367h2.851v4.706h3.674v2.587ZM48.869 9.873h-.49c-1.673 0-3.033.712-4.08 2.135v-1.935h-2.887V23.25h2.887v-6.434c0-1.222.36-2.189 1.08-2.9.728-.72 1.729-1.08 3-1.08h.49V9.873Z" clip-rule="evenodd"></path></svg></div></div><div role="separator" class="bg-border-layout-soft h-(--divider-thickness) w-full" style="--divider-thickness:1.5px"></div><div class="grid grid-cols-1 laptop:grid-cols-[1fr_1fr_2fr] w-full py-12 px-12 gap-8"><div class="flex flex-col gap-4"><p class="text-headline-5 text-content-layout-1 text-left">Product</p><div class="flex flex-col gap-2"><a href="/products/readyset-cloud" class="text-content-layout-2 hover:text-content-layout-1 transition-colors">Readyset Cloud</a><a href="/products/readyset-private" class="text-content-layout-2 hover:text-content-layout-1 transition-colors">Readyset Private</a><a href="/products/readyset-community" class="text-content-layout-2 hover:text-content-layout-1 transition-colors">Readyset Community</a><a href="/tools/querypilot" class="text-content-layout-2 hover:text-content-layout-1 transition-colors">QueryPilot</a><a href="/tools/rdst" class="text-content-layout-2 hover:text-content-layout-1 transition-colors">rdst CLI</a><a href="/pricing" class="text-content-layout-2 hover:text-content-layout-1 transition-colors">Pricing</a></div></div><div class="flex flex-col gap-8"><div class="flex flex-col gap-4"><p class="text-headline-5 text-content-layout-1 text-left">Resources</p><div class="flex flex-col gap-2"><a class="text-content-layout-2 hover:text-content-layout-1 transition-colors active" href="/blog" data-status="active" aria-current="page">Blog</a><a href="/case-studies" class="text-content-layout-2 hover:text-content-layout-1 transition-colors">Case Studies</a><a href="/resources/company" class="text-content-layout-2 hover:text-content-layout-1 transition-colors">Company</a><a href="https://docs.readyset.io" target="_blank" rel="noopener noreferrer" class="text-content-layout-2 hover:text-content-layout-1 transition-colors">Documentation</a></div></div><div class="flex flex-col gap-4"><p class="text-headline-5 text-content-layout-1 text-left">Legal</p><div class="flex flex-col gap-2"><a href="/privacy-policy" class="text-content-layout-2 hover:text-content-layout-1 transition-colors">Privacy Policy</a><a href="/term-of-use" class="text-content-layout-2 hover:text-content-layout-1 transition-colors">Terms of Service</a></div></div></div><div class="flex flex-col gap-4"><p class="text-headline-5 text-content-layout-1 text-left">Join our newsletter</p><p class="text-body-small text-content-layout-3 text-left">Stay updated with the latest news, insights, and developments from Readyset — straight to your inbox.</p><form class="flex flex-col gap-2 w-full"><div class="flex gap-2 w-full"><div class="relative w-full text-content-layout-1"><input type="email" class="flex h-10 w-full rounded-lg border-(length:--border-base) border-border-layout-1 bg-surface-layout-2 px-3 py-2 text-body-medium placeholder:text-content-layout-3 focus-visible:outline-none focus-visible:shadow-focus disabled:cursor-not-allowed disabled:opacity-50" aria-label="Email address" placeholder="your@email.address" autoComplete="email" value=""/></div><button type="submit" class="relative flex items-center justify-center min-w-max cursor-pointer select-none transition duration-fast ease-base transform focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-border-primary-soft focus-visible:ring-offset-2 focus-visible:ring-offset-surface-layout-1 active:scale-[0.98] active:origin-center text-button-medium py-3 px-4 rounded-2xl h-10 text-content-primary-solid bg-surface-primary-solid hover:bg-surface-primary-solid-hover active:bg-surface-primary-solid-active [&amp;_#loader]:border-t-content-primary-solid"><div class="flex items-center">Subscribe</div></button></div></form></div></div><div role="separator" class="bg-border-layout-soft h-(--divider-thickness) w-full" style="--divider-thickness:1.5px"></div><div class="flex flex-col laptop:flex-row laptop:justify-between laptop:items-center gap-4 w-full py-12 px-12"><div class="flex items-center gap-6"><svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 145 33" class="fill-current text-content-layout-1 h-6"><path d="M16 31.328c0 .736-.597 1.333-1.333 1.333H4a4 4 0 0 1-4-4V17.994c0-.736.597-1.333 1.333-1.333H16v14.667Z"></path><path d="M21.333 11.328v5.333h-4c-.736 0-1.333.597-1.333 1.333v4h-5.333v-5.333h4c.736 0 1.333-.597 1.333-1.333v-4h5.333Z"></path><path d="M32 15.328c0 .736-.597 1.333-1.333 1.333H16V1.994c0-.736.597-1.333 1.333-1.333H28a4 4 0 0 1 4 4v10.667Z"></path><path fill-rule="evenodd" d="M91.272 5.367h2.864V23.25h-2.864v-1.99c-1.145 1.45-2.665 2.175-4.557 2.175-1.204 0-2.296-.294-3.275-.882a6.299 6.299 0 0 1-2.304-2.45c-.548-1.036-.822-2.193-.822-3.47 0-1.276.274-2.433.822-3.469a6.209 6.209 0 0 1 2.304-2.437c.98-.597 2.071-.896 3.275-.896 1.9 0 3.42.722 4.557 2.164V5.367Zm-3.997 15.308c1.146 0 2.096-.386 2.852-1.157.763-.779 1.145-1.74 1.145-2.885 0-1.152-.382-2.114-1.145-2.884-.756-.78-1.706-1.17-2.852-1.17-1.154 0-2.117.39-2.889 1.17-.764.77-1.145 1.732-1.145 2.884 0 1.145.381 2.106 1.145 2.885.772.771 1.735 1.157 2.89 1.157Zm-24.258-2.898c.05-.364.075-.737.075-1.119 0-1.956-.652-3.581-1.955-4.874-1.295-1.302-2.918-1.953-4.87-1.953-1.278 0-2.444.295-3.498.883a6.442 6.442 0 0 0-2.466 2.438c-.597 1.028-.896 2.172-.896 3.432 0 1.948.68 3.577 2.042 4.887 1.37 1.31 3.084 1.965 5.143 1.965 1.203 0 2.32-.24 3.35-.722 1.037-.48 1.892-1.144 2.564-1.99l-1.93-1.653c-.39.514-.938.94-1.643 1.28-.698.34-1.453.51-2.267.51-1.054 0-1.959-.277-2.714-.833-.747-.563-1.237-1.314-1.47-2.25h10.535Zm-9.19-4.538c.69-.556 1.494-.834 2.416-.834.93 0 1.74.278 2.428.834.697.555 1.154 1.289 1.37 2.2h-7.584c.233-.911.69-1.645 1.37-2.2Zm21.469-3.166h2.863V23.25h-2.864v-1.99c-1.145 1.45-2.664 2.175-4.557 2.175-1.204 0-2.296-.294-3.275-.882a6.299 6.299 0 0 1-2.304-2.45c-.548-1.036-.822-2.193-.822-3.47 0-1.276.274-2.433.822-3.469a6.209 6.209 0 0 1 2.304-2.437c.98-.597 2.071-.896 3.275-.896 1.901 0 3.42.722 4.558 2.164v-1.922Zm-3.998 10.602c1.146 0 2.096-.386 2.852-1.157.764-.779 1.145-1.74 1.145-2.885 0-1.152-.381-2.114-1.145-2.884-.756-.78-1.706-1.17-2.852-1.17-1.154 0-2.117.39-2.889 1.17-.763.77-1.145 1.732-1.145 2.884 0 1.145.382 2.106 1.145 2.885.772.771 1.735 1.157 2.89 1.157Zm35.186-10.602h3.188L102.941 25.6l-1.027 2.356h-3.319c.461-.939 1.313-2.68 1.718-3.537l.697-1.443-5.51-12.902h3.275l3.842 9.532 3.867-9.532Zm8.586 13.363c-2.092 0-3.894-.664-5.405-1.99l1.482-2.015c1.071 1.02 2.362 1.53 3.873 1.53.755 0 1.374-.124 1.855-.373.482-.257.722-.61.722-1.057a.93.93 0 0 0-.161-.535c-.1-.157-.225-.29-.374-.398-.141-.108-.353-.211-.635-.31-.282-.1-.531-.179-.747-.237a25.515 25.515 0 0 0-.884-.224l-.934-.224a16.527 16.527 0 0 1-.909-.286 6.683 6.683 0 0 1-.909-.373 5.835 5.835 0 0 1-.76-.497 2.716 2.716 0 0 1-.647-.672 3.57 3.57 0 0 1-.561-1.952c0-.862.241-1.6.722-2.213a4.153 4.153 0 0 1 1.868-1.343c.764-.29 1.627-.436 2.59-.436 1.901 0 3.549.577 4.944 1.729l-1.507 2.014c-.996-.845-2.125-1.268-3.387-1.268-.672 0-1.228.112-1.668.336-.432.224-.648.551-.648.982 0 .166.029.315.087.448a.93.93 0 0 0 .299.36c.15.108.291.2.423.274.133.075.328.153.586.236l.66.187.772.174c.373.083.68.157.921.224.249.058.552.149.909.273a4.943 4.943 0 0 1 1.669.87c.266.2.477.42.635.66.166.232.299.514.398.845.108.332.162.693.162 1.082 0 .87-.245 1.63-.734 2.276-.49.638-1.142 1.115-1.955 1.43-.806.315-1.723.473-2.752.473Zm20.26-6.778c0 .382-.025.755-.075 1.12h-10.534c.232.936.722 1.687 1.469 2.25.755.556 1.66.833 2.715.833.813 0 1.569-.17 2.266-.51.706-.34 1.254-.766 1.644-1.28l1.93 1.654c-.673.845-1.528 1.508-2.565 1.99-1.03.48-2.146.72-3.35.72-2.059 0-3.773-.654-5.143-1.964-1.361-1.31-2.042-2.94-2.042-4.887 0-1.26.299-2.404.897-3.432a6.436 6.436 0 0 1 2.465-2.438c1.054-.588 2.221-.883 3.499-.883 1.951 0 3.574.651 4.869 1.953 1.303 1.293 1.955 2.918 1.955 4.874Zm-6.849-4.253c-.921 0-1.726.278-2.415.834-.681.555-1.138 1.289-1.37 2.2h7.583c-.216-.911-.672-1.645-1.37-2.2-.689-.556-1.498-.834-2.428-.834Zm15.99.255h-3.674v6.212c0 .597.162 1.049.486 1.355.332.299.801.448 1.407.448.573 0 1.166-.15 1.781-.448v2.674c-.706.356-1.499.535-2.379.535-1.37 0-2.403-.361-3.1-1.082-.698-.722-1.046-1.729-1.046-3.022V12.66h-2.205v-2.587h2.205V5.367h2.851v4.706h3.674v2.587ZM48.869 9.873h-.49c-1.673 0-3.033.712-4.08 2.135v-1.935h-2.887V23.25h2.887v-6.434c0-1.222.36-2.189 1.08-2.9.728-.72 1.729-1.08 3-1.08h.49V9.873Z" clip-rule="evenodd"></path></svg><ul class="flex items-center gap-3"><li><a href="https://github.com/readysettech/readyset" aria-label="Readyset on GitHub" target="_blank" rel="noopener noreferrer" class="text-content-layout-3 hover:text-content-layout-1 transition-colors block"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true" class="size-5"><path fill-rule="evenodd" clip-rule="evenodd" d="M12 2C6.477 2 2 6.484 2 12.017c0 4.425 2.865 8.18 6.839 9.504.5.092.682-.217.682-.483 0-.237-.008-.868-.013-1.703-2.782.605-3.369-1.343-3.369-1.343-.454-1.158-1.11-1.466-1.11-1.466-.908-.62.069-.608.069-.608 1.003.07 1.531 1.032 1.531 1.032.892 1.53 2.341 1.088 2.91.832.092-.647.35-1.088.636-1.338-2.22-.253-4.555-1.113-4.555-4.951 0-1.093.39-1.988 1.029-2.688-.103-.253-.446-1.272.098-2.65 0 0 .84-.27 2.75 1.026A9.564 9.564 0 0 1 12 6.844c.85.004 1.705.115 2.504.337 1.909-1.296 2.747-1.027 2.747-1.027.546 1.379.202 2.398.1 2.651.64.7 1.028 1.595 1.028 2.688 0 3.848-2.339 4.695-4.566 4.943.359.31.678.92.678 1.855 0 1.338-.012 2.419-.012 2.747 0 .268.18.58.688.482A10.02 10.02 0 0 0 22 12.017C22 6.484 17.522 2 12 2Z"></path></svg></a></li><li><a href="https://www.linkedin.com/company/readysettech" aria-label="Readyset on LinkedIn" target="_blank" rel="noopener noreferrer" class="text-content-layout-3 hover:text-content-layout-1 transition-colors block"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true" class="size-5"><path d="M20.447 20.452h-3.554v-5.569c0-1.328-.026-3.037-1.852-3.037-1.853 0-2.136 1.445-2.136 2.939v5.667H9.351V9h3.414v1.561h.046c.477-.9 1.637-1.85 3.37-1.85 3.601 0 4.267 2.37 4.267 5.455v6.286ZM5.337 7.433a2.062 2.062 0 0 1-2.063-2.065 2.063 2.063 0 1 1 2.063 2.065Zm1.782 13.019H3.555V9H7.12v11.452ZM22.225 0H1.771C.792 0 0 .774 0 1.729v20.542C0 23.226.792 24 1.771 24h20.451C23.2 24 24 23.226 24 22.271V1.729C24 .774 23.2 0 22.222 0h.003Z"></path></svg></a></li><li><a href="https://x.com/readysetio" aria-label="Readyset on X" target="_blank" rel="noopener noreferrer" class="text-content-layout-3 hover:text-content-layout-1 transition-colors block"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true" class="size-5"><path d="M18.244 2.25h3.308l-7.227 8.26 8.502 11.24H16.17l-5.214-6.817L4.99 21.75H1.68l7.73-8.835L1.254 2.25H8.08l4.713 6.231 5.45-6.231Zm-1.161 17.52h1.833L7.084 4.126H5.117L17.083 19.77Z"></path></svg></a></li><li><a href="https://www.youtube.com/@readyset" aria-label="Readyset on YouTube" target="_blank" rel="noopener noreferrer" class="text-content-layout-3 hover:text-content-layout-1 transition-colors block"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true" class="size-5"><path d="M23.498 6.186a3.016 3.016 0 0 0-2.122-2.136C19.505 3.545 12 3.545 12 3.545s-7.505 0-9.377.505A3.017 3.017 0 0 0 .502 6.186C0 8.07 0 12 0 12s0 3.93.502 5.814a3.016 3.016 0 0 0 2.122 2.136c1.871.505 9.376.505 9.376.505s7.505 0 9.377-.505a3.015 3.015 0 0 0 2.122-2.136C24 15.93 24 12 24 12s0-3.93-.502-5.814ZM9.545 15.568V8.432L15.818 12l-6.273 3.568Z"></path></svg></a></li><li><a href="https://go.readyset.io/community" aria-label="Readyset community Slack" target="_blank" rel="noopener noreferrer" class="text-content-layout-3 hover:text-content-layout-1 transition-colors block"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true" class="size-5"><path d="M5.042 15.165a2.528 2.528 0 0 1-2.52 2.523A2.528 2.528 0 0 1 0 15.165a2.527 2.527 0 0 1 2.522-2.52h2.52v2.52ZM6.313 15.165a2.527 2.527 0 0 1 2.521-2.52 2.527 2.527 0 0 1 2.521 2.52v6.313A2.528 2.528 0 0 1 8.834 24a2.528 2.528 0 0 1-2.521-2.522v-6.313ZM8.834 5.042a2.528 2.528 0 0 1-2.521-2.52A2.528 2.528 0 0 1 8.834 0a2.528 2.528 0 0 1 2.521 2.522v2.52H8.834ZM8.834 6.313a2.528 2.528 0 0 1 2.521 2.521 2.528 2.528 0 0 1-2.521 2.521H2.522A2.528 2.528 0 0 1 0 8.834a2.528 2.528 0 0 1 2.522-2.521h6.312ZM18.956 8.834a2.528 2.528 0 0 1 2.522-2.521A2.528 2.528 0 0 1 24 8.834a2.528 2.528 0 0 1-2.522 2.521h-2.522V8.834ZM17.688 8.834a2.528 2.528 0 0 1-2.523 2.521 2.527 2.527 0 0 1-2.52-2.521V2.522A2.527 2.527 0 0 1 15.165 0a2.528 2.528 0 0 1 2.523 2.522v6.312ZM15.165 18.956a2.528 2.528 0 0 1 2.523 2.522A2.528 2.528 0 0 1 15.165 24a2.527 2.527 0 0 1-2.52-2.522v-2.522h2.52ZM15.165 17.688a2.527 2.527 0 0 1-2.52-2.523 2.526 2.526 0 0 1 2.52-2.52h6.313A2.527 2.527 0 0 1 24 15.165a2.528 2.528 0 0 1-2.522 2.523h-6.313Z"></path></svg></a></li></ul></div><div class="flex flex-col laptop:flex-row laptop:items-center gap-2 laptop:gap-6"><button type="button" class="text-content-layout-2 hover:text-content-layout-1 transition-colors text-left cursor-pointer">Cookie preferences</button><p class="text-body-large text-content-layout-2">© <!-- -->2026<!-- --> Readyset. All rights reserved.</p></div></div></div></div></div><!--/$--><!--/$--></main><script nonce="goa3e761xWwsor0s+F8w/g==" class="$tsr" id="$tsr-stream-barrier">(self.$R=self.$R||{})["tsr"]=[];self.$_TSR={h(){this.hydrated=!0,this.c()},e(){this.streamEnded=!0,this.c()},c(){this.hydrated&&this.streamEnded&&(delete self.$_TSR,delete self.$R.tsr)},p(e){this.initialized?e():this.buffer.push(e)},buffer:[]};$_TSR.router=($R=>$R[0]={manifest:$R[1]={routes:$R[2]={__root__:$R[3]={preloads:$R[4]=["/assets/index-CgozBq9o.js"],scripts:$R[5]=[$R[6]={attrs:$R[7]={type:"module",async:!0,src:"/assets/index-CgozBq9o.js"}}]},"/blog/$slug":$R[8]={preloads:$R[9]=["/assets/_slug-feOgb-yl.js","/assets/article-card-BslrmY68.js","/assets/newsletter-popup-C0xZnHQP.js","/assets/rich-markdown-DYApPWH1.js","/assets/index-rMShz_NE.js","/assets/use-disclosure-DaTsKJMJ.js"]}}},matches:$R[10]=[$R[11]={i:"__root__",u:1789054058506,s:"success",l:$R[12]={siteSettings:$R[13]={top_banner:$R[14]={enabled:!1,message:"",href:"",modifier:"info"},newsletter_popup:$R[15]={homepage:$R[16]={enabled:!0,title:"",description:""},blog:$R[17]={enabled:!0,title:"",description:""},blog_post:$R[18]={enabled:!0,title:"",description:""}},footer_tagline:"Revolutionize your database performance with Readyset",footer_subtagline:"Serve requests at sub-millisecond latencies with the modern database scaling and query caching system for MySQL and PostgreSQL.",footer_cta:$R[19]={title:"Still scaling the hard way?",description:"Modern applications demand instant performance, even under unpredictable load. Readyset helps you eliminate slow queries, stabilize latency, and scale confidently.",actions:$R[20]=[$R[21]={label:"Contact Sales",variant:"primary",modifier:"outline",full_width:!1,href:"/book-a-demo"},$R[22]={label:"Start for free",variant:"primary",modifier:"solid",full_width:!1,href:"/book-a-demo"}]},nav_links:$R[23]=[$R[24]={label:"Products",href:"/products/readyset-cloud",children:$R[25]=[$R[26]={label:"Readyset Community",href:"/products/readyset-community",description:"Run Readyset on your own infrastructure with full control and flexibility.",icon:"oval"},$R[27]={label:"Readyset Private",href:"/products/readyset-private",description:"Self-hosted enterprise deployment with licensed support.",icon:"secured-network"},$R[28]={label:"Readyset Cloud",href:"/products/readyset-cloud",description:"Fully managed caching for Postgres and MySQL, no infrastructure required.",icon:"cloud-server"}]},$R[29]={label:"Solutions",href:"/solutions/instant-performance",children:$R[30]=[$R[31]={label:"Instant Performance",href:"/solutions/instant-performance",description:"Sub-millisecond reads at scale.",icon:""},$R[32]={label:"Database Cost Efficiency",href:"/solutions/database-cost-efficiency",description:"Scale reads without scaling spend.",icon:""},$R[33]={label:"Drop-in Deployment",href:"/solutions/drop-in-deployment",description:"No rewrites, no schema changes.",icon:""},$R[34]={label:"Real-Time Cache Freshness",href:"/solutions/real-time-cache-freshness",description:"Cached reads that stay correct as data changes.",icon:""},$R[35]={label:"Automatic Query Caching",href:"/solutions/automatic-query-caching",description:"Auto-identified, auto-maintained caches.",icon:""},$R[36]={label:"Agent Workloads",href:"/solutions/agent-workloads",description:"Protect databases from agent-driven queries.",icon:""}]},$R[37]={label:"Resources",href:"/blog",children:$R[38]=[$R[39]={label:"Blog",href:"/blog",description:"",icon:""},$R[40]={label:"Case Studies",href:"/case-studies",description:"",icon:""},$R[41]={label:"Pricing",href:"/pricing",description:"",icon:""},$R[42]={label:"Company",href:"/resources/company",description:"",icon:""}]}],footer_links:$R[43]=[$R[44]={column_label:"Product",items:$R[45]=[$R[46]={label:"Readyset Cloud",href:"/products/readyset-cloud",external:!1},$R[47]={label:"Readyset Private",href:"/products/readyset-private",external:!1},$R[48]={label:"Readyset Community",href:"/products/readyset-community",external:!1},$R[49]={label:"QueryPilot",href:"/tools/querypilot",external:!1},$R[50]={label:"rdst CLI",href:"/tools/rdst",external:!1},$R[51]={label:"Pricing",href:"/pricing",external:!1}]},$R[52]={column_label:"Resources",items:$R[53]=[$R[54]={label:"Blog",href:"/blog",external:!1},$R[55]={label:"Case Studies",href:"/case-studies",external:!1},$R[56]={label:"Company",href:"/resources/company",external:!1},$R[57]={label:"Documentation",href:"https://docs.readyset.io",external:!0}]},$R[58]={column_label:"Legal",items:$R[59]=[$R[60]={label:"Privacy Policy",href:"/privacy-policy",external:!1},$R[61]={label:"Terms of Service",href:"/term-of-use",external:!1}]}],social_links:$R[62]=[$R[63]={platform:"github",href:"https://github.com/readysettech/readyset",label:"Readyset on GitHub"},$R[64]={platform:"linkedin",href:"https://www.linkedin.com/company/readysettech",label:"Readyset on LinkedIn"},$R[65]={platform:"twitter",href:"https://x.com/readysetio",label:"Readyset on X"},$R[66]={platform:"youtube",href:"https://www.youtube.com/@readyset",label:"Readyset on YouTube"},$R[67]={platform:"slack",href:"https://go.readyset.io/community",label:"Readyset community Slack"}],logo:$R[68]={full:"",icon:"",alt:"Readyset"},slug:"default"}},ssr:!0},$R[69]={i:"blog$slugbloginvestigating-and-optimizing-over-querying",u:1789054058509,s:"success",l:$R[70]={article:$R[71]={id:"3565807c-0e0f-452a-b700-9c25b6820f82",slug:"investigating-and-optimizing-over-querying",title:"How to Solve for the N+1 Query Problem: Investigating and Optimizing Over-Querying",excerpt:"Imagine you're running a popular e-commerce online bookstore that offers a vast collection of titles and authors to a growing user base. However, you've noticed a troubling trend over the past few months: the website is gradually slowing down, especially during peak hours when users browse various book categories. After an initial investigation, you find that the cause of the slowdown isn't an increase in user traffic or a lack of server resources. Instead, it's rooted in the very foundation of ",updatedAt:1777903709106,category:$R[72]={id:"302bcdcd-bcba-480d-a1ca-7b6764a38037",slug:"postgres",name:"PostgreSQL"},tags:$R[73]=[$R[74]={id:"f80447a2-0587-4763-893b-0c935d759535",slug:"postgres",name:"PostgreSQL"}],image:"/images/blog-placeholder.svg",hasImage:!1,date:"2024-02-14",readTime:"14 min read",authors:$R[75]=[$R[76]={id:"552f3435-f728-4436-a6a1-23dba36ed422",slug:"neda",name:"Readyset",avatar:"https://cms.readyset.io/files/authors/657385d2b4c219000125f4cd/c0376b666550a188861d4a2d.bin",bio:""}],author:$R[76],content:"Imagine you're running a popular e-commerce online bookstore that offers a vast collection of titles and authors to a growing user base. However, you've noticed a troubling trend over the past few months: the website is gradually slowing down, especially during peak hours when users browse various book categories. After an initial investigation, you find that the cause of the slowdown isn't an increase in user traffic or a lack of server resources. Instead, it's rooted in the very foundation of how your application interacts with your Postgres database.\n\nThe culprit? N+1 query problems. As more users navigate your site, more requests are made to the database to fetch information. Instead of being efficiently retrieved in grouped queries, each request individually pulls associated data like author details, reviews, and related books. What should have been a streamlined operation has turned into a burdensome load on your database, leading to longer load times and a compromised user experience.\n\nThis scenario is not unique to your online bookstore. Regardless of size or domain, many applications encounter similar performance bottlenecks due to N+1 queries. Understanding the nature of these queries, their impact on database performance, and how to optimize them is crucial for developers and database administrators. Here, we’re going into N+1 queries in a Postgres environment, providing insights and strategies to turn a potential database nightmare into a well-optimized, efficient system.\n\n## What Are N+1 Queries?\n\nN+1 queries are a common performance bottleneck in databases. This issue occurs when an application performs an initial query to retrieve a set of records, followed by additional queries for each individual record. The name 'N+1' stems from making one (1) initial query and then N additional queries, resulting in N+1 total queries for N records.\n\nLet’s say you have an application that displays user profiles and their respective posts. The application first executes a query to fetch all users. This is the \"1\" in N+1\\. The application performs another query for each user retrieved to fetch their posts. If there are ten users, this results in 10 additional queries (the \"N\" in N+1), totaling 11 queries. While this approach may seem straightforward, it's highly inefficient, especially as the number of users grows.\n\nLet’s look at what this looks like. Assume you have two tables: `users` and `posts`. Each user has multiple posts. The `posts` table has a foreign key that references the `users` table. You want to display each user along with their posts.\n\n- **Users Table:** \n - `id` (Primary Key) \n - `name`\n- **Posts Table:** \n - `id` (Primary Key) \n - `content` \n - `user_id` (Foreign Key to Users)\n\nA naive N+1 query to get these posts might look like this: \n \n```unset\n-- Query 1: Fetch all users\nSELECT id, name FROM users;\n\n-- For each user obtained from the above query, execute the following query:\nSELECT content FROM posts WHERE user_id = [user_id];\n```\n\nHere, Query 1 is the \"1\" query. Query 2 will be the “N” queries, where you iterate through the `user_id`s.\n\nAs a result, if there are 100 users, the total number of queries executed will be 101: 1 for fetching all users and 100 for fetching the posts for each user. This is a classic example of an N+1 query problem and can lead to performance issues, especially with a large number of users and posts.\n\n### N+1 Queries in ORMs\n\nIt’s common to find N+1 queries in frameworks using Object-Relational Mappings (ORMs). These are designed to convert models in an application into SQL statements to query data in relational databases. However, they often exacerbate the N+1 query issue due to their internal mechanisms.\n\nLet’s use the Python web framework Django as our example. In Django, an N+1 query problem can quickly occur when you have related models and access related data without properly optimizing your queries. Let's consider a scenario where each `User` has a foreign key to a `Profile` model: \n \n```python\n# models.py\nfrom django.db import models\n\nclass Profile(models.Model):\n bio = models.TextField()\n # other fields like date_of_birth, location, etc.\n\nclass User(models.Model):\n name = models.CharField(max_length=100)\n profile = models.OneToOneField(Profile, on_delete=models.CASCADE)\n # other fields like email, etc.\n\n# views.py\nfrom django.shortcuts import render\nfrom .models import User\n\ndef user_list(request):\n users = User.objects.all()\n user_profiles = []\n for user in users:\n profile = user.profile # This creates the N+1 problem\n user_profiles.append((user, profile))\n\n context = {'user_profiles': user_profiles}\n return render(request, 'user_list.html', context)\n```\n\nHere, the `User.objects.all()` query retrieves all users. This is the \"1\" in N+1\\. Inside the loop, accessing each user's profile may result in a separate database query to fetch the corresponding `Profile` instance. This is the \"N\" part of N+1, where N is the number of users.\n\nBecause ORMs abstract away the underlying SQL queries, developers might not immediately realize that their code generates these inefficient N+1 queries.\n\n## The Impact of N+1 Queries on Performance\n\nN+1 queries impact performance, especially in large-scale systems. These queries lead to inefficient data retrieval, increased load on the database, and, ultimately, a poor user experience. Understanding the consequences of N+1 queries is crucial for database optimization.\n\n### Increased Database Load\n\nEach additional query in an N+1 problem adds to the load on the database. This is particularly problematic with large datasets. For instance, if an application retrieves 1000 users and makes a separate query for each user's posts, it results in 1001 queries hitting the database instead of potentially just one. This extra load can slow down the database response times, affecting all application users.\n\nImagine a web application displaying a list of users and their recent activities. Without optimization, fetching 1,000 users might result in 1,001 queries (one to fetch all users and 1,000 to fetch activities for each). This heavy load can lead to longer wait times for the data to load, affecting user experience.\n\nA high volume of queries also consumes more CPU and memory resources on the database server. In our previous example, instead of a single, efficient query, the server must process 1,001 queries, consuming more resources. This impacts the query in question and affects the overall efficiency of the database server, hindering its ability to handle other requests.\n\n### Scaling Challenges\n\nApplications with N+1 query problems often struggle to scale. As the data grows, so does the number of queries. In a social media app, for example, as more users join and create posts, the N+1 issue exacerbates, leading to an exponential increase in queries. This scaling challenge can cause significant performance degradation over time, requiring more hardware resources to maintain the same level of performance.\n\nThe total time complexity for the N+1 queries pattern is O(N). As the number of records (N) increases, the total number of queries increases linearly. This linear growth can lead to significant performance degradation, especially with large datasets. Each query incurs a certain amount of overhead due to network latency, query parsing, execution planning, and data retrieval. This overhead, multiplied by the number of queries, can substantially affect performance.\n\n### Poor User Experience\n\nThe cumulative effect of these performance issues can result in longer loading times for users, negatively impacting the user experience. In an e-commerce site, if product details are fetched with N+1 queries, each additional millisecond in load time can potentially lead to lost sales as customers grow impatient and leave the site.\n\nConsider a real-time data dashboard that monitors and displays various metrics. If each metric's data is fetched using separate queries for each element, the dashboard will experience noticeable delays, failing to deliver the real-time experience expected by users.\n\n## Detecting and Investigating N+1 Queries\n\nSometimes, the simplest way to detect N+1 queries is through careful code review. Reviewers can look for loops or iterative processes that make database calls, particularly in the context of ORMs or when accessing related data.\n\nBut N+1 queries can often be subtle and not immediately evident, especially in complex applications. Here are some effective strategies and tools that can be employed to detect and investigate N+1 queries:\n\n### Query Logging\n\nEnabling query logging in Postgres is one of the first steps in detecting N+1 query problems. Logging all executed queries allows you to analyze the logs for patterns that indicate N+1 issues.\n\n```\n-- Set the logging level to log all statements\nALTER DATABASE your_database_name SET log_statement = 'all';\n```\n\nAfter enabling logging, look for sequences of similar queries that differ only in a parameter, such as multiple queries fetching details for different user IDs.\n\n```\n2024-01-23 10:00:01 UTC LOG: statement: SELECT * FROM users\n2024-01-23 10:00:02 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 1\n2024-01-23 10:00:02 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 2\n2024-01-23 10:00:02 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 3\n2024-01-23 10:00:02 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 4\n2024-01-23 10:00:02 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 5\n2024-01-23 10:00:03 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 6\n2024-01-23 10:00:03 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 7\n2024-01-23 10:00:03 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 8\n2024-01-23 10:00:03 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 9\n2024-01-23 10:00:03 UTC LOG: statement: SELECT * FROM posts WHERE user_id = 10\n...\n```\n\nThis is a strong indicator of N+1 queries.\n\n### Performance Monitoring Tools\n\nTools like[ pgBadger](https://github.com/darold/pgbadger?ref=blog.readyset.io),[ DataDog](https://www.datadoghq.com/blog/database-performance-monitoring-datadog/?ref=blog.readyset.io), or[ ScoutAPM](https://scoutapm.com/blog/understanding-n1-database-queries?ref=blog.readyset.io) offer detailed insights into database performance. They can help identify inefficient query patterns that may suggest N+1 issues.\n\nWith pgBadger, you can analyze your Postgres logs to get a report highlighting frequently executed queries. A high frequency of similar queries can be a sign of N+1 problems.\n\n![](https://cms.readyset.io/files/articles/65cb6bb6fbd2870001265837/c710994e211d520d8fe0dd35.png)\n\nThe `SELECT * FROM posts WHERE user_id = ?` query stands out due to its high frequency of execution (850 times). This pattern is indicative of an N+1 query problem. It suggests that for each user fetched by the `SELECT * FROM users` query (executed ten times), there are multiple subsequent queries to fetch posts. The average duration of each `posts` query is low (5ms), but the cumulative impact (total duration of 4250ms) is significant, pointing to a potential performance issue.\n\nThe disparity between the number of executions of the `users` query and the `posts` query is a classic sign of N+1 queries. From this, the recommendations might be to investigate the application code following the execution of `SELECT * FROM users`, especially the parts where posts for each user are accessed or displayed. Then, developers can optimize the query pattern to use JOIN operations or batch processing to reduce the total number of queries.\n\nAn application performance monitoring tool like DataDog or Scout APM will often automatically highlight inefficiencies in an application, such as N+1 queries.\n\n![](https://cms.readyset.io/files/articles/65cb6bb6fbd2870001265837/1fe58061ad314ed7a931938d.bin)\n\nThey can show the response times for these queries, the number of calls, and the query itself. This kind of report helps pinpoint where the application might be inefficiently using the database, guiding developers toward specific areas of the code that may need optimization to address the N+1 query problem.\n\nLanguages also have profiling tools built in, such as Python's `cProfile`. These can help detect N+1 queries by showing where your application spends most of its time.\n\n```python\nimport cProfile\ncProfile.run('function_that_loads_data()')\n```\n\nThis profiling can help identify functions that are making excessive database calls. A large amount of time spent in database-related functions could indicate N+1 queries.\n\nBeyond language tools, most ORMs allow enabling SQL debug logging. This feature logs every SQL query the ORM executes, making spotting repetitive query patterns indicative of N+1 problems easier. Keeping with the Python theme, in Django, SQL debug logging can be activated by setting `DEBUG = True` in your `settings.py`, which causes all SQL queries to be printed to the console during development. You can also use Django's `django.db.connection.queries` for query insights.\n\n### EXPLAIN Command\n\nThe `EXPLAIN` command in Postgres is an invaluable tool for understanding how your queries are being executed. It can help you identify queries that do not efficiently use indexes or perform full table scans, which could be part of an N+1 problem.\n\n```\nEXPLAIN SELECT * FROM posts WHERE user_id = 1;\n```\n\nThe output provides insights into the query execution plan, which can help identify inefficiencies.\n\n```\nQUERY PLAN\n-----------------------------------------------------------\n Seq Scan on posts (cost=0.00..35.50 rows=1560 width=2048)\n Filter: (user_id = 1)\n```\n\nHere, `EXPLAIN` tells us we perform a “Seq Scan on posts.” This indicates that a sequential scan is being performed on the `posts` table. Sequential scans are generally less efficient than index scans, especially for larger tables, as they involve scanning each row.\n\nThe sequential scan (Seq Scan) on the `posts` table might not be a problem for a single query. However, suppose similar queries are being executed repeatedly for different `user_id` values (as in an N+1 scenario). In that case, it indicates that the database is performing numerous full table scans, which can be highly inefficient.\n\nThe absence of an index scan suggests that there might not be an index on the `user_id` column or the query planner did not find it efficient to use the index. For an N+1 query pattern, the repeated execution of such full table scans can significantly degrade performance.\n\n## Optimization and Query Design\n\nOptimization and better query design are the only strategies to significantly reduce the impact of N+1 query problems in a Postgres environment. Doing so naturally leads to better application performance and scalability.\n\n### Eager loading\n\nWhen using ORMs, the best solution (given you can’t optimize the query yourself) is eager loading. Eager loading involves modifying the query to fetch all related data in a single query instead of separate queries for each record. This can be achieved in ORMs with specific methods or query options (like `.include` in Rails or `select_related` in Django). These reduce the number of queries to the database, improving performance.\n\nIf we want to optimize our Django example above, this can be achieved through techniques like `select_related` or `prefetch_related`, designed to handle database queries more efficiently for related objects.\n\nAssuming the `User` model has a ForeignKey to another model, let's say a `Profile` model, here's an optimized version of the Django code that avoids the N+1 query problem:\n\n```python\n# models.py\nfrom django.db import models\n\nclass Profile(models.Model):\n bio = models.TextField()\n # other fields\n\nclass User(models.Model):\n name = models.CharField(max_length=100)\n profile = models.OneToOneField(Profile, on_delete=models.CASCADE)\n # other fields\n\nclass Post(models.Model):\n user = models.ForeignKey(User, related_name='posts', on_delete=models.CASCADE)\n content = models.TextField()\n # other fields\n\n# views.py\nfrom django.shortcuts import render\nfrom .models import User\n\ndef user_profiles(request):\n # Use 'select_related' to fetch the related Profile in the same query\n users = User.objects.select_related('profile').all()\n\n # 'prefetch_related' is used for reverse ForeignKey relationships\n # This fetches all related posts in a separate query, reducing the overall number of queries\n users = users.prefetch_related('posts')\n\n return render(request, 'user_profiles.html', {'users': users})\n```\n\nThe `select_related('profile')` method is used with the `User.objects.all()` query. This fetches the associated `Profile` for each `User` in the same database query, thus avoiding separate queries for each user's profile.\n\nThe `prefetch_related('posts')` method handles the reverse ForeignKey relationship from `User` to `Post`. It performs a separate query to fetch all related posts and then efficiently pairs them with the corresponding users, which is more efficient than doing individual queries for each user's posts.\n\nThis approach significantly reduces the number of queries, particularly when you have a large number of users and posts, thus improving the performance of your Django application.\n\n### Caching\n\nIf you have particular queries you want to return fast, you can implement caching to reduce the need to query the database each time.\n\nReadyset allows you to cache SQL queries without changes to your application code. The only change needed is to swap out your primary Postgres database connection string with a Readyset connection string. Readset will connect to your primary database and register with the replication stream. After snapshotting your database, every query will be proxied through Readyset. Along with your monitoring tools from above, you can then also use Readyset to understand your query performance:\n\n![](https://cms.readyset.io/files/articles/65cb6bb6fbd2870001265837/2734aa9275276207973d0dba.bin)\n\nWhen you have detected N+1 queries that can be cached (usually, ready-heavy queries are optimal candidates), you can start caching. In this case, we want to cache our posts queries. To do so, we just prepend `CREATE CACHE FROM` to our query: \n \n```python\nCREATE CACHE FROM SELECT * FROM posts WHERE user_id = ?;\n```\n\nThe results from this query will now be served from Readyset with sub-millisecond latencies. Readyset monitors the replication stream from the primary database, looking for changes to the data, so the cache will be automatically updated whenever the underlying data is updated in the primary database.\n\nReadyset also works with ORMs to help increase performance and optimize those queries under the hood.\n\n### Query design\n\nJust as you can optimize your system around your queries, if you aren’t using an ORM, you can also optimize your queries for better performance. Here are a few options:\n\n- **Use Joins Appropriately**: Utilize SQL joins to combine data from multiple tables in a single query. Understand the differences between `INNER JOIN`, `LEFT JOIN`, etc., and choose the most appropriate use case to avoid unnecessary data retrieval.\n- **Select Only Required Columns**: Specify the columns you need in your `SELECT` statements rather than using `SELECT *`. This reduces the amount of data transferred and processed, making the query more efficient.\n- **Understand and Use Indices**: Ensure your queries effectively leverage indices, especially for columns in `JOIN`, `WHERE`, or `ORDER BY` clauses. Regularly review and optimize indices based on query patterns and data changes.\n- **Avoid Looping Queries**: Identify scenarios where your code iteratively executes queries (like in a loop) and refactor them to use bulk data retrieval techniques. Replace multiple small queries with fewer, more comprehensive queries.\n- **Limit Data with Pagination**: When dealing with large datasets, use pagination to limit the data retrieved and processed in a single query. This improves both database and application performance.\n\nThus, to optimize the basic queries from above, you could add a join into your query to fetch all the data at once:\n\n```\nSELECT u.id, u.name, p.content\nFROM users u\nLEFT JOIN posts p ON u.id = p.user_id\n```\n\n## Solving the N+1 Puzzle\n\nN+1 queries can destroy your application’s performance as you scale. What’s worse is that this issue can be abstracted from you by ORMs and other high-level frameworks, making them harder to detect and resolve. But with monitoring tools and a better understanding of the problem, you can detect these troublesome queries and work to improve your query design and optimize your performance.\n\nAt Readyset, we are building ways to help developers and platform engineers do just that. By caching queries, you can significantly improve your response latencies while still serving your users' fresh data. If this sounds like something that will help you, sign up for [Readyset Cloud](https://readyset.io/?utm%5Fcampaign=eg&utm%5Fmedium=internal&utm%5Fsource=blog) or[ reach out to us](https://go.readyset.io/community?ref=blog.readyset.io) if you have any questions.",seo:null},author:$R[76],related:$R[77]=[$R[78]={id:"0006e2e6-3237-4574-bc20-74d8e6d9df8c",slug:"getting-started-with-django-and-readyset",title:"Getting started with Django, PostgreSQL, and Readyset",excerpt:"Real-world applications face performance challenges with growing data and user traffic. Caching mitigates these challenges by storing frequently accessed data, reducing database fetches, improving response times, and enhancing application scalability, reliability, and user experience.\n\nReadyset is a caching engine for Postgres and MySQL databases, requiring minimal adjustments to existing setups. Its wire-compatible design means that the only modification necessary is to adjust the connection st",updatedAt:1777903642403,category:$R[72],tags:$R[79]=[$R[74]],image:"/images/blog-placeholder.svg",hasImage:!1,date:"2024-05-16",readTime:"11 min read",authors:$R[80]=[$R[81]={id:"b0256172-54fb-43e9-8a72-098f53d00199",slug:"rishi",name:"Rishi Raj Jain",avatar:"https://cms.readyset.io/files/authors/6644e32196ffac000112bdee/d3c2963169bf4bcdcbc43d2c.jpeg",bio:""}],author:$R[81],content:"",seo:null},$R[82]={id:"1718934a-60e9-4594-bb4e-968eb6059c01",slug:"optimizing-sql-pagination-in-postgres",title:"Optimizing SQL Pagination in Postgres",excerpt:"SQL pagination is a technique to retrieve a subset of records from a Postgres table or result set. It allows you to divide large datasets into smaller, manageable chunks called \"pages\" and retrieve them one page at a time.\n\n\n\nUnderstanding Pagination in Postgres\n\n\nPagination is commonly used in applications that display data in a paginated format, such as search results, product listings, or user records. It helps improve performance and user experience by loading and displaying only a portion o",updatedAt:1777903649447,category:$R[72],tags:$R[83]=[$R[74]],image:"/images/blog-placeholder.svg",hasImage:!1,date:"2024-04-24",readTime:"13 min read",authors:$R[84]=[$R[76]],author:$R[76],content:"",seo:null},$R[85]={id:"d87eefb8-c230-4216-b4cc-260d75e2fbe1",slug:"mastering-query-rewriting-for-faster-postgresql-performance",title:"Mastering Query Rewriting for Faster PostgreSQL Performance",excerpt:"When you first spin up your app, the emphasis is on getting started and getting data to your clients. But when you don’t have throughput, you are also not going to have enough concurrency to unveil bad queries.\n\nBut then you have success. And success means data. More users, more interactions, more everything. Suddenly, queries that performed fine are struggling under the load, hurting performance and scalability. This is all going to mean a far worse user experience when much higher costs becaus",updatedAt:1777903651835,category:$R[72],tags:$R[86]=[$R[74]],image:"/images/blog-placeholder.svg",hasImage:!1,date:"2024-04-11",readTime:"9 min read",authors:$R[87]=[$R[88]={id:"4172a81f-5f52-449b-bf38-4a7c4e3dfb1f",slug:"marcelo",name:"Marcelo Altmann",avatar:"https://cms.readyset.io/files/authors/65b15d7c857d140001d2f6de/20ce2d4a31bccdca053a0d7a.jpg",bio:"Marcelo works as a Senior Software Engineer @ Readyset."}],author:$R[88],content:"",seo:null}],preview:!1},ssr:!0}],lastMatchId:"blog$slugbloginvestigating-and-optimizing-over-querying",dehydratedData:$R[89]={queryStream:$R[90]=($R[91]=(e) => new ReadableStream({ start: (r) => {
e.on({ next: (a) => {
try {
r.enqueue(a);
} catch (t) {
}
}, throw: (a) => {
r.error(a);
}, return: () => {
try {
r.close();
} catch (a) {
}
} });
} }))($R[92]=($R[93]=() => {
let e = [], r = [], t = true, n2 = false, a = 0, s = (l, g, S) => {
for (S = 0; S < a; S++) r[S] && r[S][g](l);
}, i = (l, g, S, d) => {
for (g = 0, S = e.length; g < S; g++) d = e[g], !t && g === S - 1 ? l[n2 ? "return" : "throw"](d) : l.next(d);
}, u = (l, g) => (t && (g = a++, r[g] = l), i(l), () => {
t && (r[g] = r[a], r[a--] = void 0);
});
return { __SEROVAL_STREAM__: true, on: (l) => u(l), next: (l) => {
t && (e.push(l), s(l, "next"));
}, throw: (l) => {
t && (e.push(l), s(l, "throw"), t = false, n2 = false, r.length = 0);
}, return: (l) => {
t && (e.push(l), s(l, "return"), t = false, n2 = true, r.length = 0);
} };
})())}})($R["tsr"]);document.currentScript.remove()</script><script type="module" async="" src="/assets/index-CgozBq9o.js" nonce="goa3e761xWwsor0s+F8w/g=="></script><script nonce='goa3e761xWwsor0s+F8w/g=='>($R=>$R[92].return(void 0))($R["tsr"]);$_TSR.e();document.currentScript.remove()</script><script type="module" src="https://static.cloudflareinsights.com/beacon.min.js/v31edd6df95cf4e85bb4c19e7a9bdbcba1788362987495" integrity="sha512-iIg7k2xntmwu6/uSb5tpc/hySgZc4eoL31yB29W6tJFo2akwjPWcEqnCEdJvGexCL0KEQwVYv5BlowfhVz26hg==" nonce="goa3e761xWwsor0s+F8w/g==" data-cf-beacon='{"version":"2024.11.0","token":"d483087ab45f45f6a431c22e81c6c7f9","r":1,"spa":2}' crossorigin="anonymous"></script>
</body></html>