Files
nexus/sreweekly/articles/489/05-guarding-the-herd-managing-database-servers-at-scale.html
2026-09-12 17:23:01 +08:00

47 lines
43 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html lang="en-US"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"><meta http-equiv="X-UA-Compatible" content="ie=edge"> <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-WMDX8GX');</script> <link href="https://fonts.googleapis.com/css?family=Roboto:300,400,700,900&display=swap" rel="stylesheet"><link rel="shortcut icon" type="image/ico" href="https://engineering.monday.com/favicon.ico"/><link media="all" href="https://engineering.monday.com/wp-content/cache/autoptimize/css/autoptimize_3c9d12ec2fbf09cb43b18bcc101a3fc4.css" rel="stylesheet"><title>Guarding the herd - managing database servers at scale - monday AI engineering</title><meta name='robots' content='max-image-preview:large' /><meta name="robots" content="max-snippet:-1, max-image-preview:large, max-video-preview:-1"/><link rel="canonical" href="https://engineering.monday.com/guarding-the-herd-managing-database-servers-at-scale/" /><meta property="og:locale" content="en_US" /><meta property="og:type" content="article" /><meta property="og:title" content="Guarding the herd - managing database servers at scale - monday AI engineering" /><meta property="og:description" content="Over 400 million terabytes of data are created every day which sums up to 147 zettabytes over the whole year. ..." /><meta property="og:url" content="https://engineering.monday.com/guarding-the-herd-managing-database-servers-at-scale/" /><meta property="og:site_name" content="monday AI engineering" /><meta property="article:tag" content="Database" /><meta property="article:tag" content="Infrastructure" /><meta property="article:section" content="Engineering" /><meta property="article:published_time" content="2025-07-29T08:16:18+00:00" /><meta property="article:modified_time" content="2026-08-05T19:27:33+00:00" /><meta property="og:updated_time" content="2026-08-05T19:27:33+00:00" /><meta property="og:image" content="https://engineering.monday.com/wp-content/uploads/2025/07/image-frame-17.png" /><meta property="og:image:secure_url" content="https://engineering.monday.com/wp-content/uploads/2025/07/image-frame-17.png" /><meta property="og:image:width" content="680" /><meta property="og:image:height" content="340" /><meta name="twitter:card" content="summary_large_image" /><meta name="twitter:description" content="Over 400 million terabytes of data are created every day which sums up to 147 zettabytes over the whole year. ..." /><meta name="twitter:title" content="Guarding the herd - managing database servers at scale - monday AI engineering" /><meta name="twitter:image" content="https://engineering.monday.com/wp-content/uploads/2025/07/image-frame-17.png" /> <script type='application/ld+json' class='yoast-schema-graph yoast-schema-graph--main'>{"@context":"https://schema.org","@graph":[{"@type":"WebSite","@id":"https://engineering.monday.com/#website","url":"https://engineering.monday.com/","name":"monday AI engineering","description":"For those who build different ","potentialAction":{"@type":"SearchAction","target":"https://engineering.monday.com/?s={search_term_string}","query-input":"required name=search_term_string"}},{"@type":"ImageObject","@id":"https://engineering.monday.com/guarding-the-herd-managing-database-servers-at-scale/#primaryimage","url":"https://engineering.monday.com/wp-content/uploads/2025/07/image-frame-17.png","width":680,"height":340},{"@type":"WebPage","@id":"https://engineering.monday.com/guarding-the-herd-managing-database-servers-at-scale/#webpage","url":"https://engineering.monday.com/guarding-the-herd-managing-database-servers-at-scale/","inLanguage":"en-US","name":"Guarding the herd - managing database servers at scale - monday AI engineering","isPartOf":{"@id":"https://engineering.monday.com/#website"},"primaryImageOfPage":{"@id":"https://engineering.monday.com/guarding-the-herd-managing-database-servers-at-scale/#primaryimage"},"datePublished":"2025-07-29T08:16:18+00:00","dateModified":"2026-08-05T19:27:33+00:00","author":{"@id":"https://engineering.monday.com/#/schema/person/23113e79984c574aafcec16a175fae2a"}},{"@type":["Person"],"@id":"https://engineering.monday.com/#/schema/person/23113e79984c574aafcec16a175fae2a","name":"Adar Hilsenrat","image":{"@type":"ImageObject","@id":"https://engineering.monday.com/#authorlogo","url":"https://engineering.monday.com/wp-content/uploads/2026/02/Adar-Hilsenrat-027-Edit-scaled-e1770887076312-150x150.jpg","caption":"Adar Hilsenrat"},"description":"Senior DevOps Engineer @ monday.com","sameAs":["https://www.linkedin.com/in/adarh"]}]}</script> <link rel="alternate" title="oEmbed (JSON)" type="application/json+oembed" href="https://engineering.monday.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.monday.com%2Fguarding-the-herd-managing-database-servers-at-scale%2F" /><link rel="alternate" title="oEmbed (XML)" type="text/xml+oembed" href="https://engineering.monday.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fengineering.monday.com%2Fguarding-the-herd-managing-database-servers-at-scale%2F&#038;format=xml" /> <script type="text/javascript" src="https://engineering.monday.com/wp-includes/js/jquery/jquery.min.js?ver=3.7.1" id="jquery-core-js"></script> <script type="text/javascript" src="https://engineering.monday.com/wp-includes/js/jquery/jquery-migrate.min.js?ver=3.4.1" id="jquery-migrate-js"></script> <script type="text/javascript" src="https://engineering.monday.com/wp-content/plugins/wp-user-avatar/assets/flatpickr/flatpickr.min.js?ver=4.16.3" id="ppress-flatpickr-js"></script> <script type="text/javascript" src="https://engineering.monday.com/wp-content/plugins/wp-user-avatar/assets/select2/select2.min.js?ver=4.16.3" id="ppress-select2-js"></script> <link rel="https://api.w.org/" href="https://engineering.monday.com/wp-json/" /><link rel="alternate" title="JSON" type="application/json" href="https://engineering.monday.com/wp-json/wp/v2/posts/10912" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://engineering.monday.com/xmlrpc.php?rsd" /><meta name="generator" content="WordPress 6.9.7" /><link rel='shortlink' href='https://engineering.monday.com/?p=10912' /> <script>window.dataLayer = window.dataLayer || [];
window.dataLayer.push({
'event': 'pageMetaData',
'page': {
'type' : 'post',
'template' : 'blog post',
'title' : 'Guarding the herd - managing database servers at scale',
'topic' : '',
'category' : 'Engineering',
'language' : 'en-US'
}
});</script> <link rel="icon" href="https://engineering.monday.com/wp-content/uploads/2026/06/cropped-logo-5-32x32.png" sizes="32x32" /><link rel="icon" href="https://engineering.monday.com/wp-content/uploads/2026/06/cropped-logo-5-192x192.png" sizes="192x192" /><link rel="apple-touch-icon" href="https://engineering.monday.com/wp-content/uploads/2026/06/cropped-logo-5-180x180.png" /><meta name="msapplication-TileImage" content="https://engineering.monday.com/wp-content/uploads/2026/06/cropped-logo-5-270x270.png" /><link rel="stylesheet" href="https://use.typekit.net/byq3jhd.css" as="font"/></head><body class="wp-singular post-template-default single single-post postid-10912 single-format-standard wp-theme-monday rnd-blog"> <noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-WMDX8GX" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript> <script>window.BigBrainQ = window.BigBrainQ || [];
window.BigBrain = window.BigBrain || function(){
args = Array.prototype.slice.call(arguments)
if(args[0] == 'track'){
event_props = {}
if(args.length == 3)
event_props = args[2];
event_props['timestamp'] = parseInt(new Date().getTime()/1000);
if(args.length == 2) args.push(event_props);
else args[2] = event_props;
}
window.BigBrainQ.push(args);
};
(function(d,u,callback){
if (window.BigBrainInit) return;
window.BigBrainInit = true;
var s = d.createElement('script');
s.type = 'text/javascript'
s.src = u;
s.async = true;
s.onreadystatechange = s.onload = function(){
var st = s.readyState;
if (!callback.done && (!st || /loaded|complete/.test(st))) {
callback.done = true;
callback();
}
};
(d.body || head).appendChild(s);
})(document, "https://d18vk66ftlazd2.cloudfront.net/bigbrain-1.0.min.js", function() {
window.BigBrain = new BigBrainTracker({bigbrain_url: 'https://data.bigbrain.me', send_immediately: true});
});
// This should report page_view in each page in the blog
var pathName = 'blog' + window.location.pathname;
BigBrain('track', 'page_view', {kind: pathName, source: 'blog'});</script> <div id="pin"></div><header id="main-header" class="shadow-sm border-bottom bg-white"><div class="container clearfix"><div class="float-left clearfix d-flex align-items-center h-100"> <a class="logo-image" alt="monday AI engineering" aria-label="logo" href="https://engineering.monday.com/"> <img src="https://engineering.monday.com/wp-content/uploads/2025/08/Sub-logo_monday-engineering_Black.png" alt="monday AI engineering" /> </a></div><div id="menu-wrapper" class="float-right clearfix align-items-center h-100 d-none d-md-flex"><div class="float-left"><form method="get" id="searchform" class="clearfix" action="https://engineering.monday.com/" role="search"> <input type="text" class="search-input float-left" name="s" value="" id="s" placeholder="Search..." /> <button type="submit" aria-label="search" class="submit float-left" name="submit" id="searchsubmit"></button></form></div><nav class="float-right"><div id="main-nav" class="menu-main-menu-container"><ul id="menu-main-menu" class="main-nav clearfix"><li id="menu-item-9454" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-9454"><a href="https://monday.com">monday.com</a></li><li id="menu-item-9805" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-9805"><a target="_blank" href="https://www.startupforstartup.com/?sfid=1860&#038;_sft_post_tag=rd-product-design">Podcasts</a></li><li id="menu-item-9455" class="custon-blue-bg menu-item menu-item-type-custom menu-item-object-custom menu-item-9455"><a href="https://monday.com/jobs">Join Us</a></li></ul></div></nav></div><div id="search-wrapper" class="d-none d-md-flex"></div><div class="mobile-toggles-wrapper float-right d-block d-md-none clearfix"><div id="toggle-search-menu" class="float-left"></div><div id="toggle-mobile-menu" class="float-right"></div></div></div></header><main data-post-id="10912"><section id="hero-wrapper"><div class="container"><div class="row"><div class="col-md-12"> <img class="img-fluid" src="https://engineering.monday.com/wp-content/uploads/2025/07/image-frame-17.png" alt="Guarding the herd &#8211; managing database servers at scale"></div></div></div></section><section id="post-wrapper"><div class="container"><div class="row"><div id="article-section" class="col-md-12"><div class="author-social-media desktop-only"><div class="social-links clearfix d-flex align-items-center"><div style="display:flex;margin-bottom:10px;position:relative;" class="likes clearfix"> <a style="cursor:pointer;" aria-label="like" class="like-icon d-block" href="#" data-post-likes="35" data-post-id="10912"> <img class="img-fluid" src="https://res.cloudinary.com/monday-blogs/fl_lossy,f_auto/wp-blog/theme/icons/like.png" alt=""> </a> <span style="position:absolute;right:-20px;" class="likes-count likes-value d-block" data-post-id="10912">35</span></div> <a href="#" class="linkedin" aria-label="share on linkedin" onclick="window.open(this.href,'targetWindow','toolbar=no,location=no,status=no,menubar=no,scrollbars=yes,resizable=yes, width=800,height=450'); return false;"></a> <a href="#" aria-label="share on facebook" class="facebook" onclick="window.open(this.href,'targetWindow','toolbar=no,location=no,status=no,menubar=no,scrollbars=yes,resizable=yes, width=800,height=450'); return false;"></a> <a href="#" aria-label="share on twitter" class="twitter" onclick="window.open(this.href,'targetWindow','toolbar=no,location=no,status=no,menubar=no,scrollbars=yes,resizable=yes, width=800,height=450'); return false;"></a></div></div><div class="tags-wrapper clearfix"><div class="card-tag float-left text-">Database</div><div class="card-tag float-left purple2 text-purple2">Infrastructure</div></div><h1 id="post-title">Guarding the herd &#8211; managing database servers at scale</h1><h4 id="post-subtitle"></h4><div class="author-wrapper clearfix"><div class="author-wrapper-inner single-author clearfix float-md-left"><div class="authors-list"><div class="author-row d-flex align-items-center"><div class="author-image"> <img data-del="avatar" alt="Adar Hilsenrat" src='https://engineering.monday.com/wp-content/uploads/2026/02/Adar-Hilsenrat-027-Edit-scaled-e1770887076312-150x150.jpg' class='avatar pp-user-avatar avatar-120 photo ' height='120' width='120'/></div><div class="author-info float-left"><div class="author-name">Adar Hilsenrat</div><div class="post-info"> Jul 29, 2025<span class="dot-separator"></span>15 min read <span class="star"></span></div></div></div></div></div><div class="author-social-media float-md-right mobile-only"><div class="social-links clearfix d-flex align-items-center"> <a href="#" class="linkedin" onclick="window.open(this.href,'targetWindow','toolbar=no,location=no,status=no,menubar=no,scrollbars=yes,resizable=yes, width=800,height=450'); return false;"></a> <a href="#" class="facebook" onclick="window.open(this.href,'targetWindow','toolbar=no,location=no,status=no,menubar=no,scrollbars=yes,resizable=yes, width=800,height=450'); return false;"></a> <a href="#" class="twitter" onclick="window.open(this.href,'targetWindow','toolbar=no,location=no,status=no,menubar=no,scrollbars=yes,resizable=yes, width=800,height=450'); return false;"></a></div></div></div><div id="post-content" class="color-purple2"><p>Over 400 million terabytes of data are created every day which sums up to 147 zettabytes over the whole year. If you don’t know how much zettabyte is &#8211; 1 zettabyte is 1 billion terabytes. The amount of data the world creates and consumes is growing at a crazy pace with estimations that 90% of the world’s data was generated in the last 2 years alone. Information is one of the most valuable assets that a company can possess and databases are a critical part of every company’s infrastructure. <a href="https://monday.com/" target="_blank" rel="noreferrer noopener">monday.com</a> is no exception &#8211; we are a data-centric company.</p><h2 class="wp-block-heading">The evolution of data layer</h2><p>Let me tell you the story of monday.com. Our product launched in 2014 and it was a monolith application with a single database. We didn&#8217;t have a lot of data as we had only a few customers. The company grew very fast, and 6 years later, in 2020, we surpassed 100000 customers. As you can imagine, the volume of data also grew significantly, so we added replicas &#8211; read-only clones of the primary database &#8211; and separated reads and writes. With this addition we reached a dozen databases, a number that still can be managed even by a single DBA. Then, like many other tech companies, we decided to split into microservices as developing one monolith application started to become very hard with a growing number of developers, new features, and the complexity of the system. We extracted key parts of the system into microservices with their own, separate databases. This way, instead of a few databases, we started to manage tens of databases and this was no longer feasible for one person. At this point, the DBA team was established. There were a few people with database expertise and still everything was working fine.</p><p>Now let’s talk about today &#8211; today we have hundreds of microservices and still growing. As a consequence, we have no longer tens of databases. Now we have hundreds of them. Our team is already very big but with the exponential growth of database servers, it’s no longer feasible to just hire more people and still manage databases in an old way.</p><figure class="wp-block-image size-large"><img fetchpriority="high" decoding="async" width="1024" height="260" class="wp-image-10922" src="https://engineering.monday.com/wp-content/uploads/2025/07/Screenshot-2025-07-29-at-08.31.07-1024x260.png" alt="" srcset="https://engineering.monday.com/wp-content/uploads/2025/07/Screenshot-2025-07-29-at-08.31.07-1024x260.png 1024w, https://engineering.monday.com/wp-content/uploads/2025/07/Screenshot-2025-07-29-at-08.31.07-300x76.png 300w, https://engineering.monday.com/wp-content/uploads/2025/07/Screenshot-2025-07-29-at-08.31.07-768x195.png 768w, https://engineering.monday.com/wp-content/uploads/2025/07/Screenshot-2025-07-29-at-08.31.07.png 1049w" sizes="(max-width: 1024px) 100vw, 1024px" /></figure><p>At some point, we changed the team name from the DBA (Database Administrators) team to the DBRE (Database Reliability Engineers) team. It actually reflects the shift in our approach to managing databases. We can try to summarize it in one sentence &#8211; treat your databases as cattle, not pets. DBRE is a different mindset. You simply cannot manage every single database server because there are too many of them. You need to manage a group of database servers at once. In case of stateless workloads, it’s much easier. The Kubernetes node has some malfunction &#8211; let’s just replace the node. But in this case we have databases that are stateful. We cannot just replace the server in case there is a failure because we have our data there. It makes it tricky to treat databases as cattle and it’s not an easy task.</p><h2 class="wp-block-heading">A long way from DBA to DBRE</h2><p>We agreed that a few principles must guide us in becoming a real DBRE team.</p><h3 class="wp-block-heading">Standards</h3><p>The herd consists of the animals of one species as they have the same needs. Our database herd has to be of the same kind. We cannot afford to configure every single database separately for different needs. It doesn’t mean that we will have databases of only one kind and we will have only 1 group of databases. We can have multiple groups. In <a href="https://monday.com/" target="_blank" rel="noreferrer noopener">monday.com</a> we group the resources based on tiers.</p><figure class="wp-block-image size-large"><img decoding="async" width="1024" height="370" class="wp-image-10916" src="https://engineering.monday.com/wp-content/uploads/2025/07/image-from-clipboard-1-1024x370.png" alt="" srcset="https://engineering.monday.com/wp-content/uploads/2025/07/image-from-clipboard-1-1024x370.png 1024w, https://engineering.monday.com/wp-content/uploads/2025/07/image-from-clipboard-1-300x108.png 300w, https://engineering.monday.com/wp-content/uploads/2025/07/image-from-clipboard-1-768x277.png 768w, https://engineering.monday.com/wp-content/uploads/2025/07/image-from-clipboard-1-1536x555.png 1536w, https://engineering.monday.com/wp-content/uploads/2025/07/image-from-clipboard-1-2048x740.png 2048w" sizes="(max-width: 1024px) 100vw, 1024px" /></figure><p>Tier 1 are the most critical services that are crucial for the platform to work. Tier 3 are some peripheral services that won’t affect most of our customers if they are down. We have different standards for tier 1 and tier 3 databases. We can simplify some of the procedures for tier 3 databases, as even if the database is down for a few seconds, no one will probably notice it. For tier 1 databases, it’s unacceptable to have even the shortest downtime so our standards have to reflect that. Tier 3 databases have much lower traffic while tier 1 databases serve incomparable amounts of queries in a second. We will use different instance families for them, different storage types, and different numbers of replicas. For tier 3 databases, we can allow automatic minor version upgrades because even if something goes wrong, there is a lower chance that anyone will notice it. For tier 1 we do not allow it &#8211; we need to carefully test the upgrade and only then upgrade the database. Having those standards in place allows us to focus on the needs of specific database groups instead of reviewing every single database separately. If some standard needs to be changed for one tier 1 database, it probably also needs to be changed in the rest of the tier 1 services. The standards can be set for different aspects of database management &#8211; configuration, procedures, playbooks, monitoring and alerting.</p><h3 class="wp-block-heading">Automations</h3><p>If we have our standard configuration and procedures, we cannot apply them to each database manually. Every single action that we perform on a database should be automated. Even if we have a script to perform such a procedure automatically, we simply cannot connect to each database separately and run this script because we have hundreds of servers. We need to think bigger. What we need is a comprehensive toolkit that knows the context, the standards, and when and where it can execute certain procedures. DBAs need to learn software development. And what is more &#8211; we need to have standards also for this. We cannot write bash scripts that we keep on our laptops and execute them there. We need to stick to the standards of the software development process &#8211; write tests, have a proper CI process, reviews, approvals, CD, and gradual releases. We need everything that is considered as production readiness for software. There are no shortcuts here &#8211; our software has to be bulletproof. It will be running critical tasks on database servers. Can you think of a more critical incident than a data loss? We need to trust our code in 100%.</p><h3 class="wp-block-heading">Self-service</h3><p>If we already have our automations for all procedures on databases, why should it be us to execute them? It’s a development team that owns their microservice &#8211; why don’t we let them own their database? Our job here is to use our knowledge and skillset to actually decide what we need to automate, how to do it in a safe way and then expose this procedure to database owners. When we create a new microservice, the developer should mark a checkbox if they need a database or not, and in case they need it, it should be created automatically. When a developer wants to create a backup on demand, they should be able to do it by just clicking a button &#8211; without involving the DBRE team. When they need a new replica, increase the storage, increase the machine size, etc, they should be able to do it on their own. Of course there are some actions that maybe shouldn’t be so common and should require at least DBA approval. Maybe some critical actions shouldn’t be exposed to development teams at all. If it’s a tier 1 database, maybe we should have approvals for every possible action. Everything depends on the organization&#8217;s requirements and specific context. The idea here is that we should offload the most common and safe operations and shift them left.</p><h3 class="wp-block-heading">Observability</h3><p>Again, we cannot do it manually. We cannot connect to a specific database and run “show processlist”. We need proper monitoring for every aspect of a database. The critical part here is alerting. And again, the alerts shouldn’t necessarily come to us. For example, storage size alerts can come directly to the development team, and in case it’s low, they can increase it on their own. Some of the alerts shouldn’t notify anyone but they should just trigger a proper remediation. Of course, in case of some critical events like “server down”, it still should come directly to the DBRE team.</p><h2 class="wp-block-heading">So&#8230; what did we actually do?</h2><p>Let me introduce you to project <strong>Shepherd</strong>. If the databases are our herd, the Shepherd is the guard which takes care of them.</p><p>Our first and foremost motivation to start the project was <strong>scale</strong>. We had plenty of repetitive, manual tasks that we had to perform on all of the databases and the number of them is growing constantly. Another problem that we wanted to solve was <strong>delivery consistency</strong>. We didn’t have any standards defined and the procedures were executed by different team members on different database servers. This fact led to human errors, misconfiguration, and discrepancies. Next problem &#8211; <strong>no single source of truth</strong>. One may say &#8211; just use Terraform or Crossplane or some other IaC tool. Well, it&#8217;s not that easy. We used Terraform for our RDS instances. The problem is that the database configuration is not just RDS &#8211; there are other things such as database user configuration, permission configuration, application secrets, tooling running on a database, or connection pooler configuration, like PgBouncer or ProxySQL. All of this makes a source of truth for a database more complex than just keeping it in the Terraform module. Let’s take a specific scenario as an example &#8211; resizing a database server. Let’s say we want to increase machine size from large to extra large. It’s not just about changing the Terraform parameter called “instance_family”. The first step would be to drain the connections to the database. Once there are no connections, we will change the machine size. After it’s back up, we don’t want to just throw 100% of the traffic into it. We want to release it gradually &#8211; use our connection pooler to route 1% of queries to it. If everything is fine, increase the traffic to 5%. And so on until 100%. It’s more than just changing the parameter in Terraform &#8211; it’s the whole procedure and our single source of truth must reflect that.</p><p>All of this made us think about the requirements for the new system. We need it to be able to <strong>perform complex procedures </strong><strong>t</strong><strong>h</strong><strong>at</strong><strong> we can </strong><strong>be </strong><strong>decompose</strong><strong>d</strong><strong> into small tasks</strong>. We want those small tasks to be <strong>reusable</strong>. For example, a task to restart a server will be probably reused in many complex workflows. We want a way to <strong>test</strong> the complex workflows and find issues before they happen in production. As I said before, the key is the trust. We are talking about databases here &#8211; one mistake and the consequences may be terrible. We need a way to <strong>define the infrastructure in an imperative way</strong>. We need to have control over what exactly is happening in which order. For example, we cannot add a database to the connection pooler configuration until it’s fully ready with all other configurations. We want the system to be able to <strong>handle long-lasting processes</strong> that take days or even weeks. We have some huge databases in our infrastructure where executing a schema migration might take really long. We need a way to put a <strong>human in a loop</strong>. For some critical procedures, we want to have a way to pause the workflow, allow a DBA to check the status, approve it or not, and only then continue with the workflow. Another requirement is to easily <strong>integrate it with other systems</strong>. We have our internal developers portal called sphera and we want databases to be part of it. We want to give developers a friendly UI to monitor their databases and perform actions on them. And last but not least &#8211; <strong>a </strong><strong>single source of truth</strong>. We need to have one place where we keep 100% of database configuration.</p><p>We decided that what we need is <strong>a workflow engine</strong>. We want to have a platform where we will be able to run our complex workflows without taking care of scheduling, retries or human in the loop. There are a lot of different workflow engines in the market but we chose <strong>Temporal</strong>. It’s an open-source workflow engine that is getting more and more popular. We picked Temporal because it offers much more than our requirements. It provides great visibility, easy local development, support for many languages, and easy testing.</p><p>Picking Temporal already fulfilled most of our requirements, but there is also one more &#8211; a single source of truth. The workflow engine doesn’t solve it &#8211; we need to have a way to pass proper input parameters to the workflows that we will have. We are a DBRE team, and we know databases well, so we thought &#8211; why not create a database to keep the configuration of all our infrastructure components? We decided that we would have Shepherd’s database where we would keep the desired configuration, actual configuration, and change log. Our idea was to modify the desired configuration, watch for differences between actual and desired configuration and if there is a diff &#8211; run a workflow called “sync” that will apply the desired configuration.</p><figure class="wp-block-image size-large"><img loading="lazy" decoding="async" width="1024" height="351" class="wp-image-10917" src="https://engineering.monday.com/wp-content/uploads/2025/07/image-from-clipboard-2-1024x351.png" alt="" srcset="https://engineering.monday.com/wp-content/uploads/2025/07/image-from-clipboard-2-1024x351.png 1024w, https://engineering.monday.com/wp-content/uploads/2025/07/image-from-clipboard-2-300x103.png 300w, https://engineering.monday.com/wp-content/uploads/2025/07/image-from-clipboard-2-768x263.png 768w, https://engineering.monday.com/wp-content/uploads/2025/07/image-from-clipboard-2-1536x527.png 1536w, https://engineering.monday.com/wp-content/uploads/2025/07/image-from-clipboard-2-2048x702.png 2048w" sizes="auto, (max-width: 1024px) 100vw, 1024px" /></figure><p>You may already know this concept from ArgoCD &#8211; we have the desired and live manifests and ArgoCD takes care of applying the desired configuration every time it changes. We want to use the same concept for our databases. It will also give us great visibility over drifts. We will see which databases are out of sync and if they need some manual intervention to remediate. Having a change log will allow us to quickly roll back to the previous configuration if something is not working well.</p><p>Now, the part of integrating with other systems. I’ve already mentioned sphera &#8211; our internal developers portal. We want to use it as a UI for our single source of truth. Everyone will be able to go to sphera, see the desired and actual configuration, see if we don’t have a drift, change the desired configuration, and monitor databases. It will be our self-service for everything related to database management.</p><h2 class="wp-block-heading">It&#8217;s that easy&#8230; Well, not really</h2><p>In a perfect world, that would be it &#8211; we have standards, we have automation, we give self-service, we have monitoring and alerting. But we all know that the world is not perfect. We won’t be able to automate every single action in the near future. Preparing such workflows can take a significant amount of time for development and proper, thorough testing before it’s released to work with production databases. And even when it’s released &#8211; we all know that there are chances that it will fail the first time we need to use it because of discrepancies between test/staging/production environments. The incidents will still happen and we won’t predict everything &#8211; we won’t be ready for every possible failure in our databases. That’s why we still need a backdoor. We still need a way to go to a specific database and do manual things. We need a way to change parameters in AWS. We need a way to manually restart a server. What I’m trying to say is that Shepherd is not only a workflow engine. It will still provide us with tools to do things in an old-fashioned way. We will have a code that restarts a database. It will be used in workflows, but we will still use the same code to implement, e.g., a CLI for our team to be able to restart a specific database with 1 command. Another example &#8211; we will open connections to databases in workflows to execute some SQLs there. We will use the same code to implement a CLI command which opens a connection to a database in seconds in case we need to do manual tweaks. So what is Shepherd after all? It&#8217;s a monorepo with all database-management-related code that will be used to prepare tools &#8211; from fully automated workflows through scripts that can be triggered manually to simple operations like connecting to the database to run SQL.</p><h2 class="wp-block-heading">Conclusion</h2><p>We have been working on Shepherd for quite some time already. We have our Temporal, and we have workflows running in production. The work is still in progress, and we are currently developing more workflows that will cover the most critical flows of database management.</p><p>Shepherd is going to be the heart of our team &#8211; it will allow us to keep everything moving forward fast, at scale, in a reliable way. The brain is the knowledge, experience, and expertise of all our team members. In the end, it&#8217;s on us to design and implement the solutions and to know whether to click or not to click the button to perform an action.</p><p>&nbsp;</p></div><div class="blog-author-about style--1"><div class="blog-author-image"> <img data-del="avatar" alt="Adar Hilsenrat" src='https://engineering.monday.com/wp-content/uploads/2026/02/Adar-Hilsenrat-027-Edit-scaled-e1770887076312-150x150.jpg' class='avatar pp-user-avatar avatar-120 photo ' height='120' width='120'/></div><div class="blog-author-description"> Senior DevOps Engineer @ monday.com</div></div><div class="spacer">&nbsp;</div><div class="spacer">&nbsp;</div></div></div></div></section><section id="section-related-posts"><div class="container"><div class="row"><div class="col-md-12"><h3 id="related-posts-title">Read More</h3><div class="separator"></div></div></div><div class="related-post-box-row row"><div class="col-sm-6 col-lg-4"> <a class="related-post-box d-block" href="https://engineering.monday.com/building-a-robust-harness-for-agent-in-production-feed-agent-case-study/"><h5>Related Post</h5><div class="related-post-box-img"> <img class="img-fluid" src="https://engineering.monday.com/wp-content/uploads/2026/09/1e86e0f1-e3c7-4050-aa87-af9064c8926a_780x520_acf_cropped_315x215_acf_cropped.jpeg" alt="" width="315" height="215" /></div><h4 class="related-post-box-title">Building a Robust Harness for agent in production: Feed agent case-study</h4><div class="author-wrapper clearfix single-author"><div class="authors-list"><div class="author-row d-flex align-items-center"><div class="author-image"> <img data-del="avatar" alt="Kamie Shami-Schnitzer" src='https://engineering.monday.com/wp-content/uploads/2026/09/ChatGPT-Image-Dec-24-2025-06_54_42-PM-1-e1788861489681-150x150.png' class='avatar pp-user-avatar avatar-120 photo ' height='120' width='120'/></div><div class="author-info float-left"><div class="author-name">Kamie Shami-Schnitzer</div><div class="post-info">Sep 9<span class="dot-separator"></span>11 min read <span class="star"></span></div></div></div></div><div class="float-right"><div class="likes clearfix"> <span class="like-icon d-block float-left" href="#" data-post-likes="2" data-post-id="12592"> <img class="img-fluid" src="https://res.cloudinary.com/monday-blogs/fl_lossy,f_auto/wp-blog/theme/icons/like.png" alt=""> </span> <span class="likes-count likes-value d-block float-left" data-post-id="12592">2</span></div></div></div> </a></div><div class="col-sm-6 col-lg-4"> <a class="related-post-box d-block" href="https://engineering.monday.com/i-built-a-monitor-nearly-200-engineers-turned-it-into-a-colleague/"><h5>Related Post</h5><div class="related-post-box-img"> <img class="img-fluid" src="https://engineering.monday.com/wp-content/uploads/2026/08/bd91868f-7875-4eb3-90c7-ae8a8b119f89_780x520_acf_cropped_315x215_acf_cropped.png" alt="" width="315" height="215" /></div><h4 class="related-post-box-title">I built a monitor. Nearly 200 engineers turned it into a colleague.</h4><div class="author-wrapper clearfix single-author"><div class="authors-list"><div class="author-row d-flex align-items-center"><div class="author-image"> <img data-del="avatar" alt="Uzeyir Abdullayev" src='https://engineering.monday.com/wp-content/uploads/2026/08/T024J3LAA-U073YHKE90E-6c12201c379c-512-150x150.jpeg' class='avatar pp-user-avatar avatar-120 photo ' height='120' width='120'/></div><div class="author-info float-left"><div class="author-name">Uzeyir Abdullayev</div><div class="post-info">Sep 1<span class="dot-separator"></span>15 min read <span class="star"></span></div></div></div></div><div class="float-right"><div class="likes clearfix"> <span class="like-icon d-block float-left" href="#" data-post-likes="6" data-post-id="12548"> <img class="img-fluid" src="https://res.cloudinary.com/monday-blogs/fl_lossy,f_auto/wp-blog/theme/icons/like.png" alt=""> </span> <span class="likes-count likes-value d-block float-left" data-post-id="12548">6</span></div></div></div> </a></div><div class="col-sm-6 col-lg-4"> <a class="related-post-box d-block" href="https://engineering.monday.com/from-weeks-to-days-unifying-the-mobile-and-desktop-monday-com-marketplace-with-agentic-loops/"><h5>Related Post</h5><div class="related-post-box-img"> <img class="img-fluid" src="https://engineering.monday.com/wp-content/uploads/2026/08/agentic_pipeline_mobile_marketplace_cover_v3-scaled_315x215_acf_cropped.png" alt="" width="315" height="215" /></div><h4 class="related-post-box-title">From weeks to days: Unifying the mobile and desktop monday.com marketplace with Agentic Loops</h4><div class="author-wrapper clearfix single-author"><div class="authors-list"><div class="author-row d-flex align-items-center"><div class="author-image"> <img data-del="avatar" alt="Łukasz Dąbrowski" src='https://engineering.monday.com/wp-content/uploads/2026/08/SCR-20260824-jnvf-150x150.png' class='avatar pp-user-avatar avatar-120 photo ' height='120' width='120'/></div><div class="author-info float-left"><div class="author-name">Łukasz Dąbrowski</div><div class="post-info">Aug 28<span class="dot-separator"></span>12 min read <span class="star"></span></div></div></div></div><div class="float-right"><div class="likes clearfix"> <span class="like-icon d-block float-left" href="#" data-post-likes="4" data-post-id="12524"> <img class="img-fluid" src="https://res.cloudinary.com/monday-blogs/fl_lossy,f_auto/wp-blog/theme/icons/like.png" alt=""> </span> <span class="likes-count likes-value d-block float-left" data-post-id="12524">4</span></div></div></div> </a></div></div></div></section></main><footer id="main-footer"><div class="container"><div class="row"><div class="col-12 col-md"><nav id="footer-nav"><div id="footer-main-nav" class="menu-footer-menu-container"><ul id="menu-footer-menu" class="footer-main-nav clearfix d-flex align-items-center justify-content-center"><li id="menu-item-8" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-8"><a href="https://monday.com/terms/tos">Terms of use</a></li><li id="menu-item-9" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-9"><a href="https://monday.com/terms/privacy">Privacy Policy</a></li><li id="menu-item-10" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-10"><a href="https://monday.com/product/">Product</a></li><li id="menu-item-11" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-11"><a href="https://monday.com/help">Contact us</a></li><li id="menu-item-12" class="menu-item menu-item-type-custom menu-item-object-custom menu-item-12"><a href="https://monday.com/pricing/">Pricing</a></li></ul></div></nav><div class=""> <a id="footer-logo" class="d-block mr-auto ml-auto" href="https://monday.com/"> <img src="https://res.cloudinary.com/monday-blogs/fl_lossy,f_auto/wp-blog/theme/monday-logo.webp" alt="logo" width="120"> </a><p class="copyright"> All Rights Reserved &copy; monday.com <a href="tel:+12017784567">+1 (201) 778-4567</a></p><p class="address"> Derech Menachem Begin 52, Tel Aviv-Yafo</p></div><div class="footer-social-links d-flex align-items-center justify-content-center"> <a target="_blank" aria-label="facebook" class="Facebook_black" href="https://www.facebook.com/mondaydotcom"></a> <a target="_blank" aria-label="instagram" class="Instagram_black" href="https://www.instagram.com/monday.com_engineering/"></a> <a target="_blank" aria-label="linkedin" class="LinkedIN_black" href="https://www.linkedin.com/company/monday-com-engineering"></a> <a target="_blank" aria-label="twitter" class="Twitter_black" href="https://twitter.com/mondaydotcom"></a> <a target="_blank" aria-label="youtube" class="Youtube_black" href="https://www.youtube.com/c/mondayEngineering"></a></div></div></div></div></footer> <script src="https://engineering.monday.com/wp-content/themes/monday/lib/jquery-3.4.1.min.js" crossorigin="anonymous"></script> <script src="https://engineering.monday.com/wp-content/themes/monday/lib/jquery-migrate-1.4.1.min.js" crossorigin="anonymous"></script> <script src="https://engineering.monday.com/wp-content/themes/monday/lib/imagesloaded.pkgd.min.js"></script> <script src="https://engineering.monday.com/wp-content/themes/monday/lib/masonry.pkgd.js"></script> <script src="https://engineering.monday.com/wp-content/themes/monday/lib/bootstrap.min.js"></script> <script src="https://engineering.monday.com/wp-content/themes/monday/js/main.js"></script> <script type="speculationrules">{"prefetch":[{"source":"document","where":{"and":[{"href_matches":"/*"},{"not":{"href_matches":["/wp-*.php","/wp-admin/*","/wp-content/uploads/*","/wp-content/*","/wp-content/plugins/*","/wp-content/themes/monday/*","/*\\?(.+)"]}},{"not":{"selector_matches":"a[rel~=\"nofollow\"]"}},{"not":{"selector_matches":".no-prefetch, .no-prefetch a"}}]},"eagerness":"conservative"}]}</script> <script type="text/javascript" id="monday-data-layer-js-extra">var mondayDataLayer = {"searchResults":"-1","filterClicked":""};
//# sourceURL=monday-data-layer-js-extra</script> <script type="text/javascript" src="https://engineering.monday.com/wp-content/plugins/monday-data-layer-events//data-layer.js?ver=1789195622" id="monday-data-layer-js"></script> <script type="text/javascript" id="ppress-frontend-script-js-extra">var pp_ajax_form = {"ajaxurl":"https://engineering.monday.com/wp-admin/admin-ajax.php","confirm_delete":"Are you sure?","deleting_text":"Deleting...","deleting_error":"An error occurred. Please try again.","nonce":"7915e80535","disable_ajax_form":"false","is_checkout":"0","is_checkout_tax_enabled":"0","is_checkout_autoscroll_enabled":"true"};
//# sourceURL=ppress-frontend-script-js-extra</script> <script type="text/javascript" src="https://engineering.monday.com/wp-content/plugins/wp-user-avatar/assets/js/frontend.min.js?ver=4.16.3" id="ppress-frontend-script-js"></script> <script type="text/javascript" src="https://engineering.monday.com/wp-content/plugins/page-links-to/dist/new-tab.js?ver=3.3.7" id="page-links-to-js"></script> </body></html>