Files
nexus/sreweekly/articles/477/05-upgrading-eck-operator-a-side-by-side-kubernetes-operator-upgrade-appr.html
2026-09-12 17:23:01 +08:00

235 lines
87 KiB
HTML
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html lang="en-US"> <head><title>Upgrading ECK Operator: A Side-by-Side Kubernetes Operator Upgrade Approach | Mercari Engineering</title><!-- Google Tag Manager --><script>
(function (w, d, s, l, i) {
w[l] = w[l] || [];
w[l].push({ 'gtm.start': new Date().getTime(), event: 'gtm.js' });
var f = d.getElementsByTagName(s)[0],
j = d.createElement(s),
dl = l != 'dataLayer' ? '&l=' + l : '';
j.async = true;
j.src = 'https://www.googletagmanager.com/gtm.js?id=' + i + dl;
f.parentNode.insertBefore(j, f);
})(window, document, 'script', 'dataLayer', 'GTM-WFXJ8Q3');
</script><!-- End Google Tag Manager --><meta charset="UTF-8"><link rel="canonical" href="https://engineering.mercari.com/en/blog/entry/20250428-upgrading-eck-operator-a-side-by-side-kubernetes-operator-upgrade-approach/"><meta name="description" content="Greetings, I&#38;#8217;m Abhishek Munagekar from the Search Infrastructure Team at Mercari. Our team manages several Elastic"><meta name="robots" content="index, follow"><meta property="og:title" content="Upgrading ECK Operator: A Side-by-Side Kubernetes Operator Upgrade Approach | Mercari Engineering"><meta property="og:type" content="website"><meta property="og:image" content="https://storage.googleapis.com/prd-engineering-asset/2025/12/521c5551-image_fx-1024x559-1.png"><meta property="og:url" content="https://engineering.mercari.com/en/blog/entry/20250428-upgrading-eck-operator-a-side-by-side-kubernetes-operator-upgrade-approach/"><meta property="og:description" content="Greetings, I&#38;#8217;m Abhishek Munagekar from the Search Infrastructure Team at Mercari. Our team manages several Elastic"><meta property="og:locale" content="en_US"><meta property="og:locale:alternate" content="ja_JP"><link rel="icon" type="image/x-icon" href="/favicon.ico"><link rel="shortcut icon" type="image/vnd.microsoft.icon" href="/favicons/favicon.ico"><link rel="icon" type="image/vnd.microsoft.icon" href="/favicons/favicon.ico"><link rel="alternate" type="application/rss+xml" href="/blog/feed.xml" title="メルカリエンジニアリングブログ"><link rel="alternate" type="application/rss+xml" href="/en/blog/feed.xml" title="Mercari Engineering blog"><link rel="alternate" href="" hreflang="ja"><link rel="alternate" href="" hreflang="ja-JP"><link rel="alternate" href="https://engineering.mercari.com/en/blog/entry/20250428-upgrading-eck-operator-a-side-by-side-kubernetes-operator-upgrade-approach/" hreflang="en"><link rel="alternate" href="https://engineering.mercari.com/en/blog/entry/20250428-upgrading-eck-operator-a-side-by-side-kubernetes-operator-upgrade-approach/" hreflang="en-US"><link rel="sitemap" href="https://engineering.mercari.com//sitemap-index.xml"><meta name="viewport" content="width=device-width, initial-scale=1"><meta name="twitter:card" property="twitter:card" content="summary_large_image"><meta name="twitter:site" content="MercariDev"><meta name="twitter:site" content="mercaridevjp"><meta name="description" content="Greetings, I&#38;#8217;m Abhishek Munagekar from the Search Infrastructure Team at Mercari. Our team manages several Elastic"><meta property="og:description" content="Greetings, I&#38;#8217;m Abhishek Munagekar from the Search Infrastructure Team at Mercari. Our team manages several Elastic"><meta property="og:url" content="https://engineering.mercari.com/en/blog/entry/20250428-upgrading-eck-operator-a-side-by-side-kubernetes-operator-upgrade-approach/"><meta property="og:image" content="https://storage.googleapis.com/prd-engineering-asset/2025/12/521c5551-image_fx-1024x559-1.png"><meta property="og:title" content="Upgrading ECK Operator: A Side-by-Side Kubernetes Operator Upgrade Approach"><meta data-hid="og:type" property="og:type" content="website"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta data-hid="thumbnail" name="thumbnail" content="https://engineering.mercari.com//img/ogp/ogp_a.jpg"><link data-n-head="ssr" rel="apple-touch-icon" sizes="57x57" href="/favicons/apple-touch-icon-57x57.png"><link data-n-head="ssr" rel="apple-touch-icon" sizes="60x60" href="/favicons/apple-touch-icon-60x60.png"><link data-n-head="ssr" rel="apple-touch-icon" sizes="72x72" href="/favicons/apple-touch-icon-72x72.png"><link data-n-head="ssr" rel="apple-touch-icon" sizes="76x76" href="/favicons/apple-touch-icon-76x76.png"><link data-n-head="ssr" rel="apple-touch-icon" sizes="114x114" href="/favicons/apple-touch-icon-114x114.png"><link data-n-head="ssr" rel="apple-touch-icon" sizes="120x120" href="/favicons/apple-touch-icon-120x120.png"><link data-n-head="ssr" rel="apple-touch-icon" sizes="144x144" href="/favicons/apple-touch-icon-144x144.png"><link data-n-head="ssr" rel="apple-touch-icon" sizes="152x152" href="/favicons/apple-touch-icon-152x152.png"><link data-n-head="ssr" rel="apple-touch-icon" sizes="180x180" href="/favicons/apple-touch-icon-180x180.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="36x36" href="/favicons/android-chrome-36x36.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="48x48" href="/favicons/android-chrome-48x48.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="72x72" href="/favicons/android-chrome-72x72.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="96x96" href="/favicons/android-chrome-96x96.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="128x128" href="/favicons/android-chrome-128x128.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="144x144" href="/favicons/android-chrome-144x144.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="152x152" href="/favicons/android-chrome-152x152.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="192x192" href="/favicons/android-chrome-192x192.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="256x256" href="/favicons/android-chrome-256x256.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="384x384" href="/favicons/android-chrome-384x384.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="512x512" href="/favicons/android-chrome-512x512.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="36x36" href="/favicons/icon-36x36.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="48x48" href="/favicons/icon-48x48.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="72x72" href="/favicons/icon-72x72.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="96x96" href="/favicons/icon-96x96.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="128x128" href="/favicons/icon-128x128.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="144x144" href="/favicons/icon-144x144.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="152x152" href="/favicons/icon-152x152.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="160x160" href="/favicons/icon-160x160.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="192x192" href="/favicons/icon-192x192.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="196x196" href="/favicons/icon-196x196.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="256x256" href="/favicons/icon-256x256.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="384x384" href="/favicons/icon-384x384.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="512x512" href="/favicons/icon-512x512.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="16x16" href="/favicons/icon-16x16.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="24x24" href="/favicons/icon-24x24.png"><link data-n-head="ssr" rel="icon" type="image/png" sizes="32x32" href="/favicons/icon-32x32.png"><meta name="astro-view-transitions-enabled" content="true"><meta name="astro-view-transitions-fallback" content="animate"><link rel="stylesheet" href="/_astro/_slug_.kTvJvROE.css">
<link rel="stylesheet" href="/_astro/_slug_.DRERk4z5.css">
<style>@charset "UTF-8";._layout-enter_p1it6_2,._fade-enter_p1it6_3{opacity:0}._layout-enter-active_p1it6_6,._fade-enter-active_p1it6_7{transition:opacity .5s}._layout-leave-active_p1it6_10,._fade-leave-active_p1it6_11{transition:opacity .1s}._layout-leave-to_p1it6_14,._fade-leave-to_p1it6_15{opacity:0}._card_p1it6_19{background:#ffffffe6;padding:30px 20px 20px 30px}@media screen and (max-width: 575px){._card_p1it6_19{padding:5.3333333333vw}}._card_p1it6_19:hover ._post__link_p1it6_28:after{width:100%}._card_p1it6_19:hover ._post_p1it6_28{opacity:.5}._card_p1it6_19:hover ._post__link_p1it6_28{transform:translate(5px)}._card_p1it6_19 *{transition:opacity .2s}._new_p1it6_41{position:relative}._new_p1it6_41:after{content:"NEW";color:#fff;font-size:1rem;position:absolute;background:#ff0211;width:40px;height:40px;top:-20px;left:-20px;display:flex;border-radius:100%;justify-content:center;line-height:4}@media screen and (max-width: 575px){._new_p1it6_41:after{left:auto;top:-5.3333333333vw;right:-5.3333333333vw;width:10.6666666667vw;height:10.6666666667vw;font-size:3.2vw;line-height:3.5}}._post_p1it6_28{padding-top:10px;display:flex;flex-wrap:no-wrap}@media screen and (max-width: 575px){._post_p1it6_28{padding-top:2.6666666667vw;display:block}}._post__thumbnail_p1it6_82{width:196px;max-height:110px;object-fit:cover}@media screen and (max-width: 575px){._post__thumbnail_p1it6_82{width:100%;max-height:44vw}}._post__text_p1it6_93{margin-left:25px;flex:1 1 0%;display:flex;flex-wrap:wrap}@media screen and (max-width: 575px){._post__text_p1it6_93{margin-left:0;padding-top:5.3333333333vw}}._post__title_p1it6_105{font-size:2rem;width:100%;font-family:Avenir-Black,Arial,游ゴシック体,YuGothic,游ゴシック,Yu Gothic,sans-serif;font-weight:700;font-style:normal}@media screen and (max-width: 575px){._post__title_p1it6_105{font-size:4.8vw}}._post__meta_p1it6_117{width:100%;display:flex;padding-top:20px;justify-content:space-between}@media screen and (max-width: 575px){._post__meta_p1it6_117{padding-top:5.3333333333vw}}._post__to-post_p1it6_128{display:inline-flex;align-items:flex-end}._post__link_p1it6_28{transition:transform .2s;padding-bottom:6px;position:relative}._post__link_p1it6_28:after{content:"";width:0%;height:1px;background:#3d3d3d;position:absolute;bottom:0;left:0;transition:width .2s ease-out}._meta-wrap_p1it6_148 .tag-list{font-size:1.2rem;color:#969696;display:flex;flex-wrap:wrap}@media screen and (max-width: 575px){._meta-wrap_p1it6_148 .tag-list{font-size:2.9333333333vw}}._meta-wrap_p1it6_148 .tag-list__item{margin-left:5px}@media screen and (max-width: 575px){._meta-wrap_p1it6_148 .tag-list__item{margin-left:1.3333333333vw}}._author_p1it6_168{font-size:1.2rem}@media screen and (max-width: 575px){._author_p1it6_168{font-size:2.9333333333vw}}._author__name_p1it6_176{color:#969696;margin-right:5px}._layout-enter_d76db_1,._fade-enter_d76db_2{opacity:0}._layout-enter-active_d76db_5,._fade-enter-active_d76db_6{transition:opacity .5s}._layout-leave-active_d76db_9,._fade-leave-active_d76db_10{transition:opacity .1s}._layout-leave-to_d76db_13,._fade-leave-to_d76db_14{opacity:0}._tag-list_d76db_18{font-size:1.5rem;color:#969696;display:flex;flex-wrap:wrap}@media screen and (max-width: 575px){._tag-list_d76db_18{font-size:3.2vw}}._tag-list__item_d76db_29:nth-of-type(n+2){margin-left:15px}@media screen and (max-width: 575px){._tag-list__item_d76db_29{margin-left:1.3333333333vw}}._tag-list__item_d76db_29 a,._tag-list__item_d76db_29 p{display:inline-flex;align-items:center}._tag-list__item_d76db_29 a:before,._tag-list__item_d76db_29 p:before{content:"";display:inline-block;background:url(/img/icon_category.svg) 0/100% no-repeat;width:12px;height:12px;margin-right:5px}
._layout-enter_1jn14_1,._fade-enter_1jn14_2{opacity:0}._layout-enter-active_1jn14_5,._fade-enter-active_1jn14_6{transition:opacity .5s}._layout-leave-active_1jn14_9,._fade-leave-active_1jn14_10{transition:opacity .1s}._layout-leave-to_1jn14_13,._fade-leave-to_1jn14_14{opacity:0}._post-meta_1jn14_18{width:100%;margin-bottom:30px}._post-title_1jn14_23{font-size:4rem;padding-top:15px;margin-bottom:5px;font-weight:700}._post-title_1jn14_23:before{content:"";display:block;height:0;width:0;margin-top:calc((1 - 1.6) * .5em)}._post-title_1jn14_23:after{content:"";display:block;height:0;width:0;margin-bottom:calc((1 - 1.6) * .5em)}@media screen and (max-width: 575px){._post-title_1jn14_23{padding-top:6.6666666667vw;font-size:7.4666666667vw;line-height:1.1}._post-title_1jn14_23:before{content:"";display:block;height:0;width:0;margin-top:calc((1 - 1.1) * .5em)}._post-title_1jn14_23:after{content:"";display:block;height:0;width:0;margin-bottom:calc((1 - 1.1) * .5em)}}._link_1jn14_65:hover{opacity:.6;transition:opacity .2s}._layout-enter_1c1i9_1,._fade-enter_1c1i9_2{opacity:0}._layout-enter-active_1c1i9_5,._fade-enter-active_1c1i9_6{transition:opacity .5s}._layout-leave-active_1c1i9_9,._fade-leave-active_1c1i9_10{transition:opacity .1s}._layout-leave-to_1c1i9_13,._fade-leave-to_1c1i9_14{opacity:0}._post-author_1c1i9_18{font-size:1.5rem;margin-left:5px;display:inline-flex}@media screen and (max-width: 575px){._post-author_1c1i9_18{font-size:2.9333333333vw}}._post-author_name_1c1i9_28{color:#969696}._post-author_1c1i9_18 span{margin-right:5px}._post-author_1c1i9_18 dt{font-weight:400}._post-author_1c1i9_18 a:hover{opacity:.6;transition:opacity .2s}
._layout-enter_3t7h3_1,._fade-enter_3t7h3_2{opacity:0}._layout-enter-active_3t7h3_5,._fade-enter-active_3t7h3_6{transition:opacity .5s}._layout-leave-active_3t7h3_9,._fade-leave-active_3t7h3_10{transition:opacity .1s}._layout-leave-to_3t7h3_13,._fade-leave-to_3t7h3_14{opacity:0}._link_3t7h3_18{display:inline-flex;align-items:center}._link_3t7h3_18 a{text-decoration:none}._link_3t7h3_18:hover *{opacity:.5}._link_3t7h3_18:hover ._link__arrow_3t7h3_28{transform:translate(5px)}._link_3t7h3_18 *{transition:opacity .2s}._link__tag_3t7h3_34{border-bottom:1px solid #3d3d3d;margin-right:15px;font-weight:700}._link__arrow_3t7h3_28{transition:transform .2s}@media screen and (max-width: 575px){._link__arrow_3t7h3_28{width:3.7333333333vw;height:2.4vw}}
</style>
<link rel="stylesheet" href="/_astro/_slug_.DflaUU4B.css">
<style>._layout-enter_15cy3_1,._fade-enter_15cy3_2{opacity:0}._layout-enter-active_15cy3_5,._fade-enter-active_15cy3_6{transition:opacity .5s}._layout-leave-active_15cy3_9,._fade-leave-active_15cy3_10{transition:opacity .1s}._layout-leave-to_15cy3_13,._fade-leave-to_15cy3_14{opacity:0}._catItem_15cy3_18{line-height:1.4;display:flex}._catItem_15cy3_18 span:not(:first-child){margin-left:2px}._catItem_15cy3_18 ._RnBRw_15cy3_25{width:15px;height:15px;margin-right:5px;margin-top:3px;color:#b4b4b4}
.layout-enter,.fade-enter{opacity:0}.layout-enter-active,.fade-enter-active{transition:opacity .5s}.layout-leave-active,.fade-leave-active{transition:opacity .1s}.layout-leave-to,.fade-leave-to{opacity:0}.toTop{width:63px;height:63px;border:1px solid #707684;border-radius:50%;display:flex;justify-content:center;align-items:center;background:#ffffffb3;position:fixed;bottom:40px;right:100px;cursor:pointer;z-index:99;opacity:1;transition:.2s}@media screen and (max-width: 767px){.toTop{right:14px;bottom:14px;transition:.3s}}.toTop:disabled{opacity:0;transition:.2s}@media screen and (max-width: 767px){.toTop:disabled{opacity:1;transform:translateY(calc(100% + 14px))}}.toTop .toTop_svg{width:16px;margin-bottom:2px}
</style><script type="module" src="/_astro/hoisted.kDiS2mRD.js"></script></head> <body> <!-- Google Tag Manager (noscript) --> <noscript> <iframe src="https://www.googletagmanager.com/ns.html?id=GTM-WFXJ8Q3" height="0" width="0" style="display:none;visibility:hidden">
</iframe> </noscript> <!-- End Google Tag Manager (noscript) --> <style>astro-island,astro-slot,astro-static-slot{display:contents}</style><script>(()=>{var e=async t=>{await(await t())()};(self.Astro||(self.Astro={})).load=e;window.dispatchEvent(new Event("astro:load"));})();;(()=>{var A=Object.defineProperty;var g=(i,o,a)=>o in i?A(i,o,{enumerable:!0,configurable:!0,writable:!0,value:a}):i[o]=a;var d=(i,o,a)=>g(i,typeof o!="symbol"?o+"":o,a);{let i={0:t=>m(t),1:t=>a(t),2:t=>new RegExp(t),3:t=>new Date(t),4:t=>new Map(a(t)),5:t=>new Set(a(t)),6:t=>BigInt(t),7:t=>new URL(t),8:t=>new Uint8Array(t),9:t=>new Uint16Array(t),10:t=>new Uint32Array(t),11:t=>1/0*t},o=t=>{let[l,e]=t;return l in i?i[l](e):void 0},a=t=>t.map(o),m=t=>typeof t!="object"||t===null?t:Object.fromEntries(Object.entries(t).map(([l,e])=>[l,o(e)]));class y extends HTMLElement{constructor(){super(...arguments);d(this,"Component");d(this,"hydrator");d(this,"hydrate",async()=>{var b;if(!this.hydrator||!this.isConnected)return;let e=(b=this.parentElement)==null?void 0:b.closest("astro-island[ssr]");if(e){e.addEventListener("astro:hydrate",this.hydrate,{once:!0});return}let c=this.querySelectorAll("astro-slot"),n={},h=this.querySelectorAll("template[data-astro-template]");for(let r of h){let s=r.closest(this.tagName);s!=null&&s.isSameNode(this)&&(n[r.getAttribute("data-astro-template")||"default"]=r.innerHTML,r.remove())}for(let r of c){let s=r.closest(this.tagName);s!=null&&s.isSameNode(this)&&(n[r.getAttribute("name")||"default"]=r.innerHTML)}let p;try{p=this.hasAttribute("props")?m(JSON.parse(this.getAttribute("props"))):{}}catch(r){let s=this.getAttribute("component-url")||"<unknown>",v=this.getAttribute("component-export");throw v&&(s+=` (export ${v})`),console.error(`[hydrate] Error parsing props for component ${s}`,this.getAttribute("props"),r),r}let u;await this.hydrator(this)(this.Component,p,n,{client:this.getAttribute("client")}),this.removeAttribute("ssr"),this.dispatchEvent(new CustomEvent("astro:hydrate"))});d(this,"unmount",()=>{this.isConnected||this.dispatchEvent(new CustomEvent("astro:unmount"))})}disconnectedCallback(){document.removeEventListener("astro:after-swap",this.unmount),document.addEventListener("astro:after-swap",this.unmount,{once:!0})}connectedCallback(){if(!this.hasAttribute("await-children")||document.readyState==="interactive"||document.readyState==="complete")this.childrenConnectedCallback();else{let e=()=>{document.removeEventListener("DOMContentLoaded",e),c.disconnect(),this.childrenConnectedCallback()},c=new MutationObserver(()=>{var n;((n=this.lastChild)==null?void 0:n.nodeType)===Node.COMMENT_NODE&&this.lastChild.nodeValue==="astro:end"&&(this.lastChild.remove(),e())});c.observe(this,{childList:!0}),document.addEventListener("DOMContentLoaded",e)}}async childrenConnectedCallback(){let e=this.getAttribute("before-hydration-url");e&&await import(e),this.start()}async start(){let e=JSON.parse(this.getAttribute("opts")),c=this.getAttribute("client");if(Astro[c]===void 0){window.addEventListener(`astro:${c}`,()=>this.start(),{once:!0});return}try{await Astro[c](async()=>{let n=this.getAttribute("renderer-url"),[h,{default:p}]=await Promise.all([import(this.getAttribute("component-url")),n?import(n):()=>()=>{}]),u=this.getAttribute("component-export")||"default";if(!u.includes("."))this.Component=h[u];else{this.Component=h;for(let f of u.split("."))this.Component=this.Component[f]}return this.hydrator=p,this.hydrate},e,this)}catch(n){console.error(`[astro-island] Error hydrating ${this.getAttribute("component-url")}`,n)}}attributeChangedCallback(){this.hydrate()}}d(y,"observedAttributes",["props"]),customElements.get("astro-island")||customElements.define("astro-island",y)}})();</script><astro-island uid="Z2aEtGv" component-url="/_astro/TheHeader.ByGQ2pxB.js" component-export="TheHeader" renderer-url="/_astro/client.DNGgaD7G.js" props="{&quot;currentPath&quot;:[0,&quot;/en/blog/entry/20250428-upgrading-eck-operator-a-side-by-side-kubernetes-operator-upgrade-approach/&quot;],&quot;categories&quot;:[0,{&quot;jp&quot;:[1,[[0,{&quot;uri&quot;:[0,&quot;/category/ai/&quot;],&quot;name&quot;:[0,&quot;AI&quot;],&quot;totalPosts&quot;:[0,116]}],[0,{&quot;uri&quot;:[0,&quot;/category/backend/&quot;],&quot;name&quot;:[0,&quot;Backend&quot;],&quot;totalPosts&quot;:[0,275]}],[0,{&quot;uri&quot;:[0,&quot;/category/client-side/&quot;],&quot;name&quot;:[0,&quot;Client side&quot;],&quot;totalPosts&quot;:[0,161]}],[0,{&quot;uri&quot;:[0,&quot;/category/development/&quot;],&quot;name&quot;:[0,&quot;Development&quot;],&quot;totalPosts&quot;:[0,234]}],[0,{&quot;uri&quot;:[0,&quot;/category/infrastructure/&quot;],&quot;name&quot;:[0,&quot;Infrastructure&quot;],&quot;totalPosts&quot;:[0,149]}],[0,{&quot;uri&quot;:[0,&quot;/category/organization/&quot;],&quot;name&quot;:[0,&quot;Organization&quot;],&quot;totalPosts&quot;:[0,195]}],[0,{&quot;uri&quot;:[0,&quot;/category/qa/&quot;],&quot;name&quot;:[0,&quot;QA&quot;],&quot;totalPosts&quot;:[0,89]}],[0,{&quot;uri&quot;:[0,&quot;/category/research-advanced-tech/&quot;],&quot;name&quot;:[0,&quot;Research &amp;amp; Advanced Tech&quot;],&quot;totalPosts&quot;:[0,31]}],[0,{&quot;uri&quot;:[0,&quot;/category/security/&quot;],&quot;name&quot;:[0,&quot;Security&quot;],&quot;totalPosts&quot;:[0,34]}]]],&quot;en&quot;:[1,[[0,{&quot;uri&quot;:[0,&quot;/category/ai/&quot;],&quot;name&quot;:[0,&quot;AI&quot;],&quot;totalPosts&quot;:[0,59]}],[0,{&quot;uri&quot;:[0,&quot;/category/backend/&quot;],&quot;name&quot;:[0,&quot;Backend&quot;],&quot;totalPosts&quot;:[0,88]}],[0,{&quot;uri&quot;:[0,&quot;/category/client-side/&quot;],&quot;name&quot;:[0,&quot;Client side&quot;],&quot;totalPosts&quot;:[0,85]}],[0,{&quot;uri&quot;:[0,&quot;/category/development/&quot;],&quot;name&quot;:[0,&quot;Development&quot;],&quot;totalPosts&quot;:[0,117]}],[0,{&quot;uri&quot;:[0,&quot;/category/infrastructure/&quot;],&quot;name&quot;:[0,&quot;Infrastructure&quot;],&quot;totalPosts&quot;:[0,84]}],[0,{&quot;uri&quot;:[0,&quot;/category/organization/&quot;],&quot;name&quot;:[0,&quot;Organization&quot;],&quot;totalPosts&quot;:[0,74]}],[0,{&quot;uri&quot;:[0,&quot;/category/qa/&quot;],&quot;name&quot;:[0,&quot;QA&quot;],&quot;totalPosts&quot;:[0,25]}],[0,{&quot;uri&quot;:[0,&quot;/category/research-advanced-tech/&quot;],&quot;name&quot;:[0,&quot;Research &amp;amp; Advanced Tech&quot;],&quot;totalPosts&quot;:[0,11]}],[0,{&quot;uri&quot;:[0,&quot;/category/security/&quot;],&quot;name&quot;:[0,&quot;Security&quot;],&quot;totalPosts&quot;:[0,40]}]]]}]}" ssr="" client="load" opts="{&quot;name&quot;:&quot;TheHeader&quot;,&quot;value&quot;:true}" await-children=""><header class="_header_133ho_18"><div class="_header__inner_133ho_30"><div class="_brand_133ho_51"><a href="/en/"><div><img src="/img/mercari.svg" alt="mercari" width="140" height="29" class="_brand__mercari_133ho_58"/><img src="/img/engineering.svg" alt="engineering" width="205" height="36" class="_brand__engineering_133ho_71"/></div></a><div class="_brand__description_133ho_77 false">Mercari Engineering Portal</div></div><div class="
_navigations_133ho_96
_navigations__pc_133ho_118"><div class="_navigations__top_133ho_155"><ul class="_langSwitch_14x13_18"><li class="_langSwitch__lang_14x13_24"><a data-prefetch-disabled="true" class="_langSwitch__link_14x13_37" href="/blog/entry/20250428-upgrading-eck-operator-a-side-by-side-kubernetes-operator-upgrade-approach/">日本語</a></li><li class="_langSwitch__lang_14x13_24"><a data-prefetch-disabled="true" class="_langSwitch__link_14x13_37" aria-selected="true">English</a></li></ul><div class="_header_sns_133ho_176"><div class="_sns_16o34_18"><a href="https://twitter.com/MercariDev" rel="noopener" target="_blank"><img src="/img/x_black.svg" class="_sns__X_16o34_24" alt="X" width="27" height="22"/></a><a href="https://www.linkedin.com/company/mercari-inc-/" rel="noopener" target="_blank"><img src="/img/linkedin_black.svg" alt="LinkedIn" width="27" height="22"/></a><a href="/en/blog/feed.xml" rel="noopener" target="_blank"><img src="/img/rss_black.svg" alt="RSS" width="27" height="22"/></a></div></div></div><nav><ul class="_navigation_1sjtz_18"><li class="_navigation__item_1sjtz_23 undefined"><div class="_menu__heading_1sjtz_28">Organization</div><ul class="_menu__list_1sjtz_33"><li class="_menu__item_1sjtz_40"><a href="/en/culture" class="menu__link _menu__link_1sjtz_60 ">Culture</a></li><li class="_menu__item_1sjtz_40"><a href="/en/roles" class="menu__link _menu__link_1sjtz_60 ">Roles</a></li></ul></li><li class="_navigation__item_1sjtz_23 undefined"><div class="_menu__heading_1sjtz_28">Technology</div><ul class="_menu__list_1sjtz_33"><li class="_menu__item_1sjtz_40"><a href="/en/technology-stack" class="menu__link _menu__link_1sjtz_60 ">Technology-stack</a></li><li class="_menu__item_1sjtz_40"><a href="/en/open-source" class="menu__link _menu__link_1sjtz_60 ">Open-source</a></li><li class="_menu__item_1sjtz_40"><a href="/en/output" class="menu__link _menu__link_1sjtz_60 ">Output</a></li><li class="_menu__item_1sjtz_40"><a href="/en/learning-materials" class="menu__link _menu__link_1sjtz_60 ">Learning-materials</a></li></ul></li><li class="_navigation__item_1sjtz_23 undefined"><div class="_menu__heading_1sjtz_28">Other</div><ul class="_menu__list_1sjtz_33"><li class="_menu__item_1sjtz_40"><a href="/en/events" class="menu__link _menu__link_1sjtz_60 ">Event</a></li><li class="_menu__item_1sjtz_40 _blog_1sjtz_46"><a href="/en/blog" class="menu__link _menu__link_1sjtz_60 ">Blog</a><div class="_dropdown_1sjtz_90"><nav><ul class="_navDropdown_1l90c_18"><li class="_navDropdown_item_1l90c_27"><a href="/en/blog/category/ai/"><p class="_catItem_15cy3_18"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" width="20" height="16" viewBox="0 0 20 16" class="undefined undefined _RnBRw_15cy3_25 RnBRw"><path fill="currentColor" fill-rule="nonzero" d="M18.799 15.25h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zM2 16a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4z"></path></svg><span>AI</span></p></a></li><li class="_navDropdown_item_1l90c_27"><a href="/en/blog/category/backend/"><p class="_catItem_15cy3_18"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" width="20" height="16" viewBox="0 0 20 16" class="undefined undefined _RnBRw_15cy3_25 RnBRw"><path fill="currentColor" fill-rule="nonzero" d="M18.799 15.25h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zM2 16a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4z"></path></svg><span>Backend</span></p></a></li><li class="_navDropdown_item_1l90c_27"><a href="/en/blog/category/client-side/"><p class="_catItem_15cy3_18"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" width="20" height="16" viewBox="0 0 20 16" class="undefined undefined _RnBRw_15cy3_25 RnBRw"><path fill="currentColor" fill-rule="nonzero" d="M18.799 15.25h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zM2 16a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4z"></path></svg><span>Client side</span></p></a></li><li class="_navDropdown_item_1l90c_27"><a href="/en/blog/category/development/"><p class="_catItem_15cy3_18"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" width="20" height="16" viewBox="0 0 20 16" class="undefined undefined _RnBRw_15cy3_25 RnBRw"><path fill="currentColor" fill-rule="nonzero" d="M18.799 15.25h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zM2 16a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4z"></path></svg><span>Development</span></p></a></li><li class="_navDropdown_item_1l90c_27"><a href="/en/blog/category/infrastructure/"><p class="_catItem_15cy3_18"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" width="20" height="16" viewBox="0 0 20 16" class="undefined undefined _RnBRw_15cy3_25 RnBRw"><path fill="currentColor" fill-rule="nonzero" d="M18.799 15.25h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zM2 16a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4z"></path></svg><span>Infrastructure</span></p></a></li><li class="_navDropdown_item_1l90c_27"><a href="/en/blog/category/organization/"><p class="_catItem_15cy3_18"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" width="20" height="16" viewBox="0 0 20 16" class="undefined undefined _RnBRw_15cy3_25 RnBRw"><path fill="currentColor" fill-rule="nonzero" d="M18.799 15.25h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zM2 16a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4z"></path></svg><span>Organization</span></p></a></li><li class="_navDropdown_item_1l90c_27"><a href="/en/blog/category/qa/"><p class="_catItem_15cy3_18"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" width="20" height="16" viewBox="0 0 20 16" class="undefined undefined _RnBRw_15cy3_25 RnBRw"><path fill="currentColor" fill-rule="nonzero" d="M18.799 15.25h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zM2 16a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4z"></path></svg><span>QA</span></p></a></li><li class="_navDropdown_item_1l90c_27"><a href="/en/blog/category/research-advanced-tech/"><p class="_catItem_15cy3_18"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" width="20" height="16" viewBox="0 0 20 16" class="undefined undefined _RnBRw_15cy3_25 RnBRw"><path fill="currentColor" fill-rule="nonzero" d="M18.799 15.25h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zM2 16a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4z"></path></svg><span>Research &amp; Advanced Tech</span></p></a></li><li class="_navDropdown_item_1l90c_27"><a href="/en/blog/category/security/"><p class="_catItem_15cy3_18"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" width="20" height="16" viewBox="0 0 20 16" class="undefined undefined _RnBRw_15cy3_25 RnBRw"><path fill="currentColor" fill-rule="nonzero" d="M18.799 15.25h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zm0-6h-11.5a1.25 1.25 0 1 1 0-2.5h11.5a1.25 1.25 0 0 1 0 2.5zM2 16a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4zm0-6a2 2 0 1 0 0-4 2 2 0 0 0 0 4z"></path></svg><span>Security</span></p></a></li></ul></nav></div></li><li class="_menu__item_1sjtz_40"><a href="https://careers.mercari.com/job-categories/engineering/" class="menu__link _menu__link_1sjtz_60 " rel="noopener" target="_blank">Careers</a></li></ul></li></ul></nav></div><div class="_hamburgerWrap_133ho_193"><button class="
_hamburger_wmr22_18
false" aria-label="Menu"><span></span><span></span><span></span></button></div></div><div class="
_navigations_133ho_96
_navigations__sp_133ho_123
false"><nav class="_navigation_7gm6v_18"><dl class="_navigation__inner_7gm6v_23"><div class="
_navigation__inner__item_7gm6v_30
_menu_7gm6v_38" style="flex:
3
3
0%"><dt class="_menu__heading_7gm6v_53">Organization</dt><dd><a href="/en/culture" class="
menu__link _menu__link_7gm6v_58
">Culture</a></dd><dd><a href="/en/roles" class="
menu__link _menu__link_7gm6v_58
">Roles</a></dd></div><div class="
_navigation__inner__item_7gm6v_30
_menu_7gm6v_38" style="flex:
5
5
0%"><dt class="_menu__heading_7gm6v_53">Technology</dt><dd><a href="/en/technology-stack" class="
menu__link _menu__link_7gm6v_58
">Technology-stack</a></dd><dd><a href="/en/open-source" class="
menu__link _menu__link_7gm6v_58
">Open-source</a></dd><dd><a href="/en/output" class="
menu__link _menu__link_7gm6v_58
">Output</a></dd><dd><a href="/en/learning-materials" class="
menu__link _menu__link_7gm6v_58
">Learning-materials</a></dd></div><div class="
_navigation__inner__item_7gm6v_30
_menu_7gm6v_38" style="flex:
4
4
0%"><dt class="_menu__heading_7gm6v_53">Other</dt><dd><a href="/en/events" class="
menu__link _menu__link_7gm6v_58
">Event</a></dd><dd><a href="/en/blog" class="
menu__link _menu__link_7gm6v_58
">Blog</a></dd><dd><a href="https://careers.mercari.com/job-categories/engineering/" class="menu__link _menu__link_7gm6v_58" target="_blank">Careers</a></dd></div></dl></nav><div class="_sp_bottom_133ho_162"><ul class="_langSwitch_14x13_18"><li class="_langSwitch__lang_14x13_24"><a data-prefetch-disabled="true" class="_langSwitch__link_14x13_37" href="/blog/entry/20250428-upgrading-eck-operator-a-side-by-side-kubernetes-operator-upgrade-approach/">日本語</a></li><li class="_langSwitch__lang_14x13_24"><a data-prefetch-disabled="true" class="_langSwitch__link_14x13_37" aria-selected="true">English</a></li></ul><div class="_sns_16o34_18"><a href="https://twitter.com/MercariDev" rel="noopener" target="_blank"><img src="/img/x_black.svg" class="_sns__X_16o34_24" alt="X" width="27" height="22"/></a><a href="https://www.linkedin.com/company/mercari-inc-/" rel="noopener" target="_blank"><img src="/img/linkedin_black.svg" alt="LinkedIn" width="27" height="22"/></a><a href="/en/blog/feed.xml" rel="noopener" target="_blank"><img src="/img/rss_black.svg" alt="RSS" width="27" height="22"/></a></div></div></div></header><!--astro:end--></astro-island> <main> <div class="_hero_xz05m_18 _pageHeroEntry_1eyul_18"><div class="container _container_xz05m_44"><time datetime="2025-04-28">2025/04/28</time><div><div class="_post-meta_1jn14_18"><h1 class="_post-title_1jn14_23 "><div>Upgrading ECK Operator: A Side-by-Side Kubernetes Operator Upgrade Approach</div></h1><ul class="_tag-list_d76db_18 tag-list"><li class="_tag-list__item_d76db_29"><a href="/en/blog/category/infrastructure/">Infrastructure</a></li></ul><dl class="_post-author_1c1i9_18"><dt>Author:</dt><dd><a href="/en/blog/author/munagekar/"><span class="_post-author_name_1c1i9_28 post-author_name">Abhishek Munagekar</span></a></dd></dl><span> , <time datetime="2025-04-28">2025/04/28</time></span></div></div><img src="https://storage.googleapis.com/prd-engineering-asset/2025/12/521c5551-image_fx-1024x559-1.png" alt="Upgrading ECK Operator: A Side-by-Side Kubernetes Operator Upgrade Approach" class="_post__thumbnail_1eyul_24"/><div class="_lang-switch_1eyul_40"></div></div></div> <section> <div class="container _container_11e9j_22 container--inner"> <astro-island uid="Z1Uo0TD" component-url="/_astro/PageContent.CfXlRJsU.js" component-export="PageContent" renderer-url="/_astro/client.DNGgaD7G.js" props="{&quot;html&quot;:[0,&quot;&lt;p&gt;Greetings, I&amp;#8217;m Abhishek Munagekar from the Search Infrastructure Team at Mercari. Our team manages several Elasticsearch clusters deployed on Kubernetes, forming a crucial part of our search infrastructure. We rely on the &lt;a href=\&quot;https://www.elastic.co/elastic-cloud-kubernetes\&quot; title=\&quot;Elastic Cloud on Kubernetes\&quot;&gt;Elastic Cloud on Kubernetes&lt;/a&gt; (ECK) Operator to orchestrate these clusters, all housed within a dedicated namespace maintained by our team.&lt;/p&gt;\n&lt;p&gt;To leverage the advancements in recently released ECK operator versions, we embarked on an upgrade project. Operator upgrades are inherently complex and risky, often involving significant changes that can affect system stability.&lt;/p&gt;\n&lt;p&gt;In this article, I&amp;#8217;ll delve into the challenges we encountered and the strategies we employed to manage operator upgrades for stateful workloads like Elasticsearch. Additionally, I&amp;#8217;ll detail how we modified the ECK operator to facilitate a more resilient side-by-side upgrade process.&lt;/p&gt;\n&lt;h2&gt;Minimizing Risk in a Critical Infrastructure&lt;/h2&gt;\n&lt;p&gt;At Mercari, our Elasticsearch infrastructure is integral to multiple business units, notably powering the marketplace search functionality. Any disruption or downtime to this infrastructure carries the potential for significant financial repercussions. Therefore, our primary objective during ECK operator upgrades is to mitigate risk to the absolute minimum. This necessitates a cautious and strategic approach, favoring gradual rollouts over abrupt &lt;strong&gt;big-bang&lt;/strong&gt; deployments, employing side-by-side upgrades instead of in-place replacements, and ensuring robust disaster recovery plans.&lt;/p&gt;\n&lt;p&gt;We utilize a suite of safety nets and backup mechanisms, including Elasticsearch snapshots, real-time write request backups, standby cluster preparations, and rigorous testing across multiple environments. While the details of these mechanisms are extensive, they fall beyond the scope of this particular article.&lt;/p&gt;\n&lt;h2&gt;In-place Upgrade Mechanism used by the Native ECK Operator&lt;/h2&gt;\n&lt;p&gt;Typically, Kubernetes operators, including the native ECK operator, perform in-place upgrades, where an existing component is directly replaced with a newer version. In contrast, a side-by-side upgrade involves running two versions of the same component concurrently. Here&amp;#8217;s a comparative overview:&lt;/p&gt;\n&lt;table&gt;\n&lt;thead&gt;\n&lt;tr&gt;\n&lt;th&gt;Feature&lt;/th&gt;\n&lt;th&gt;In-place Upgrade&lt;/th&gt;\n&lt;th&gt;Side-by-side Upgrade&lt;/th&gt;\n&lt;/tr&gt;\n&lt;/thead&gt;\n&lt;tbody&gt;\n&lt;tr&gt;\n&lt;td&gt;&lt;strong&gt;Downtime&lt;/strong&gt;&lt;/td&gt;\n&lt;td&gt;Possible&lt;/td&gt;\n&lt;td&gt;Minimized&lt;/td&gt;\n&lt;/tr&gt;\n&lt;tr&gt;\n&lt;td&gt;&lt;strong&gt;Rollback&lt;/strong&gt;&lt;/td&gt;\n&lt;td&gt;More Difficult&lt;/td&gt;\n&lt;td&gt;Feasible&lt;/td&gt;\n&lt;/tr&gt;\n&lt;tr&gt;\n&lt;td&gt;&lt;strong&gt;Resource Usage&lt;/strong&gt;&lt;/td&gt;\n&lt;td&gt;Lower&lt;/td&gt;\n&lt;td&gt;Higher (Double)&lt;/td&gt;\n&lt;/tr&gt;\n&lt;tr&gt;\n&lt;td&gt;&lt;strong&gt;Complexity&lt;/strong&gt;&lt;/td&gt;\n&lt;td&gt;Lower&lt;/td&gt;\n&lt;td&gt;Higher&lt;/td&gt;\n&lt;/tr&gt;\n&lt;tr&gt;\n&lt;td&gt;&lt;strong&gt;Examples&lt;/strong&gt;&lt;/td&gt;\n&lt;td&gt;OS upgrades&lt;/td&gt;\n&lt;td&gt;Database Upgrades&lt;/td&gt;\n&lt;/tr&gt;\n&lt;/tbody&gt;\n&lt;/table&gt;\n&lt;p&gt;In-place upgrades carry inherent risks, particularly with stateful workloads like Elasticsearch. If issues arise, rollback is complex and time-consuming, leading to prolonged recovery periods. This is in contrast to stateless workloads, where recovery is generally faster and less risky.&lt;/p&gt;\n&lt;h2&gt;Limiting Standard ECK Upgrades&lt;/h2&gt;\n&lt;p&gt;A standard ECK operator upgrade triggers a rolling restart of Elasticsearch nodes across all clusters simultaneously. This all-at-once approach is unacceptable for our high-stakes production environment, where a more gradual rollout is essential. The ECK operator offers an annotation, &lt;code&gt;eck.k8s.elastic.co/managed=false&lt;/code&gt;, to temporarily unmanage Elasticsearch clusters, allowing for one-by-one upgrades.&lt;/p&gt;\n&lt;p&gt;However, this solution conflicts with our infrastructure&amp;#8217;s CPU-based autoscaling mechanism. Our system monitors data nodeset CPU usage and scales Elasticsearch by modifying the manifest, with the ECK operator provisioning the necessary nodes. Disabling the operator&amp;#8217;s management effectively halts our autoscaling (detailed in &lt;a href=\&quot;https://engineering.mercari.com/en/blog/entry/20230620-f0782fd75f/\&quot; title=\&quot;this blog article\&quot;&gt;this blog article&lt;/a&gt;).&lt;/p&gt;\n&lt;p&gt;One workaround would be to manually scale workloads to maximum capacity, apply the unmanaged annotation, and then proceed with a serial upgrade process, by removing the unmanaged annotation one at a time.&lt;/p&gt;\n&lt;p&gt;Following is a flowchart for the proposed plan.&lt;/p&gt;\n&lt;p&gt;&lt;img src=\&quot;https://storage.googleapis.com/prd-engineering-asset/2025/12/f9da010e-rejected-upgrade-plan-for-eck.png\&quot; alt=\&quot;Upgrade Plan Using ECK Unmanaged Label\&quot; /&gt;&lt;/p&gt;\n&lt;p&gt;But this was rejected for the following reasons:&lt;/p&gt;\n&lt;ul&gt;\n&lt;li&gt;&lt;strong&gt;Costly&lt;/strong&gt;: Disables crucial autoscaling features.&lt;/li&gt;\n&lt;li&gt;&lt;strong&gt;Inflexible&lt;/strong&gt;: Prevents scaling during unexpected traffic surges.&lt;/li&gt;\n&lt;li&gt;&lt;strong&gt;Restrictive&lt;/strong&gt;: Blocks any configuration changes to Elasticsearch during the upgrade.&lt;/li&gt;\n&lt;/ul&gt;\n&lt;h1&gt;Our Solution: A Custom Side-by-Side Upgrade Strategy&lt;/h1&gt;\n&lt;p&gt;To circumvent these limitations, we chose to implement a custom side-by-side upgrade approach that mimics the granular control of &lt;code&gt;eck.k8s.elastic.co/managed=false&lt;/code&gt; but is tied to the operator&amp;#8217;s version.&lt;/p&gt;\n&lt;h2&gt;Introducing Operator Version Labeling&lt;/h2&gt;\n&lt;p&gt;We introduced a new label:&lt;/p&gt;\n&lt;pre&gt;&lt;code&gt;eaas.search.mercari.in/desired-controller-version = x.y.z&lt;/code&gt;&lt;/pre&gt;\n&lt;p&gt;This label is applied to all Elasticsearch clusters, initially set to the current (older) operator version. We then modified the ECK operator&amp;#8217;s logic(referencing &lt;a href=\&quot;https://github.com/elastic/cloud-on-k8s/blob/c0496019a2ed1e37a2d127f64c0ba2b26ad23291/pkg/controller/common/unmanaged.go#L23\&quot; title=\&quot;this GitHub link\&quot;&gt;this GitHub link&lt;/a&gt;) to recognize this label and control cluster management accordingly.&lt;/p&gt;\n&lt;h2&gt;Modifying the Controller for Dual Version Support&lt;/h2&gt;\n&lt;p&gt;Both the existing (older) and the new ECK operator versions were modified to support this label. Functionally, we adapted the &lt;code&gt;IsUnmanaged&lt;/code&gt; function and the main controller loop to:&lt;/p&gt;\n&lt;ul&gt;\n&lt;li&gt;Check for the &lt;code&gt;eaas.search.mercari.in/desired-controller-version&lt;/code&gt; label.&lt;/li&gt;\n&lt;li&gt;Skip reconciliation if the label is missing or if the label&amp;#8217;s version does not match the operator&amp;#8217;s build version.&lt;/li&gt;\n&lt;/ul&gt;\n&lt;p&gt;&lt;img src=\&quot;https://storage.googleapis.com/prd-engineering-asset/2025/12/0d0484d5-controller-logic.png\&quot; alt=\&quot;\&quot; /&gt;&lt;/p&gt;\n&lt;p&gt;Here&amp;#8217;s the relevant code snippet:&lt;/p&gt;\n&lt;pre&gt;&lt;code class=\&quot;language-go\&quot;&gt;const desiredECKControllerVersionLabel = &amp;quot;eaas.search.mercari.in/desired-controller-version&amp;quot;\n\nfunc IsUnmanaged(ctx context.Context, object metav1.Object) bool {\n managed, exists := object.GetAnnotations()[ManagedAnnotation]\n if exists &amp;amp;&amp;amp; managed == &amp;quot;false&amp;quot; {\n return true\n }\n\n desiredVersion, exists := object.GetLabels()[desiredECKControllerVersionLabel]\n\n if !exists {\n ulog.FromContext(ctx).Info(fmt.Sprintf(&amp;quot;Object doesn&amp;#039;t have %s label. Skipping reconciliation&amp;quot;, desiredECKControllerVersionLabel), &amp;quot;namespace&amp;quot;, object.GetNamespace(), &amp;quot;name&amp;quot;, object.GetName())\n return true\n }\n\n if desiredVersion != about.GetBuildInfo().Version {\n ulog.FromContext(ctx).Info(\n fmt.Sprintf(&amp;quot;Object is not the target of this controller by %s label. Skipping reconciliation&amp;quot;, desiredECKControllerVersionLabel),\n &amp;quot;desired_version&amp;quot;, desiredVersion,\n &amp;quot;operator_version&amp;quot;, about.GetBuildInfo().Version,\n &amp;quot;namespace&amp;quot;, object.GetNamespace(),\n &amp;quot;name&amp;quot;, object.GetName(),\n )\n return true\n }\n\n paused, exists := object.GetAnnotations()[LegacyPauseAnnoation]\n if exists {\n ulog.FromContext(ctx).Info(fmt.Sprintf(&amp;quot;%s is deprecated, please use %s&amp;quot;, LegacyPauseAnnoation, ManagedAnnotation), &amp;quot;namespace&amp;quot;, object.GetNamespace(), &amp;quot;name&amp;quot;, object.GetName())\n }\n return exists &amp;amp;&amp;amp; paused == &amp;quot;true&amp;quot;\n}\n&lt;/code&gt;&lt;/pre&gt;\n&lt;h1&gt;Handling Custom Resource Definitions (CRDs)&lt;/h1&gt;\n&lt;p&gt;The ECK operator defines a custom resource of &lt;strong&gt;Kind: Elasticsearch&lt;/strong&gt;. This is a cluster-scoped resource, not a namespaced resource, so we cannot define two distinct versions of the CRD concurrently within the same cluster.&lt;/p&gt;\n&lt;p&gt;In this scenario, we rely on the backward compatibility of the CRD definition. It&amp;#8217;s crucial to note that while CRDs are expected to be backward compatible, they may not be forward compatible. Backward compatibility ensures that older operator versions can work with newer CRD definitions. However, forward compatibility, which would mean newer operators can seamlessly work with older CRD definitions, is not guaranteed.&lt;/p&gt;\n&lt;p&gt;This implies that the latest version of the CRD must be deployed to the cluster when running two different versions of the ECK operator side-by-side. Failure to do so could lead to issues where the newer operator version cannot interpret newer CRD fields or configurations, resulting in deployment or operational errors. Therefore, before initiating an upgrade, ensuring the newest CRD version is applied is a critical prerequisite.&lt;/p&gt;\n&lt;h1&gt;Handling Validating Webhook&lt;/h1&gt;\n&lt;p&gt;ECK also defines a validating webhook, which validates the Elasticsearch manifests before they are applied to the cluster. When running two versions of the ECK operator concurrently, it is crucial to ensure that each operator version only validates the Elasticsearch clusters for which its &lt;code&gt;desired-controller-version&lt;/code&gt; matches.&lt;/p&gt;\n&lt;p&gt;The default webhook configuration, without any restrictions, would mean that an Elasticsearch manifest could be validated by both versions of the operator. This poses a significant risk because newer operator versions might introduce new features or modifications to the validation logic. These changes could render validation performed by an older operator version incompatible or incorrect for the expectations of the newer operator, or vice versa. This discrepancy could potentially lead to deployment failures, configuration errors, or unexpected behavior.&lt;/p&gt;\n&lt;p&gt;Instead of modifying the controller logic itself, a simple object selector was added to the webhook configuration.&lt;/p&gt;\n&lt;pre&gt;&lt;code class=\&quot;language-yaml\&quot;&gt; objectSelector:\n matchLabels:\n eaas.search.mercari.in/controller-version: x.y.z&lt;/code&gt;&lt;/pre&gt;\n&lt;p&gt;&lt;img src=\&quot;https://storage.googleapis.com/prd-engineering-asset/2025/12/324c7ad5-validating-webhook-e1745573742670.png\&quot; alt=\&quot;\&quot; /&gt;&lt;/p&gt;\n&lt;p&gt;This &lt;code&gt;objectSelector&lt;/code&gt; with &lt;code&gt;matchLabels&lt;/code&gt; ensures that each ECK operator version only validates Elasticsearch manifests that have the corresponding &lt;code&gt;desired-controller-version&lt;/code&gt;. By isolating the validation process based on the operator version, we prevent potential conflicts and ensure that manifests are only validated by the operator version that is expected to manage them.&lt;/p&gt;\n&lt;h2&gt;Leader Election for High Availability in ECK Operator Upgrades&lt;/h2&gt;\n&lt;p&gt;The ECK operator employs leader election to ensure high availability. Multiple instances of the operator can run concurrently, but only one acts as the active leader responsible for processing changes. This leader election mechanism relies on Kubernetes leases, specifically by acquiring a Kubernetes lease.&lt;/p&gt;\n&lt;p&gt;In a standard, in-place upgrade scenario, the ECK operator uses a constant Kubernetes lease named &lt;code&gt;elastic-operator-leader&lt;/code&gt;. Regardless of the operator version, they all contend for this same lease. When an in-place upgrade occurs, the new operator version simply replaces the old and takes over this existing lease.&lt;br /&gt;\nThe following diagram illustrates the leader election process during a standard in-place upgrade:&lt;/p&gt;\n&lt;p&gt;&lt;img src=\&quot;https://storage.googleapis.com/prd-engineering-asset/2025/12/323c851d-default-eck-operator-leader-election-e1745573800171.png\&quot; alt=\&quot;\&quot; /&gt;&lt;/p&gt;\n&lt;p&gt;However, the default lease strategy presents a challenge for our side-by-side upgrade approach. Since both the older and newer ECK operator versions would try to acquire the same elastic-operator-leader lease, it would result in contention and only one version of the operator could run at a given time. To facilitate our dual-version scenario, we needed a way to separate the leader election for each version.&lt;/p&gt;\n&lt;p&gt;To address this, we modified the ECK operator&amp;#8217;s leader election logic to create distinct Kubernetes leases based on the operator&amp;#8217;s version. This ensures that each operator version has its own separate leader election process, allowing them to run in high availability side-by-side without conflict.&lt;/p&gt;\n&lt;p&gt;We made changes to the LeaderElectionID in the &lt;a href=\&quot;https://github.com/elastic/cloud-on-k8s/blob/be88fb68c4638f4c18dc7fdea1d52c9b425f5b0b/cmd/manager/main.go#L569\&quot; title=\&quot;ECK operator code\&quot;&gt;ECK operator code&lt;/a&gt;. This ID now includes the operator&amp;#8217;s version:&lt;/p&gt;\n&lt;pre&gt;&lt;code class=\&quot;language-go\&quot;&gt;func GetLeaderElectionLeaseName() string {\n buildInfo := about.GetBuildInfo()\n k8sVersion := strings.ReplaceAll(buildInfo.Version, &amp;quot;.&amp;quot;, &amp;quot;-&amp;quot;)\n leasePrefix := &amp;quot;elastic-operator-leader-v&amp;quot;\n leaseName := fmt.Sprintf(&amp;quot;%s%s&amp;quot;, leasePrefix, k8sVersion)\n return leaseName\n }\n\nLeaderElectionID: GetLeaderElectionLeaseName()\n&lt;/code&gt;&lt;/pre&gt;\n&lt;p&gt;In essence, this change transforms the default elastic-operator-leader lease into version-specific leases, such as &lt;code&gt;elastic-operator-leader-v2-16-1&lt;/code&gt; for version 2.16.1. With these versioned leases, each ECK operator instance will only participate in leader election with instances of the same version. The following diagram shows the leader election process with our side-by-side upgrade:&lt;/p&gt;\n&lt;p&gt;&lt;img src=\&quot;https://storage.googleapis.com/prd-engineering-asset/2025/12/323c851d-default-eck-operator-leader-election-e1745573800171.png\&quot; alt=\&quot;\&quot; /&gt;&lt;/p&gt;\n&lt;h2&gt;Testing Our Approach Thoroughly&lt;/h2&gt;\n&lt;p&gt;The Search Infrastructure Team at Mercari leverages three distinct environments to ensure the stability and safety of our infrastructure changes:&lt;/p&gt;\n&lt;ul&gt;\n&lt;li&gt;&lt;strong&gt;Laboratory Environment&lt;/strong&gt;: This environment serves as a dedicated playground, allowing the infrastructure team to rigorously test changes without impacting the development environment. It&amp;#8217;s our sandbox for experimentation and initial validation.&lt;/li&gt;\n&lt;li&gt;&lt;strong&gt;Development Environment&lt;/strong&gt;: This environment mirrors the production setup to a significant degree and is primarily used for Quality Assurance (QA) testing and the development of new features. This is where we validate changes under conditions closely resembling those in production.&lt;/li&gt;\n&lt;li&gt;&lt;strong&gt;Production Environment&lt;/strong&gt;: This is the live environment serving real user traffic, demanding the highest level of stability and reliability.&lt;/li&gt;\n&lt;/ul&gt;\n&lt;p&gt;Before any production deployment, changes are meticulously tested in both the laboratory and development environments. We conduct comprehensive testing to ensure both the older and newer versions of the ECK operator can coexist without conflicts. This includes verifying the labeling system, controller logic modifications, CRD handling, and validating webhook changes. We also perform thorough rollback tests to guarantee that we can quickly revert to the previous state if issues arise. This rigorous testing across multiple environments is crucial to minimizing risk in our high-stakes production environment.&lt;/p&gt;\n&lt;h1&gt;Rollout to Production: A Phased and Monitored Process&lt;/h1&gt;\n&lt;p&gt;Our production rollout follows a phased and closely monitored approach to minimize risk. This involves:&lt;/p&gt;\n&lt;ol&gt;\n&lt;li&gt;Preparation: Verify CRDs and webhook configurations are compatible with the new operator version.&lt;/li&gt;\n&lt;li&gt;Labeling: Tag all Elasticsearch clusters with &lt;code&gt;eaas.search.mercari.in/controller-version&lt;/code&gt; set to the current operator version for tracking.&lt;/li&gt;\n&lt;li&gt;Dual Deployment: Deploy both old and new ECK operators concurrently.&lt;/li&gt;\n&lt;li&gt;Gradual Rollout: Upgrade clusters incrementally by updating their labels to point to the new operator version (&lt;code&gt;eaas.search.mercari.in/controller-version=&amp;lt;new_version&amp;gt;&lt;/code&gt;) cluster-by-cluster.&lt;/li&gt;\n&lt;li&gt;Continuous Monitoring: Track key metrics like error rates, system stability, and resource usage during each upgrade.&lt;/li&gt;\n&lt;li&gt;Validation &amp;amp; Rollback: After each cluster upgrade, validate success or rollback by reverting labels and configurations if needed.&lt;/li&gt;\n&lt;li&gt;Completion: Upgrade remaining clusters, validate, and then remove the older operator version.&lt;/li&gt;\n&lt;/ol&gt;\n&lt;p&gt;The following diagram illustrates the workflow that we follow.&lt;/p&gt;\n&lt;p&gt;&lt;img src=\&quot;https://storage.googleapis.com/prd-engineering-asset/2025/12/22f5ac94-final-workflow.png\&quot; alt=\&quot;\&quot; /&gt;&lt;/p&gt;\n&lt;h1&gt;Conclusion&lt;/h1&gt;\n&lt;p&gt;In summary, upgrading critical systems like the ECK operator needs careful planning and testing. Mercari&amp;#8217;s specific needs led us to create a unique side-by-side upgrade strategy. By carefully changing the operator and using a step-by-step release, we successfully reduced risks and kept our search system running smoothly.&lt;/p&gt;\n&lt;p&gt;It&amp;#8217;s often hard to perfectly copy real-world workloads in testing environments. This can lead to bugs slipping through. This challenge highlights a limitation of the standard approach, as standard operator upgrades are usually tested in development before going to production all at once. &lt;/p&gt;\n&lt;p&gt;While Kubernetes applications use methods like gradual releases and canary deployments, operator upgrades often use an all-at-once method. We found this wasn&amp;#8217;t ideal for our critical search infrastructure.&lt;/p&gt;\n&lt;p&gt;With our successful ECK operator upgrade using the side-by-side approach, we plan to use this strategy for other critical operator upgrades in our production system. We hope our approach helps other teams manage Kubernetes operators, especially those which handle stateful workloads.&lt;/p&gt;\n&quot;]}" ssr="" client="load" opts="{&quot;name&quot;:&quot;PageContent&quot;,&quot;value&quot;:true}" await-children=""><div class="_body_1o0xy_19"><div class="page-content undefined"><p>Greetings, I&#8217;m Abhishek Munagekar from the Search Infrastructure Team at Mercari. Our team manages several Elasticsearch clusters deployed on Kubernetes, forming a crucial part of our search infrastructure. We rely on the <a href="https://www.elastic.co/elastic-cloud-kubernetes" title="Elastic Cloud on Kubernetes">Elastic Cloud on Kubernetes</a> (ECK) Operator to orchestrate these clusters, all housed within a dedicated namespace maintained by our team.</p>
<p>To leverage the advancements in recently released ECK operator versions, we embarked on an upgrade project. Operator upgrades are inherently complex and risky, often involving significant changes that can affect system stability.</p>
<p>In this article, I&#8217;ll delve into the challenges we encountered and the strategies we employed to manage operator upgrades for stateful workloads like Elasticsearch. Additionally, I&#8217;ll detail how we modified the ECK operator to facilitate a more resilient side-by-side upgrade process.</p>
<h2>Minimizing Risk in a Critical Infrastructure</h2>
<p>At Mercari, our Elasticsearch infrastructure is integral to multiple business units, notably powering the marketplace search functionality. Any disruption or downtime to this infrastructure carries the potential for significant financial repercussions. Therefore, our primary objective during ECK operator upgrades is to mitigate risk to the absolute minimum. This necessitates a cautious and strategic approach, favoring gradual rollouts over abrupt <strong>big-bang</strong> deployments, employing side-by-side upgrades instead of in-place replacements, and ensuring robust disaster recovery plans.</p>
<p>We utilize a suite of safety nets and backup mechanisms, including Elasticsearch snapshots, real-time write request backups, standby cluster preparations, and rigorous testing across multiple environments. While the details of these mechanisms are extensive, they fall beyond the scope of this particular article.</p>
<h2>In-place Upgrade Mechanism used by the Native ECK Operator</h2>
<p>Typically, Kubernetes operators, including the native ECK operator, perform in-place upgrades, where an existing component is directly replaced with a newer version. In contrast, a side-by-side upgrade involves running two versions of the same component concurrently. Here&#8217;s a comparative overview:</p>
<table>
<thead>
<tr>
<th>Feature</th>
<th>In-place Upgrade</th>
<th>Side-by-side Upgrade</th>
</tr>
</thead>
<tbody>
<tr>
<td><strong>Downtime</strong></td>
<td>Possible</td>
<td>Minimized</td>
</tr>
<tr>
<td><strong>Rollback</strong></td>
<td>More Difficult</td>
<td>Feasible</td>
</tr>
<tr>
<td><strong>Resource Usage</strong></td>
<td>Lower</td>
<td>Higher (Double)</td>
</tr>
<tr>
<td><strong>Complexity</strong></td>
<td>Lower</td>
<td>Higher</td>
</tr>
<tr>
<td><strong>Examples</strong></td>
<td>OS upgrades</td>
<td>Database Upgrades</td>
</tr>
</tbody>
</table>
<p>In-place upgrades carry inherent risks, particularly with stateful workloads like Elasticsearch. If issues arise, rollback is complex and time-consuming, leading to prolonged recovery periods. This is in contrast to stateless workloads, where recovery is generally faster and less risky.</p>
<h2>Limiting Standard ECK Upgrades</h2>
<p>A standard ECK operator upgrade triggers a rolling restart of Elasticsearch nodes across all clusters simultaneously. This all-at-once approach is unacceptable for our high-stakes production environment, where a more gradual rollout is essential. The ECK operator offers an annotation, <code>eck.k8s.elastic.co/managed=false</code>, to temporarily unmanage Elasticsearch clusters, allowing for one-by-one upgrades.</p>
<p>However, this solution conflicts with our infrastructure&#8217;s CPU-based autoscaling mechanism. Our system monitors data nodeset CPU usage and scales Elasticsearch by modifying the manifest, with the ECK operator provisioning the necessary nodes. Disabling the operator&#8217;s management effectively halts our autoscaling (detailed in <a href="https://engineering.mercari.com/en/blog/entry/20230620-f0782fd75f/" title="this blog article">this blog article</a>).</p>
<p>One workaround would be to manually scale workloads to maximum capacity, apply the unmanaged annotation, and then proceed with a serial upgrade process, by removing the unmanaged annotation one at a time.</p>
<p>Following is a flowchart for the proposed plan.</p>
<p><img src="https://storage.googleapis.com/prd-engineering-asset/2025/12/f9da010e-rejected-upgrade-plan-for-eck.png" alt="Upgrade Plan Using ECK Unmanaged Label" /></p>
<p>But this was rejected for the following reasons:</p>
<ul>
<li><strong>Costly</strong>: Disables crucial autoscaling features.</li>
<li><strong>Inflexible</strong>: Prevents scaling during unexpected traffic surges.</li>
<li><strong>Restrictive</strong>: Blocks any configuration changes to Elasticsearch during the upgrade.</li>
</ul>
<h1>Our Solution: A Custom Side-by-Side Upgrade Strategy</h1>
<p>To circumvent these limitations, we chose to implement a custom side-by-side upgrade approach that mimics the granular control of <code>eck.k8s.elastic.co/managed=false</code> but is tied to the operator&#8217;s version.</p>
<h2>Introducing Operator Version Labeling</h2>
<p>We introduced a new label:</p>
<pre><code>eaas.search.mercari.in/desired-controller-version = x.y.z</code></pre>
<p>This label is applied to all Elasticsearch clusters, initially set to the current (older) operator version. We then modified the ECK operator&#8217;s logic(referencing <a href="https://github.com/elastic/cloud-on-k8s/blob/c0496019a2ed1e37a2d127f64c0ba2b26ad23291/pkg/controller/common/unmanaged.go#L23" title="this GitHub link">this GitHub link</a>) to recognize this label and control cluster management accordingly.</p>
<h2>Modifying the Controller for Dual Version Support</h2>
<p>Both the existing (older) and the new ECK operator versions were modified to support this label. Functionally, we adapted the <code>IsUnmanaged</code> function and the main controller loop to:</p>
<ul>
<li>Check for the <code>eaas.search.mercari.in/desired-controller-version</code> label.</li>
<li>Skip reconciliation if the label is missing or if the label&#8217;s version does not match the operator&#8217;s build version.</li>
</ul>
<p><img src="https://storage.googleapis.com/prd-engineering-asset/2025/12/0d0484d5-controller-logic.png" alt="" /></p>
<p>Here&#8217;s the relevant code snippet:</p>
<pre><code class="language-go">const desiredECKControllerVersionLabel = &quot;eaas.search.mercari.in/desired-controller-version&quot;
func IsUnmanaged(ctx context.Context, object metav1.Object) bool {
managed, exists := object.GetAnnotations()[ManagedAnnotation]
if exists &amp;&amp; managed == &quot;false&quot; {
return true
}
desiredVersion, exists := object.GetLabels()[desiredECKControllerVersionLabel]
if !exists {
ulog.FromContext(ctx).Info(fmt.Sprintf(&quot;Object doesn&#039;t have %s label. Skipping reconciliation&quot;, desiredECKControllerVersionLabel), &quot;namespace&quot;, object.GetNamespace(), &quot;name&quot;, object.GetName())
return true
}
if desiredVersion != about.GetBuildInfo().Version {
ulog.FromContext(ctx).Info(
fmt.Sprintf(&quot;Object is not the target of this controller by %s label. Skipping reconciliation&quot;, desiredECKControllerVersionLabel),
&quot;desired_version&quot;, desiredVersion,
&quot;operator_version&quot;, about.GetBuildInfo().Version,
&quot;namespace&quot;, object.GetNamespace(),
&quot;name&quot;, object.GetName(),
)
return true
}
paused, exists := object.GetAnnotations()[LegacyPauseAnnoation]
if exists {
ulog.FromContext(ctx).Info(fmt.Sprintf(&quot;%s is deprecated, please use %s&quot;, LegacyPauseAnnoation, ManagedAnnotation), &quot;namespace&quot;, object.GetNamespace(), &quot;name&quot;, object.GetName())
}
return exists &amp;&amp; paused == &quot;true&quot;
}
</code></pre>
<h1>Handling Custom Resource Definitions (CRDs)</h1>
<p>The ECK operator defines a custom resource of <strong>Kind: Elasticsearch</strong>. This is a cluster-scoped resource, not a namespaced resource, so we cannot define two distinct versions of the CRD concurrently within the same cluster.</p>
<p>In this scenario, we rely on the backward compatibility of the CRD definition. It&#8217;s crucial to note that while CRDs are expected to be backward compatible, they may not be forward compatible. Backward compatibility ensures that older operator versions can work with newer CRD definitions. However, forward compatibility, which would mean newer operators can seamlessly work with older CRD definitions, is not guaranteed.</p>
<p>This implies that the latest version of the CRD must be deployed to the cluster when running two different versions of the ECK operator side-by-side. Failure to do so could lead to issues where the newer operator version cannot interpret newer CRD fields or configurations, resulting in deployment or operational errors. Therefore, before initiating an upgrade, ensuring the newest CRD version is applied is a critical prerequisite.</p>
<h1>Handling Validating Webhook</h1>
<p>ECK also defines a validating webhook, which validates the Elasticsearch manifests before they are applied to the cluster. When running two versions of the ECK operator concurrently, it is crucial to ensure that each operator version only validates the Elasticsearch clusters for which its <code>desired-controller-version</code> matches.</p>
<p>The default webhook configuration, without any restrictions, would mean that an Elasticsearch manifest could be validated by both versions of the operator. This poses a significant risk because newer operator versions might introduce new features or modifications to the validation logic. These changes could render validation performed by an older operator version incompatible or incorrect for the expectations of the newer operator, or vice versa. This discrepancy could potentially lead to deployment failures, configuration errors, or unexpected behavior.</p>
<p>Instead of modifying the controller logic itself, a simple object selector was added to the webhook configuration.</p>
<pre><code class="language-yaml"> objectSelector:
matchLabels:
eaas.search.mercari.in/controller-version: x.y.z</code></pre>
<p><img src="https://storage.googleapis.com/prd-engineering-asset/2025/12/324c7ad5-validating-webhook-e1745573742670.png" alt="" /></p>
<p>This <code>objectSelector</code> with <code>matchLabels</code> ensures that each ECK operator version only validates Elasticsearch manifests that have the corresponding <code>desired-controller-version</code>. By isolating the validation process based on the operator version, we prevent potential conflicts and ensure that manifests are only validated by the operator version that is expected to manage them.</p>
<h2>Leader Election for High Availability in ECK Operator Upgrades</h2>
<p>The ECK operator employs leader election to ensure high availability. Multiple instances of the operator can run concurrently, but only one acts as the active leader responsible for processing changes. This leader election mechanism relies on Kubernetes leases, specifically by acquiring a Kubernetes lease.</p>
<p>In a standard, in-place upgrade scenario, the ECK operator uses a constant Kubernetes lease named <code>elastic-operator-leader</code>. Regardless of the operator version, they all contend for this same lease. When an in-place upgrade occurs, the new operator version simply replaces the old and takes over this existing lease.<br />
The following diagram illustrates the leader election process during a standard in-place upgrade:</p>
<p><img src="https://storage.googleapis.com/prd-engineering-asset/2025/12/323c851d-default-eck-operator-leader-election-e1745573800171.png" alt="" /></p>
<p>However, the default lease strategy presents a challenge for our side-by-side upgrade approach. Since both the older and newer ECK operator versions would try to acquire the same elastic-operator-leader lease, it would result in contention and only one version of the operator could run at a given time. To facilitate our dual-version scenario, we needed a way to separate the leader election for each version.</p>
<p>To address this, we modified the ECK operator&#8217;s leader election logic to create distinct Kubernetes leases based on the operator&#8217;s version. This ensures that each operator version has its own separate leader election process, allowing them to run in high availability side-by-side without conflict.</p>
<p>We made changes to the LeaderElectionID in the <a href="https://github.com/elastic/cloud-on-k8s/blob/be88fb68c4638f4c18dc7fdea1d52c9b425f5b0b/cmd/manager/main.go#L569" title="ECK operator code">ECK operator code</a>. This ID now includes the operator&#8217;s version:</p>
<pre><code class="language-go">func GetLeaderElectionLeaseName() string {
buildInfo := about.GetBuildInfo()
k8sVersion := strings.ReplaceAll(buildInfo.Version, &quot;.&quot;, &quot;-&quot;)
leasePrefix := &quot;elastic-operator-leader-v&quot;
leaseName := fmt.Sprintf(&quot;%s%s&quot;, leasePrefix, k8sVersion)
return leaseName
}
LeaderElectionID: GetLeaderElectionLeaseName()
</code></pre>
<p>In essence, this change transforms the default elastic-operator-leader lease into version-specific leases, such as <code>elastic-operator-leader-v2-16-1</code> for version 2.16.1. With these versioned leases, each ECK operator instance will only participate in leader election with instances of the same version. The following diagram shows the leader election process with our side-by-side upgrade:</p>
<p><img src="https://storage.googleapis.com/prd-engineering-asset/2025/12/323c851d-default-eck-operator-leader-election-e1745573800171.png" alt="" /></p>
<h2>Testing Our Approach Thoroughly</h2>
<p>The Search Infrastructure Team at Mercari leverages three distinct environments to ensure the stability and safety of our infrastructure changes:</p>
<ul>
<li><strong>Laboratory Environment</strong>: This environment serves as a dedicated playground, allowing the infrastructure team to rigorously test changes without impacting the development environment. It&#8217;s our sandbox for experimentation and initial validation.</li>
<li><strong>Development Environment</strong>: This environment mirrors the production setup to a significant degree and is primarily used for Quality Assurance (QA) testing and the development of new features. This is where we validate changes under conditions closely resembling those in production.</li>
<li><strong>Production Environment</strong>: This is the live environment serving real user traffic, demanding the highest level of stability and reliability.</li>
</ul>
<p>Before any production deployment, changes are meticulously tested in both the laboratory and development environments. We conduct comprehensive testing to ensure both the older and newer versions of the ECK operator can coexist without conflicts. This includes verifying the labeling system, controller logic modifications, CRD handling, and validating webhook changes. We also perform thorough rollback tests to guarantee that we can quickly revert to the previous state if issues arise. This rigorous testing across multiple environments is crucial to minimizing risk in our high-stakes production environment.</p>
<h1>Rollout to Production: A Phased and Monitored Process</h1>
<p>Our production rollout follows a phased and closely monitored approach to minimize risk. This involves:</p>
<ol>
<li>Preparation: Verify CRDs and webhook configurations are compatible with the new operator version.</li>
<li>Labeling: Tag all Elasticsearch clusters with <code>eaas.search.mercari.in/controller-version</code> set to the current operator version for tracking.</li>
<li>Dual Deployment: Deploy both old and new ECK operators concurrently.</li>
<li>Gradual Rollout: Upgrade clusters incrementally by updating their labels to point to the new operator version (<code>eaas.search.mercari.in/controller-version=&lt;new_version&gt;</code>) cluster-by-cluster.</li>
<li>Continuous Monitoring: Track key metrics like error rates, system stability, and resource usage during each upgrade.</li>
<li>Validation &amp; Rollback: After each cluster upgrade, validate success or rollback by reverting labels and configurations if needed.</li>
<li>Completion: Upgrade remaining clusters, validate, and then remove the older operator version.</li>
</ol>
<p>The following diagram illustrates the workflow that we follow.</p>
<p><img src="https://storage.googleapis.com/prd-engineering-asset/2025/12/22f5ac94-final-workflow.png" alt="" /></p>
<h1>Conclusion</h1>
<p>In summary, upgrading critical systems like the ECK operator needs careful planning and testing. Mercari&#8217;s specific needs led us to create a unique side-by-side upgrade strategy. By carefully changing the operator and using a step-by-step release, we successfully reduced risks and kept our search system running smoothly.</p>
<p>It&#8217;s often hard to perfectly copy real-world workloads in testing environments. This can lead to bugs slipping through. This challenge highlights a limitation of the standard approach, as standard operator upgrades are usually tested in development before going to production all at once. </p>
<p>While Kubernetes applications use methods like gradual releases and canary deployments, operator upgrades often use an all-at-once method. We found this wasn&#8217;t ideal for our critical search infrastructure.</p>
<p>With our successful ECK operator upgrade using the side-by-side approach, we plan to use this strategy for other critical operator upgrades in our production system. We hope our approach helps other teams manage Kubernetes operators, especially those which handle stateful workloads.</p>
</div></div><!--astro:end--></astro-island> <div class="_contents_shareButtons_11e9j_18"> <astro-island uid="zr9XA" component-url="/_astro/ShareButtons.Dw0B3qRt.js" component-export="ShareButtons" renderer-url="/_astro/client.DNGgaD7G.js" props="{&quot;pathname&quot;:[0,&quot;/en/blog/entry/20250428-upgrading-eck-operator-a-side-by-side-kubernetes-operator-upgrade-approach/&quot;],&quot;title&quot;:[0,&quot;Upgrading ECK Operator: A Side-by-Side Kubernetes Operator Upgrade Approach&quot;],&quot;tags&quot;:[1,[[0,{&quot;name&quot;:[0,&quot;Elasticsearch&quot;],&quot;slug&quot;:[0,&quot;elasticsearch&quot;]}],[0,{&quot;name&quot;:[0,&quot;Infrastructure&quot;],&quot;slug&quot;:[0,&quot;infrastructure&quot;]}],[0,{&quot;name&quot;:[0,&quot;Kubernetes&quot;],&quot;slug&quot;:[0,&quot;kubernetes&quot;]}],[0,{&quot;name&quot;:[0,&quot;kubernetes-operator&quot;],&quot;slug&quot;:[0,&quot;kubernetes-operator&quot;]}],[0,{&quot;name&quot;:[0,&quot;sre&quot;],&quot;slug&quot;:[0,&quot;sre&quot;]}]]]}" ssr="" client="load" opts="{&quot;name&quot;:&quot;ShareButtons&quot;,&quot;value&quot;:true}" await-children=""><div class="_share-button_siy4t_18"><ul class="_share-button_list_siy4t_18"><li class="_share-button_item_siy4t_23"><a href="https://twitter.com/intent/tweet?url=https://engineering.mercari.com%0a&amp;text=Upgrading%20ECK%20Operator%3A%20A%20Side-by-Side%20Kubernetes%20Operator%20Upgrade%20Approach%20%7C%20Mercari%20Engineering&amp;via=MercariDev&amp;hashtags=Elasticsearch,sre%0a&amp;related=MercariDev" rel="noopener" target="_blank" class="_share-button_link_siy4t_27"><img src="/img/x.svg" class="_share-button_img_siy4t_42 _img_x_siy4t_54" alt="X"/></a></li><li class="_share-button_item_siy4t_23"><a href="https://www.facebook.com/sharer/sharer.php?u=https://engineering.mercari.com" rel="noopener" target="_blank" class="_share-button_link_siy4t_27"><img src="/img/share_icon_facebook.svg" class="_share-button_img_siy4t_42" alt="Facebook"/></a></li><li class="_share-button_item_siy4t_23"><a href="https://www.linkedin.com/sharing/share-offsite/?url=https://engineering.mercari.com" rel="noopener" target="_blank" class="_share-button_link_siy4t_27"><img src="/img/In-Blue-128@2x.png" class="_share-button_img_siy4t_42" alt="linkedin"/></a></li><li class="_share-button_item_siy4t_23"><a href="https://b.hatena.ne.jp/entry/" class="hatena-bookmark-button" data-hatena-bookmark-layout="basic-counter" data-hatena-bookmark-width="50" data-hatena-bookmark-height="30" title="このエントリーをはてなブックマークに追加"><img src="https://b.st-hatena.com/images/v4/public/entry-button/button-only@2x.png" alt="このエントリーをはてなブックマークに追加" width="20" height="20" style="border:none;"/></a></li></ul></div><!--astro:end--></astro-island> </div> <ul class="_tagList_wl3e1_18"><li class="_tagList_item_wl3e1_22"><a href="/en/blog/tag/elasticsearch/"><svg xmlns="http://www.w3.org/2000/svg" width="25.027" height="25.027" viewBox="0 0 25.027 25.027" class="_icon_tag_wl3e1_29"><path id="パス_1041" data-name="パス 1041" d="M517.478,303.493l-4.679-4.679a.669.669,0,0,0-.942,0l-4.678,4.679a1.854,1.854,0,0,0-.472,1.138v17.475a.669.669,0,0,0,.667.666h9.909a.668.668,0,0,0,.666-.666V304.631A1.853,1.853,0,0,0,517.478,303.493Zm-5.15,4.168a2.347,2.347,0,1,1,2.347-2.347A2.348,2.348,0,0,1,512.328,307.661Z" transform="translate(-130.062 -569.453) rotate(45)" fill="currentColor"></path></svg><span><div>Elasticsearch</div></span></a></li><li class="_tagList_item_wl3e1_22"><a href="/en/blog/tag/infrastructure/"><svg xmlns="http://www.w3.org/2000/svg" width="25.027" height="25.027" viewBox="0 0 25.027 25.027" class="_icon_tag_wl3e1_29"><path id="パス_1041" data-name="パス 1041" d="M517.478,303.493l-4.679-4.679a.669.669,0,0,0-.942,0l-4.678,4.679a1.854,1.854,0,0,0-.472,1.138v17.475a.669.669,0,0,0,.667.666h9.909a.668.668,0,0,0,.666-.666V304.631A1.853,1.853,0,0,0,517.478,303.493Zm-5.15,4.168a2.347,2.347,0,1,1,2.347-2.347A2.348,2.348,0,0,1,512.328,307.661Z" transform="translate(-130.062 -569.453) rotate(45)" fill="currentColor"></path></svg><span><div>Infrastructure</div></span></a></li><li class="_tagList_item_wl3e1_22"><a href="/en/blog/tag/kubernetes/"><svg xmlns="http://www.w3.org/2000/svg" width="25.027" height="25.027" viewBox="0 0 25.027 25.027" class="_icon_tag_wl3e1_29"><path id="パス_1041" data-name="パス 1041" d="M517.478,303.493l-4.679-4.679a.669.669,0,0,0-.942,0l-4.678,4.679a1.854,1.854,0,0,0-.472,1.138v17.475a.669.669,0,0,0,.667.666h9.909a.668.668,0,0,0,.666-.666V304.631A1.853,1.853,0,0,0,517.478,303.493Zm-5.15,4.168a2.347,2.347,0,1,1,2.347-2.347A2.348,2.348,0,0,1,512.328,307.661Z" transform="translate(-130.062 -569.453) rotate(45)" fill="currentColor"></path></svg><span><div>Kubernetes</div></span></a></li><li class="_tagList_item_wl3e1_22"><a href="/en/blog/tag/kubernetes-operator/"><svg xmlns="http://www.w3.org/2000/svg" width="25.027" height="25.027" viewBox="0 0 25.027 25.027" class="_icon_tag_wl3e1_29"><path id="パス_1041" data-name="パス 1041" d="M517.478,303.493l-4.679-4.679a.669.669,0,0,0-.942,0l-4.678,4.679a1.854,1.854,0,0,0-.472,1.138v17.475a.669.669,0,0,0,.667.666h9.909a.668.668,0,0,0,.666-.666V304.631A1.853,1.853,0,0,0,517.478,303.493Zm-5.15,4.168a2.347,2.347,0,1,1,2.347-2.347A2.348,2.348,0,0,1,512.328,307.661Z" transform="translate(-130.062 -569.453) rotate(45)" fill="currentColor"></path></svg><span><div>kubernetes-operator</div></span></a></li><li class="_tagList_item_wl3e1_22"><a href="/en/blog/tag/sre/"><svg xmlns="http://www.w3.org/2000/svg" width="25.027" height="25.027" viewBox="0 0 25.027 25.027" class="_icon_tag_wl3e1_29"><path id="パス_1041" data-name="パス 1041" d="M517.478,303.493l-4.679-4.679a.669.669,0,0,0-.942,0l-4.678,4.679a1.854,1.854,0,0,0-.472,1.138v17.475a.669.669,0,0,0,.667.666h9.909a.668.668,0,0,0,.666-.666V304.631A1.853,1.853,0,0,0,517.478,303.493Zm-5.15,4.168a2.347,2.347,0,1,1,2.347-2.347A2.348,2.348,0,0,1,512.328,307.661Z" transform="translate(-130.062 -569.453) rotate(45)" fill="currentColor"></path></svg><span><div>sre</div></span></a></li></ul> </div> </section> <section class="_relatedPosts_1e0te_18"><div class="container"><h2>Related article</h2><ul class="_post-list_1e0te_25"><li class="_post-list__item_1e0te_28"><a href="/en/blog/entry/20260401-legacy-image-provider-to-cloudflare-images-traffic-estimation-and-safe-rollout/" class="_post-list__link_1e0te_35"><div class="_card_p1it6_19 false"><time datetime="2026-04-02">2026/04/02</time><div class="_post_p1it6_28"><img class="_post__thumbnail_p1it6_82" src="https://storage.googleapis.com/prd-engineering-asset/2026/04/834345d7-cloudflare-images-migration-flow.png" alt="Legacy Image Provider to Cloudflare Images: Traffic Estimation and Safe Rollout"/><div class="_post__text_p1it6_93"><h3 class="_post__title_p1it6_105">Legacy Image Provider to Cloudflare Images: Traffic Estimation and Safe Rollout</h3><div class="_post__meta_p1it6_117"><div class="_meta-wrap_p1it6_148"><ul class="_tag-list_d76db_18 tag-list"><li class="_tag-list__item_d76db_29"><p>Infrastructure</p></li></ul><p class="_author_p1it6_168">Author:<span class="_author__name_p1it6_176"> boyu</span></p></div><div class="_post__to-post_p1it6_128"><div class="_post__link_p1it6_28"><img src="/img/arrow.svg" width="14" height="9" alt="To post"/></div></div></div></div></div></div></a></li><li class="_post-list__item_1e0te_28"><a href="/en/blog/entry/20260216-turborepo-remote-cache-accelerating-ci-to-move-fast/" class="_post-list__link_1e0te_35"><div class="_card_p1it6_19 false"><time datetime="2026-02-17">2026/02/17</time><div class="_post_p1it6_28"><img class="_post__thumbnail_p1it6_82" src="https://storage.googleapis.com/prd-engineering-asset/2026/02/760bc731-remote-cache-featured-image.jpg" alt="Turborepo Remote Cache: Accelerating CI to “Move Fast”"/><div class="_post__text_p1it6_93"><h3 class="_post__title_p1it6_105">Turborepo Remote Cache: Accelerating CI to “Move Fast”</h3><div class="_post__meta_p1it6_117"><div class="_meta-wrap_p1it6_148"><ul class="_tag-list_d76db_18 tag-list"><li class="_tag-list__item_d76db_29"><p>Development</p></li><li class="_tag-list__item_d76db_29"><p>Infrastructure</p></li></ul><p class="_author_p1it6_168">Author:<span class="_author__name_p1it6_176"> Zuma</span></p></div><div class="_post__to-post_p1it6_128"><div class="_post__link_p1it6_28"><img src="/img/arrow.svg" width="14" height="9" alt="To post"/></div></div></div></div></div></div></a></li><li class="_post-list__item_1e0te_28"><a href="/en/blog/entry/20251218-capturing-network-packets-in-kubernetes/" class="_post-list__link_1e0te_35"><div class="_card_p1it6_19 false"><time datetime="2025-12-18">2025/12/18</time><div class="_post_p1it6_28"><img class="_post__thumbnail_p1it6_82" src="https://storage.googleapis.com/prd-engineering-asset/2025/12/0d77977d-adventcalendar2025_mercari.png" alt="Capturing Network Packets in Kubernetes"/><div class="_post__text_p1it6_93"><h3 class="_post__title_p1it6_105">Capturing Network Packets in Kubernetes</h3><div class="_post__meta_p1it6_117"><div class="_meta-wrap_p1it6_148"><ul class="_tag-list_d76db_18 tag-list"><li class="_tag-list__item_d76db_29"><p>Infrastructure</p></li></ul><p class="_author_p1it6_168">Author:<span class="_author__name_p1it6_176"> mshibuya</span></p></div><div class="_post__to-post_p1it6_128"><div class="_post__link_p1it6_28"><img src="/img/arrow.svg" width="14" height="9" alt="To post"/></div></div></div></div></div></div></a></li></ul></div></section> <script>(()=>{var e=async t=>{await(await t())()};(self.Astro||(self.Astro={})).only=e;window.dispatchEvent(new Event("astro:only"));})();</script><astro-island uid="gmbUv" component-url="/_astro/ButtonToTop.DpQxT0b_.js" component-export="ButtonToTop" renderer-url="/_astro/client.DNGgaD7G.js" props="{}" ssr="" client="only" opts="{&quot;name&quot;:&quot;ButtonToTop&quot;,&quot;value&quot;:&quot;preact&quot;}"></astro-island> </main> <footer id="the-footer" class="_footer_zthyf_19"> <div class="container"> <ul class="_navigation_zthyf_32"> <li class="_navigation__item_zthyf_43 undefined"> <div class="_menu__heading_zthyf_46">Company</div> <ul> <li class="_menu__item_zthyf_60"> <a href="https://about.mercari.com/en/" rel="noopener" target="_blank"> Mercari, Inc </a> </li><li class="_menu__item_zthyf_60"> <a href="https://en.merpay.com/" rel="noopener" target="_blank"> Merpay, Inc </a> </li><li class="_menu__item_zthyf_60"> <a href="https://www.mercari.com/about/" rel="noopener" target="_blank"> Mercari US </a> </li><li class="_menu__item_zthyf_60"> <a href="https://about.mercoin.com/en/" rel="noopener" target="_blank"> Mercoin </a> </li><li class="_menu__item_zthyf_60"> <a href="https://about.in.mercari.com/" rel="noopener" target="_blank"> Mercari India </a> </li> </ul> </li><li class="_navigation__item_zthyf_43 undefined"> <div class="_menu__heading_zthyf_46">Owned media</div> <ul> <li class="_menu__item_zthyf_60"> <a href="https://mercan.mercari.com/en/" rel="noopener" target="_blank"> mercan </a> </li><li class="_menu__item_zthyf_60"> <a href="https://note.com/mercari_design/" rel="noopener" target="_blank"> Design blog </a> </li><li class="_menu__item_zthyf_60"> <a href="https://ai.mercari.com/en/" rel="noopener" target="_blank"> AI Web </a> </li><li class="_menu__item_zthyf_60"> <a href="https://note.com/mercari_data" rel="noopener" target="_blank"> Analytics blog </a> </li><li class="_menu__item_zthyf_60"> <a href="https://r4d.mercari.com/en/" rel="noopener" target="_blank"> mercari R4D </a> </li><li class="_menu__item_zthyf_60"> <a href="https://blog.mercari.com/making-mercari/home" rel="noopener" target="_blank"> Engineering blog (USA) </a> </li> </ul> </li><li class="_navigation__item_zthyf_43 undefined"> <div class="_menu__heading_zthyf_46">Careers</div> <ul> <li class="_menu__item_zthyf_60"> <a href="https://careers.mercari.com/job-categories/engineering/" rel="noopener" target="_blank"> Engineering </a> </li><li class="_menu__item_zthyf_60"> <a href="https://careers.mercari.com/" rel="noopener" target="_blank"> Mercari (JP) </a> </li><li class="_menu__item_zthyf_60"> <a href="https://www.mercari.com/careers/" rel="noopener" target="_blank"> Mercari (USA) </a> </li><li class="_menu__item_zthyf_60"> <a href="https://about.in.mercari.com/join-us/" rel="noopener" target="_blank"> Mercari (India) </a> </li> </ul> </li><li class="_navigation__item_zthyf_43 undefined"> <div class="_menu__heading_zthyf_46">Resources</div> <ul> <li class="_menu__item_zthyf_60"> <a href="https://github.com/mercari" rel="noopener" target="_blank"> GitHub - mercari </a> </li><li class="_menu__item_zthyf_60"> <a href="https://speakerdeck.com/mercari" rel="noopener" target="_blank"> Speakerdeck - mercari </a> </li> </ul> </li><li class="_navigation__item_zthyf_43 undefined"> <div class="_menu__heading_zthyf_46">Events</div> <ul> <li class="_menu__item_zthyf_60"> <a href="https://mercari.connpass.com/" rel="noopener" target="_blank"> connpass - Mercari </a> </li><li class="_menu__item_zthyf_60"> <a href="https://www.meetup.com/ja-JP/MercariDev/" rel="noopener" target="_blank"> Meetup.com </a> </li> </ul> </li> </ul> <div class="_brand_zthyf_98"> <a href="/en/"> <img src="/img/mercari_white.svg" alt="mercari" width="140" height="29"> </a> </div> <small class="_copyright_zthyf_108"> <span class="_copyright__mark_zthyf_121">&copy;</span>2020 Mercari, Inc.
</small> </div> </footer> <script>(function(){const pageViewEvent = {"routeName":"/en/blog/entry/20250428-upgrading-eck-operator-a-side-by-side-kubernetes-operator-upgrade-approach/","pageType":"PageView","pageUrl":"/en/blog/entry/20250428-upgrading-eck-operator-a-side-by-side-kubernetes-operator-upgrade-approach/","event":"page_view"};
/**
* `@nuxtjs/gtm` から移行
* @see https://github.com/nuxt-community/gtm-module/blob/ae9199e6cab39485bcc85758bdfb36ebace3cdcf/lib/plugin.mock.js#L8-L20
*/
dataLayer.push({ ...pageViewEvent, pageTitle: document.title });
})();</script> </body> </html> <script>(function(){const contentGroup = {"Categories":"Infrastructure","Tags":"Elasticsearch,Infrastructure,Kubernetes,kubernetes-operator,sre","Authors":"Abhishek Munagekar"};
// コンテンツグループをセット(GA4)
dataLayer.push(contentGroup);
})();</script>