642 lines
107 KiB
HTML
642 lines
107 KiB
HTML
<html>
|
|
<head>
|
|
|
|
<title>How we built observability with Google Cloud services</title>
|
|
|
|
|
|
|
|
<link rel="canonical" href="https://punits.dev/blog/building-observability-with-google-cloud-services/" />
|
|
|
|
|
|
|
|
|
|
<meta name="og:title" content="How we built observability with Google Cloud services">
|
|
<meta name="twitter:title" content="How we built observability with Google Cloud services">
|
|
|
|
|
|
<meta name="twitter:description" content="Beyond the easy-to-reproduce bugs, there are some production issues (bugs, slowness, crashes) that are not easy to root cause. This post details how we setup observability within our production setup make it easier for us to root cause and quickly decipher the impact of such issues.">
|
|
<meta name="description" content="Beyond the easy-to-reproduce bugs, there are some production issues (bugs, slowness, crashes) that are not easy to root cause. This post details how we setup observability within our production setup make it easier for us to root cause and quickly decipher the impact of such issues.">
|
|
<meta name="og:description" content="Beyond the easy-to-reproduce bugs, there are some production issues (bugs, slowness, crashes) that are not easy to root cause. This post details how we setup observability within our production setup make it easier for us to root cause and quickly decipher the impact of such issues.">
|
|
|
|
|
|
<meta name="twitter:card" content="summary_large_image">
|
|
<meta name="twitter:image" content="https://punits.dev/images/observability-v3.png">
|
|
<meta property="og:image" content="https://punits.dev/images/observability-v3.png">
|
|
|
|
|
|
<meta http-equiv="Content-Type" content="text/html" charset="UTF-8" />
|
|
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
|
|
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">
|
|
<link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png">
|
|
<link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png">
|
|
<link rel="manifest" href="/site.webmanifest">
|
|
<link rel="mask-icon" href="/safari-pinned-tab.svg" color="#5bbad5">
|
|
<meta name="msapplication-TileColor" content="#da532c">
|
|
<meta name="theme-color" content="#ffffff">
|
|
|
|
|
|
|
|
|
|
|
|
<style>
|
|
body.content pre.chroma { color: #f8f8f2; background-color: #5b5463 }
|
|
.chroma { color: #f8f8f2; background-color: #5b5463 }
|
|
.chroma .x { }
|
|
.chroma .err { color: #ffbd9c; background-color: #5b5463 }
|
|
.chroma .lntd { vertical-align: top; padding: 0; margin: 0; border: 0; }
|
|
.chroma .lntable { border-spacing: 0; padding: 0; margin: 0; border: 0; width: auto; overflow: auto; display: block; }
|
|
.chroma .hl { display: block; width: 100%;background-color: #ffffcc }
|
|
.chroma .lnt { margin-right: 0.4em; padding: 0 0.4em 0 0.4em;color: #7f7f7f }
|
|
.chroma .ln { margin-right: 0.4em; padding: 0 0.4em 0 0.4em;color: #7f7f7f }
|
|
.chroma .k { color: #96ffea }
|
|
.chroma .kc { color: #96ffea }
|
|
.chroma .kd { color: #96ffea }
|
|
.chroma .kn { color: #ffa2ee }
|
|
.chroma .kp { color: #96ffea }
|
|
.chroma .kr { color: #96ffea }
|
|
.chroma .kt { color: #96ffea }
|
|
.chroma .n { }
|
|
.chroma .na { color: #b6ffff }
|
|
.chroma .nb { }
|
|
.chroma .bp { }
|
|
.chroma .nc { color: #b6ffff }
|
|
.chroma .no { color: #96ffea }
|
|
.chroma .nd { color: #b6ffff }
|
|
.chroma .ni { }
|
|
.chroma .ne { color: #b6ffff }
|
|
.chroma .nf { color: #b6ffff }
|
|
.chroma .fm { }
|
|
.chroma .nl { }
|
|
.chroma .nn { }
|
|
.chroma .nx { color: #b6ffff }
|
|
.chroma .py { }
|
|
.chroma .nt { color: #ffa2ee }
|
|
.chroma .nv { }
|
|
.chroma .vc { }
|
|
.chroma .vg { }
|
|
.chroma .vi { }
|
|
.chroma .vm { }
|
|
.chroma .l { color: #dbc7ff }
|
|
.chroma .ld { color: #faf087 }
|
|
.chroma .s { color: #faf087 }
|
|
.chroma .sa { color: #faf087 }
|
|
.chroma .sb { color: #faf087 }
|
|
.chroma .sc { color: #faf087 }
|
|
.chroma .dl { color: #faf087 }
|
|
.chroma .sd { color: #faf087 }
|
|
.chroma .s2 { color: #faf087 }
|
|
.chroma .se { color: #dbc7ff }
|
|
.chroma .sh { color: #faf087 }
|
|
.chroma .si { color: #faf087 }
|
|
.chroma .sx { color: #faf087 }
|
|
.chroma .sr { color: #faf087 }
|
|
.chroma .s1 { color: #faf087 }
|
|
.chroma .ss { color: #faf087 }
|
|
.chroma .m { color: #dbc7ff }
|
|
.chroma .mb { color: #dbc7ff }
|
|
.chroma .mf { color: #dbc7ff }
|
|
.chroma .mh { color: #dbc7ff }
|
|
.chroma .mi { color: #dbc7ff }
|
|
.chroma .il { color: #dbc7ff }
|
|
.chroma .mo { color: #dbc7ff }
|
|
.chroma .o { color: #ffa2ee }
|
|
.chroma .ow { color: #ffa2ee }
|
|
.chroma .p { }
|
|
.chroma .c { color: #e6e0c3 }
|
|
.chroma .ch { color: #e6e0c3 }
|
|
.chroma .cm { color: #e6e0c3 }
|
|
.chroma .c1 { color: #e6e0c3 }
|
|
.chroma .cs { color: #e6e0c3 }
|
|
.chroma .cp { color: #e6e0c3 }
|
|
.chroma .cpf { color: #e6e0c3 }
|
|
.chroma .g { }
|
|
.chroma .gd { color: #ffa2ee }
|
|
.chroma .ge { font-style: italic }
|
|
.chroma .gr { }
|
|
.chroma .gh { }
|
|
.chroma .gi { color: #b6ffff }
|
|
.chroma .go { }
|
|
.chroma .gp { }
|
|
.chroma .gs { font-weight: bold }
|
|
.chroma .gu { color: #e6e0c3 }
|
|
.chroma .gt { }
|
|
.chroma .gl { }
|
|
.chroma .w { }
|
|
</style>
|
|
|
|
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="/css/style.min.c02ee42c05d9b9ed56ca1e01135dc11b57d0911d44b6a4ed3be15e9b667287fd.css" integrity="sha256-wC7kLAXZue1Wyh4BE13BG1fQkR1EtqTtO+Fem2Zyh/0="/>
|
|
|
|
<script type="text/javascript">
|
|
;window.NREUM||(NREUM={});NREUM.init={browser_consent_mode:{enabled:false},privacy:{cookies_enabled:true},performance:{capture_measures:true},ajax:{deny_list:["bam.nr-data.net"],capture_payloads:'none'}};
|
|
|
|
;NREUM.loader_config={accountID:"8345149",trustKey:"8345149",agentID:"1103527911",licenseKey:"NRJS-fc79124a923e14ec04a",applicationID:"1103527911"};
|
|
;NREUM.info={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net",licenseKey:"NRJS-fc79124a923e14ec04a",applicationID:"1103527911",sa:1};
|
|
;
|
|
(()=>{var e={8122(e,t,r){"use strict";r.d(t,{a:()=>s});var n=r(944);function s(e,t){try{if(!e||"object"!=typeof e)return(0,n.R)(3);if(!t||"object"!=typeof t)return(0,n.R)(4);const r=Object.create(Object.getPrototypeOf(t),Object.getOwnPropertyDescriptors(t)),i=0===Object.keys(r).length?e:r;for(let o in i)if(void 0!==e[o])try{if(null===e[o]){r[o]=null;continue}Array.isArray(e[o])&&Array.isArray(t[o])?r[o]=Array.from(new Set([...e[o],...t[o]])):e[o]instanceof Map||e[o]instanceof Set||e[o]instanceof Date||e[o]instanceof RegExp?r[o]=e[o]:"object"==typeof e[o]&&null!==t[o]&&"object"==typeof t[o]?r[o]=s(e[o],t[o]):r[o]=e[o]}catch(e){r[o]||(0,n.R)(1,e)}return r}catch(e){(0,n.R)(2,e)}}},2555(e,t,r){"use strict";r.d(t,{f:()=>o});var n=r(384),s=r(8122);const i={beacon:n.NT.beacon,errorBeacon:n.NT.errorBeacon,licenseKey:void 0,applicationID:void 0,sa:void 0,queueTime:void 0,applicationTime:void 0,ttGuid:void 0,user:void 0,account:void 0,product:void 0,extra:void 0,jsAttributes:{},userAttributes:void 0,atts:void 0,transactionName:void 0,tNamePlain:void 0};function o(e){try{return!!e.licenseKey&&!!e.errorBeacon&&!!e.applicationID}catch(e){return!1}}r.d(t,["D",0,e=>(0,s.a)(e,i)])},7699(e,t,r){"use strict";var n=r(860);const s={[n.K7.logging]:!0,[n.K7.genericEvents]:!0,[n.K7.jserrors]:!0,[n.K7.ajax]:!0};r.d(t,["It",0,1e6,"KC",0,s,"qh",0,"SESSION_ERROR"])},9324(e,t,r){"use strict";r.d(t,["A",0,"@newrelic/rrweb","x",0,"1.319.0"])},981(e,t,r){"use strict";r.d(t,["R",0,{AJAX:"AjaxRequest",PA:"PageAction",UA:"UserAction",BP:"BrowserPerformance",WS:"WebSocket",SPV:"SecurityPolicyViolation",JSE:"JavaScriptError",LOG:"Log",PVE:"PageView",PVT:"PageViewTiming",SR:"SessionReplay",ST:"SessionTrace",BI:"BrowserInteraction"}])},6154(e,t,r){"use strict";var n=r(1863);const s="undefined"!=typeof window&&!!window.document,i="undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self.navigator instanceof WorkerNavigator||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis.navigator instanceof WorkerNavigator),o=s?window:"undefined"!=typeof WorkerGlobalScope&&("undefined"!=typeof self&&self instanceof WorkerGlobalScope&&self||"undefined"!=typeof globalThis&&globalThis instanceof WorkerGlobalScope&&globalThis),a=Boolean("hidden"===o?.document?.visibilityState),c=""+o?.location,d=/iPad|iPhone|iPod/.test(o.navigator?.userAgent),u=d&&"undefined"==typeof SharedWorker,l=(()=>{const e=o.navigator?.userAgent?.match(/Firefox[/\s](\d+\.\d+)/);return Array.isArray(e)&&e.length>=2?+e[1]:0})(),f=Date.now()-(0,n.t)();r.d(t,["OF",0,d,"RI",0,s,"WN",0,f,"bv",0,i,"gm",0,o,"lR",0,l,"m",0,c,"mw",0,a,"sb",0,u,"zk",0,()=>{const e=o?.performance?.getEntriesByType?.("navigation")?.[0];if(e&&e.responseStart>0&&e.responseStart<o.performance.now())return e}])},7295(e,t,r){"use strict";r.d(t,{Xv:()=>o,gX:()=>s,iW:()=>i});var n=[];function s(e){if(!e||i(e))return!1;if(0===n.length)return!0;if("*"===n[0].hostname)return!1;for(var t=0;t<n.length;t++){var r=n[t];if(r.hostname.test(e.hostname)&&r.pathname.test(e.pathname))return!1}return!0}function i(e){return void 0===e.hostname}function o(e){if(n=[],e&&e.length)for(var t=0;t<e.length;t++){let r=e[t];if(!r)continue;if("*"===r)return void(n=[{hostname:"*"}]);0===r.indexOf("http://")?r=r.substring(7):0===r.indexOf("https://")&&(r=r.substring(8));const s=r.indexOf("/");let i,o;s>0?(i=r.substring(0,s),o=r.substring(s)):(i=r,o="*");let[c]=i.split(":");n.push({hostname:a(c),pathname:a(o,!0)})}}function a(e,t=!1){const r=e.replace(/[.+?^${}()|[\]\\]/g,e=>"\\"+e).replace(/\*/g,".*?");return new RegExp((t?"^":"")+r+"$")}},3241(e,t,r){"use strict";r.d(t,{W:()=>s});var n=r(6154);function s(e={}){try{n.gm.dispatchEvent(new CustomEvent("newrelic",{detail:e}))}catch(e){}}},1687(e,t,r){"use strict";r.d(t,{Ak:()=>a,Ze:()=>d,x3:()=>c});var n=r(3241),s=r(3606),i=r(860),o=r(2646);function a(e,t){if(!e)return;const r={staged:!1,priority:i.P3[t]||0};e.runtime.drainRegistry.get(t)||e.runtime.drainRegistry.set(t,r)}function c(e,t){if(!e)return;const r=e.runtime.drainRegistry;r&&(r.get(t)&&r.delete(t),l(e,t,!1),r.size&&u(e))}function d(e,t="feature",r=!1){if(e){if(!e.runtime.drainRegistry.get(t)||r)return l(e,t);e.runtime.drainRegistry.get(t).staged=!0,u(e)}}function u(e){if(!e)return;const t=Array.from(e.runtime.drainRegistry);t.every(([e,t])=>t.staged)&&(t.sort((e,t)=>e[1].priority-t[1].priority),t.forEach(([t])=>{e.runtime.drainRegistry.delete(t),l(e,t)}))}function l(e,t,r=!0){if(!e)return;const i=e.ee,a=s.i.handlers;if(i&&!i.aborted&&i.backlog&&a){if((0,n.W)({type:"lifecycle",name:"drain",feature:t}),r){const e=i.backlog[t],r=a[t];if(r){for(let t=0;e&&t<e.length;++t)f(e[t],r);Object.entries(r).forEach(([e,t])=>{Object.values(t||{}).forEach(t=>{t[0]?.on&&t[0].context()instanceof o.y&&!t[0].listeners(e).includes(t[1])&&t[0].on(e,t[1])})})}}i.isolatedBacklog||delete a[t],i.backlog[t]=null,i.emit("drain-"+t,[])}}function f(e,t){var r=e[1];Object.values(t[r]||{}).forEach(t=>{var r=e[0];if(t[0]===r){var n=t[1],s=e[3],i=e[2];n.apply(s,i)}})}},7836(e,t,r){"use strict";var n=r(384),s=r(8990),i=r(2646),o=r(5607);const a="nr@context:".concat(o.W),c=function e(t,r){var n={},o={},u={},l=!1;try{l=16===r.length&&d.initializedAgents?.[r]?.runtime.isolatedBacklog}catch(e){}var f={on:p,addEventListener:p,removeEventListener:function(e,t){var r=n[e];if(!r)return;for(var s=0;s<r.length;s++)r[s]===t&&r.splice(s,1)},emit:function(e,r,n,s,i){!1!==i&&(i=!0);if(c.aborted&&!s)return;t&&i&&t.emit(e,r,n);var a=h(n);g(e).forEach(e=>{e.apply(a,r)});var d=v()[o[e]];d&&d.push([f,e,r,a]);return a},get:m,listeners:g,context:h,buffer:function(e,t){const r=v();if(t=t||"feature",f.aborted)return;Object.entries(e||{}).forEach(([e,n])=>{o[n]=t,t in r||(r[t]=[])})},abort:function(){f._aborted=!0,Object.keys(f.backlog).forEach(e=>{delete f.backlog[e]})},isBuffering:function(e){return!!v()[o[e]]},debugId:r,backlog:l?{}:t&&"object"==typeof t.backlog?t.backlog:{},isolatedBacklog:l};return Object.defineProperty(f,"aborted",{get:()=>{let e=f._aborted||!1;return e||(t&&(e=t.aborted),e)}}),f;function h(e){return e&&e instanceof i.y?e:e?(0,s.I)(e,a,()=>new i.y(a)):new i.y(a)}function p(e,t){n[e]=g(e).concat(t)}function g(e){return n[e]||[]}function m(t){return u[t]=u[t]||e(f,t)}function v(){return f.backlog}}(void 0,"globalEE"),d=(0,n.Zm)();d.ee||(d.ee=c),r.d(t,["P",0,a,"ee",0,c])},2646(e,t,r){"use strict";r.d(t,{y:()=>n});class n{constructor(e){this.contextId=e}}},9908(e,t,r){"use strict";r.d(t,{d:()=>n,p:()=>s});var n=r(7836).ee.get("handle");function s(e,t,r,s,i){i?(i.buffer([e],s),i.emit(e,t,r)):(n.buffer([e],s),n.emit(e,t,r))}},3606(e,t,r){"use strict";r.d(t,{i:()=>i});var n=r(9908);i.on=o;var s=i.handlers={};function i(e,t,r,i){o(i||n.d,s,e,t,r)}function o(e,t,r,s,i){i||(i="feature"),e||(e=n.d);var o=t[i]=t[i]||{};(o[r]=o[r]||[]).push([e,s])}},3878(e,t,r){"use strict";function n(e,t){return{capture:e,passive:!1,signal:t}}function s(e,t,r=!1,s){window.addEventListener(e,t,n(r,s))}function i(e,t,r=!1,s){document.addEventListener(e,t,n(r,s))}r.d(t,{DD:()=>i,jT:()=>n,sp:()=>s})},5607(e,t,r){"use strict";const n=(0,r(9566).bz)();r.d(t,["W",0,n])},9566(e,t,r){"use strict";r.d(t,{LA:()=>o,ZF:()=>a,bz:()=>i,el:()=>c});var n=r(6154);function s(e,t){return e?15&e[t]:16*Math.random()|0}function i(){const e=n.gm?.crypto||n.gm?.msCrypto;let t,r=0;return e&&e.getRandomValues&&(t=e.getRandomValues(new Uint8Array(30))),"xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".split("").map(e=>"x"===e?s(t,r++).toString(16):"y"===e?(3&s()|8).toString(16):e).join("")}function o(e){const t=n.gm?.crypto||n.gm?.msCrypto;let r,i=0;t&&t.getRandomValues&&(r=t.getRandomValues(new Uint8Array(e)));const o=[];for(var a=0;a<e;a++)o.push(s(r,i++).toString(16));return o.join("")}function a(){return o(16)}function c(){return o(32)}},2614(e,t,r){"use strict";r.d(t,["BB",0,18e5,"Wt",0,"NRBA_SESSION::","g",0,{OFF:0,FULL:1,ERROR:2},"iL",0,{SAME_TAB:"same-tab",CROSS_TAB:"cross-tab"},"tS",0,{STARTED:"session-started",PAUSE:"session-pause",RESET:"session-reset",RESUME:"session-resume",UPDATE:"session-update"},"wk",0,144e5])},733(e,t,r){"use strict";function n(e,t){return function(e){let t=2166136261;for(let r=0;r<e.length;r++)t^=e.charCodeAt(r),t=Math.imul(t,16777619);return(t>>>0).toString(16).padStart(8,"0")}("".concat(String(e),":").concat(String(t)))}r.d(t,{Y:()=>n})},1863(e,t,r){"use strict";function n(){return Math.floor(performance.now())}r.d(t,{t:()=>n})},9119(e,t,r){"use strict";r.d(t,{L:()=>i});var n=/([^?#]*)[^#]*(#[^?]*|$).*/,s=/([^?#]*)().*/;function i(e,t){return e?e.replace(t?n:s,"$1$2"):e}},7485(e,t,r){"use strict";r.d(t,{D:()=>s});var n=r(6154);function s(e){if(0===(e||"").indexOf("data:"))return{protocol:"data"};try{const t=new URL(e,location.href),r={port:t.port,hostname:t.hostname,pathname:t.pathname,search:t.search,protocol:t.protocol.slice(0,t.protocol.indexOf(":")),sameOrigin:t.protocol===n.gm?.location?.protocol&&t.host===n.gm?.location?.host};return r.port&&""!==r.port||("http:"===t.protocol&&(r.port="80"),"https:"===t.protocol&&(r.port="443")),r.pathname&&""!==r.pathname?r.pathname.startsWith("/")||(r.pathname="/".concat(r.pathname)):r.pathname="/",r}catch(e){return{}}}},7866(e,t,r){"use strict";r.d(t,["Nc",0,/^\s*at Function code \(Function code:\d+:\d+\)\s*/i,"cn",0,/^\s*(?:([^@]*)(?:\(.*?\))?@)?((?:file|http|https|chrome|safari-extension).*?):(\d+)(?::(\d+))?\s*$/i,"fL",0,/^\s*at .+ \(eval at \S+ \((?:(?:file|http|https):[^)]+)?\)(?:, [^:]*:\d+:\d+)?\)$/i,"h3",0,/function (.+?)\s*\(/,"hB",0,/^\s*at (?:((?:\[object object\])?(?:[^(]*\([^)]*\))*[^()]*(?: \[as \S+\])?) )?\(?((?:file|http|https|chrome-extension):.*?)?:(\d+)(?::(\d+))?\)?\s*$/i])},944(e,t,r){"use strict";r.d(t,{R:()=>s});var n=r(3241);function s(e,t){"function"==typeof console.debug&&(console.debug("New Relic Warning: https://github.com/newrelic/newrelic-browser-agent/blob/main/docs/warning-codes.md#".concat(e),t),(0,n.W)({drained:null,type:"data",name:"warn",feature:"warn",data:{code:e,secondary:t}}))}},8990(e,t,r){"use strict";r.d(t,{I:()=>s});var n=Object.prototype.hasOwnProperty;function s(e,t,r){if(n.call(e,t))return e[t];var s=r();if(Object.defineProperty&&Object.keys)try{return Object.defineProperty(e,t,{value:s,writable:!0,enumerable:!1}),s}catch(e){}return e[t]=s,s}},6389(e,t,r){"use strict";function n(e,t=500,r={}){const n=r?.leading||!1;let s;return(...r)=>{n&&void 0===s&&(e.apply(this,r),s=setTimeout(()=>{s=clearTimeout(s)},t)),n||(clearTimeout(s),s=setTimeout(()=>{e.apply(this,r)},t))}}function s(e){let t=!1;return(...r)=>{t||(t=!0,e.apply(this,r))}}r.d(t,{J:()=>s,s:()=>n})},1910(e,t,r){"use strict";r.d(t,{i:()=>i});var n=r(944);const s=new Map;function i(...e){return e.every(e=>{if(s.has(e))return s.get(e);const t="function"==typeof e?e.toString():"",r=t.includes("[native code]"),i=t.includes("nrWrapper");return r||i||(0,n.R)(64,e?.name||t),s.set(e,r),r})}},3304(e,t,r){"use strict";r.d(t,{A:()=>s});var n=r(7836);function s(e){try{return JSON.stringify(e,(()=>{const e=new WeakSet;return(t,r)=>{if("object"==typeof r&&null!==r){if(e.has(r))return;e.add(r)}return r}})())??""}catch(e){try{n.ee.emit("internal-error",[e])}catch(e){}return""}}},7508(e,t,r){"use strict";r.d(t,{AZ:()=>g,Qr:()=>b,QL:()=>m});var n=r(6154),s=r(1863),i=r(9119),o=r(7866);class a{dom=new c;performance=new c;constructor(e){this.url=e}get script(){const e=Math.max(this.dom.start,this.performance.end);return{start:e,end:Math.max(this.dom.end,this.performance.end,e)}}}class c{start=0;end=0;value=void 0}let d;try{d=g(m())[0]}catch(e){d=g(e)[0]}const u=e=>"script"===e.initiatorType||["link","fetch"].includes(e.initiatorType)&&e.name.endsWith(".js"),l=new Map;let f=[];function h(e){return l.get(e)}function p(e){const t=h(e);if(t)return t;const r=new a(e);if(l.set(e,r),l.size>1e3){const e=l.keys().next().value;l.delete(e)}return r}if(n.gm.MutationObserver&&n.gm.document){new MutationObserver(e=>{e.forEach(e=>{e.addedNodes.forEach(e=>{if("SCRIPT"===e.nodeName&&e.src){const t=p((0,i.L)(e.src));t.dom.start=(0,s.t)(),t.dom.value=e;const r=()=>{t.dom.end=(0,s.t)()};["load","error"].forEach(t=>e.addEventListener(t,r,{once:!0}))}})})}).observe(n.gm.document,{childList:!0,subtree:!0})}if(n.gm.PerformanceObserver?.supportedEntryTypes.includes("resource")){new PerformanceObserver(e=>{e.getEntries().filter(u).forEach(e=>{const t=p((0,i.L)(e.name));t.performance.start=Math.floor(e.startTime),t.performance.end=Math.floor(e.responseEnd),t.performance.value=e;const r=[];f.forEach(({test:t,addedAt:n},i)=>{(t(e)||(0,s.t)()-n>1e4)&&r.push(i)}),f=f.filter((e,t)=>!r.includes(t))})}).observe({type:"resource",buffered:!0})}function g(e){if(!e||"string"!=typeof e)return[];const t=new Set,r=e.split("\n");for(const e of r){const r=e.match(o.cn)||e.match(o.hB)||e.match(o.fL);if(r&&r[2])t.add((0,i.L)(r[2]));else{const r=e.match(/\(([^)]+\.js):\d+:\d+\)/)||e.match(/^\s+at\s+([^\s(]+\.js):\d+:\d+/);r&&r[1]&&t.add((0,i.L)(r[1]))}}return[...t]}function m(){let e;try{const t=Error.stackTraceLimit;Error.stackTraceLimit=50,e=(new Error).stack,Error.stackTraceLimit=t}catch(t){e=(new Error).stack}return e}function v(e,t){return(0,i.L)(e.name)===t}function y(e,t){e.fetchStart=Math.floor(t.startTime),e.fetchEnd=Math.floor(t.responseEnd),e.asset=t.name,e.type=t.initiatorType}function b(){const e={registeredAt:(0,s.t)(),reportedAt:void 0,fetchStart:0,fetchEnd:0,scriptStart:0,scriptEnd:0,asset:void 0,type:"unknown"},t=m();if(!t)return e;const r=n.gm.performance?.getEntriesByType("navigation")?.[0]?.name||"";try{const o=g(t),a=(o.length>1?o.filter(e=>d!==e):o)[0];if(!a)return e;if(r.includes(a))return e.asset=(0,i.L)(r),e.type="inline",e;e.correlation=h(a);const c=e.correlation?.performance.value||performance.getEntriesByType("resource").find(e=>v(e,a));c?y(e,c):function(e){if(!e||!n.gm.document)return!1;try{const t=n.gm.document.querySelectorAll('link[rel="preload"][as="script"]');for(const r of t)if((0,i.L)(r.href)===e)return!0}catch(e){}return!1}(a)&&(e.asset=a,e.type="preload",f.push({addedAt:(0,s.t)(),test:t=>!!v(t,a)&&(y(e,t),!0)})),Object.defineProperty(e,"scriptStart",{get:()=>e.correlation?.script.start||e.fetchEnd}),Object.defineProperty(e,"scriptEnd",{get:()=>e.correlation?.script.end||e.registeredAt})}catch(e){}return e}},5732(e,t,r){"use strict";r.d(t,{$5:()=>d,B5:()=>c,Ms:()=>s,Ux:()=>o,YA:()=>a,yx:()=>i});var n=r(7508);function s(e,t){if(!e||!t?.init.api.register.enabled)return[];const r=t.runtime.registeredEntities;return r?.filter(t=>String(t.metadata.target.id)===String(e)).map(e=>e.metadata.target)||[]}function i(e,t){if(!e||!t?.init.api.register.enabled)return[];const r=t.runtime.registeredEntities;return r?.filter(t=>t.metadata.timings?.asset?.endsWith(e)).map(e=>e.metadata.target)||[]}function o(e,t){if(!u(t))return{};const r=t.agentRef.runtime.appMetadata.agents[0].entityGuid;return e?e.attributes:{"entity.guid":r,appId:t.agentRef.info.applicationID}}function a(e,t){return c(e,t)?{"child.id":e.id,"child.type":e.type,...o(void 0,t)}:{}}function c(e,t){return!!e&&!!u(t)&&t.agentRef.init.api.register.duplicate_data_to_container}function d(e){if(!e?.init.api.register.enabled||!e?.runtime?.registeredEntities?.length)return[void 0];const t=[];try{var r=(0,n.AZ)((0,n.QL)());let s=r.length-1;for(;r[s];)t.push(...i(r[s--],e))}catch(e){}return t.length||t.push(void 0),t}function u(e){return 2===e?.harvestEndpointVersion}r.d(t,["fQ",0,{MFE:"MFE",BA:"BA"}])},5289(e,t,r){"use strict";r.d(t,{GG:()=>o,Qr:()=>c,sB:()=>a});var n=r(3878),s=r(6389);function i(){return"undefined"==typeof document||"complete"===document.readyState}function o(e,t){if(i())return e();const r=(0,s.J)(e),o=setInterval(()=>{i()&&(clearInterval(o),r())},500);(0,n.sp)("load",r,t)}function a(e){if(i())return e();(0,n.DD)("DOMContentLoaded",e)}function c(e){if(i())return e();(0,n.sp)("popstate",e)}},384(e,t,r){"use strict";r.d(t,{Zm:()=>c,bQ:()=>u,dV:()=>d,pV:()=>l});var n=r(6154),s=r(1863),i=r(944),o=r(1910);const a={beacon:"bam.nr-data.net",errorBeacon:"bam.nr-data.net"};function c(){return n.gm.NREUM||(n.gm.NREUM={}),void 0===n.gm.newrelic&&(n.gm.newrelic=n.gm.NREUM),n.gm.NREUM}function d(){let e=c();return e.o||(e.o={ST:n.gm.setTimeout,SI:n.gm.setImmediate||n.gm.setInterval,CT:n.gm.clearTimeout,XHR:n.gm.XMLHttpRequest,REQ:n.gm.Request,EV:n.gm.Event,PR:n.gm.Promise,MO:n.gm.MutationObserver,FETCH:n.gm.fetch,WS:n.gm.WebSocket},(0,o.i)(...Object.values(e.o))),e}function u(e,t){let r=c();r.initializedAgents??={},t.initializedAt={ms:(0,s.t)(),date:new Date},r.initializedAgents[e]=t,2===Object.keys(r.initializedAgents).length&&(0,i.R)(69)}function l(){return function(){let e=c();const t=e.info||{};e.info={beacon:a.beacon,errorBeacon:a.errorBeacon,...t}}(),function(){let e=c();const t=e.init||{};e.init={...t}}(),d(),function(){let e=c();const t=e.loader_config||{};e.loader_config={...t}}(),c()}r.d(t,["NT",0,a])},2843(e,t,r){"use strict";r.d(t,{G:()=>i,u:()=>s});var n=r(3878);function s(e,t=!1,r,s){(0,n.DD)("visibilitychange",function(){if(t)return void("hidden"===document.visibilityState&&e());e(document.visibilityState)},r,s)}function i(e,t,r){(0,n.sp)("pagehide",e,t,r)}},8139(e,t,r){"use strict";r.d(t,{u:()=>f});var n=r(7836),s=r(3434),i=r(8990),o=r(6154);const a={},c=o.gm.XMLHttpRequest,d="addEventListener",u="removeEventListener",l="nr@wrapped:".concat(n.P);function f(e){var t=function(e){return(e||n.ee).get("events")}(e);if(a[t.debugId]++)return t;a[t.debugId]=1;var r=(0,s.YM)(t,!0);function f(e){r.inPlace(e,[d,u],"-",p)}function p(e,t){return e[1]}return"getPrototypeOf"in Object&&(o.RI&&h(document,f),c&&h(c.prototype,f),h(o.gm,f)),t.on(d+"-start",function(e,t){var n=e[1];if(null!==n&&("function"==typeof n||"object"==typeof n)&&"newrelic"!==e[0]){var s=(0,i.I)(n,l,function(){var e={object:function(){if("function"!=typeof n.handleEvent)return;return n.handleEvent.apply(n,arguments)},function:n}[typeof n];return e?r(e,"fn-",null,e.name||"anonymous"):n});this.wrapped=e[1]=s}}),t.on(u+"-start",function(e){e[1]=this.wrapped||e[1]}),t}function h(e,t,...r){let n=e;for(;"object"==typeof n&&!Object.prototype.hasOwnProperty.call(n,d);)n=Object.getPrototypeOf(n);n&&t(n,...r)}},3434(e,t,r){"use strict";r.d(t,{YM:()=>d});var n=r(7836),s=r(5607),i=r(5732);const o="nr@original:".concat(s.W);var a=Object.prototype.hasOwnProperty,c=!1;function d(e,t,r){return e||(e=n.ee),s.inPlace=function(e,t,r,n,i,o){r||(r="");const a="-"===r.charAt(0);for(let c=0;c<t.length;c++){const d=t[c],u=e[d];l(u)||(e[d]=s(u,a?d+r:r,n,d,i,o))}},s.flag=o,s;function s(t,n,s,c,f,h){return l(t)?t:(n||(n=""),nrWrapper[o]=t,function(e,t,r){if(Object.defineProperty&&Object.keys)try{return Object.keys(e).forEach(function(r){Object.defineProperty(t,r,{get:function(){return e[r]},set:function(t){return e[r]=t,t}})}),t}catch(e){u([e],r)}for(var n in e)a.call(e,n)&&(t[n]=e[n])}(t,nrWrapper,e),nrWrapper);function nrWrapper(){var o,a,l,p;let g,m;try{a=this,o=[...arguments],m=h?(0,i.$5)(r):[void 0],l="function"==typeof s?s(o,a):s||{}}catch(t){u([t,"",[o,a,c],l],e)}d(n+"start",[o,a,c,m],l,f);const v=performance.now();let y;try{return p=t.apply(a,o),y=performance.now(),p}catch(e){throw y=performance.now(),d(n+"err",[o,a,e,m],l,f),g=e,g}finally{const e=y-v,t={start:v,end:y,duration:e,isLongTask:e>=50,methodName:c,thrownError:g};t.isLongTask&&d("long-task",[t,a,m],l,f),d(n+"end",[o,a,p,m],l,f)}}}function d(r,n,s,i){if(!c||t){var o=c;c=!0;try{e.emit(r,n,s,t,i)}catch(t){u([t,r,n,s],e)}c=o}}}function u(e,t){t||(t=n.ee);try{t.emit("internal-error",e)}catch(e){}}function l(e){return!(e&&"function"==typeof e&&e.apply&&!e[o])}r.d(t,["Jt",0,o])},9300(e,t,r){"use strict";const n=r(860).K7.ajax;r.d(t,["TZ",0,n,"f5",0,"ajaxRequest.id","mo",0,{NONE:"none",FAILURES:"failures",ALL:"all"}])},3333(e,t,r){"use strict";var n=r(981);const s=r(860).K7.genericEvents,i=[n.R.PA,n.R.UA,n.R.BP];r.d(t,["$v",0,{RESOURCES:"experimental.resources",REGISTER:"register"},"TZ",0,s,"Zp",0,["auxclick","click","copy","keydown","paste","scrollend"],"kd",0,i,"qN",0,["focus","blur"]])},6774(e,t,r){"use strict";const n=r(860).K7.jserrors;r.d(t,["T",0,n])},993(e,t,r){"use strict";const n=r(860).K7.logging;r.d(t,["A$",0,{OFF:0,ERROR:1,WARN:2,INFO:3,DEBUG:4,TRACE:5},"TZ",0,n,"p_",0,{ERROR:"ERROR",WARN:"WARN",INFO:"INFO",DEBUG:"DEBUG",TRACE:"TRACE"}])},3785(e,t,r){"use strict";r.d(t,{R:()=>c,b:()=>d});var n=r(9908),s=r(1863),i=r(860),o=r(3969),a=r(993);function c(e,t,r={},c=a.p_.INFO,d=!0,u,l=(0,s.t)()){(0,n.p)(o.xV,["API/logging/".concat(c.toLowerCase(),"/called")],void 0,i.K7.metrics,e),(0,n.p)("log",[l,t,r,c,d,u],void 0,i.K7.logging,e)}function d(e){return"string"==typeof e&&Object.values(a.p_).some(t=>t===e.toUpperCase().trim())}},3969(e,t,r){"use strict";const n=r(860).K7.metrics;r.d(t,["TZ",0,n,"XG",0,"storeEventMetrics","xV",0,"storeSupportabilityMetrics"])},6630(e,t,r){"use strict";const n=r(860).K7.pageViewEvent;r.d(t,["T",0,n])},782(e,t,r){"use strict";const n=r(860).K7.pageViewTiming;r.d(t,["T",0,n])},6344(e,t,r){"use strict";var n=r(2614);const s=r(860).K7.sessionReplay,i={[n.g.ERROR]:15e3,[n.g.FULL]:3e5,[n.g.OFF]:0};r.d(t,["Qb",0,{API:"api",RESUME:"resume",SWITCH_TO_FULL:"switchToFull",INITIALIZE:"initialize",PRELOAD:"preload"},"TZ",0,s,"Vh",0,"errorDuringReplay","_s",0,{DomContentLoaded:0,Load:1,FullSnapshot:2,IncrementalSnapshot:3,Meta:4,Custom:5},"bc",0,{RESET:{message:"Session was reset",sm:"Reset"},IMPORT:{message:"Recorder failed to import",sm:"Import"},TOO_MANY:{message:"429: Too Many Requests",sm:"Too-Many"},TOO_BIG:{message:"Payload was too large",sm:"Too-Big"},CROSS_TAB:{message:"Session Entity was set to OFF on another tab",sm:"Cross-Tab"},ENTITLEMENTS:{message:"Session Replay is not allowed and will not be started",sm:"Entitlement"}},"yP",0,i])},5270(e,t,r){"use strict";r.d(t,{Aw:()=>o,SR:()=>i,rF:()=>a});var n=r(384),s=r(7767);function i(e){return!!(0,n.dV)().o.MO&&(0,s.V)(e)&&!0===e?.session_trace.enabled}function o(e){return!0===e?.session_replay.preload&&i(e)}function a(e,t){try{if("string"==typeof t?.type){if("password"===t.type.toLowerCase())return"*".repeat(e?.length||0);if(void 0!==t?.dataset?.nrUnmask||t?.classList?.contains("nr-unmask"))return e}}catch(e){}return"string"==typeof e?e.replace(/[\S]/g,"*"):"*".repeat(e?.length||0)}},3738(e,t,r){"use strict";const n=r(860).K7.sessionTrace,s="-start",i="-end",o="fn"+s,a="fn"+i;r.d(t,["He",0,"bstResource","Kp",0,i,"Lc",0,a,"Rz",0,"pushState","TZ",0,n,"bD",0,"resource","bc",0,{CROSS_TAB:{message:"Session Entity was set to OFF on another tab"},ENTITLEMENTS:{message:"Session Trace is not allowed and will not be started"},RESET:{message:"Session was reset"},SESSION_CHANGED:{message:"Session identifier changed unexpectedly"}},"d3",0,s,"uP",0,o])},3962(e,t,r){"use strict";const n=r(860).K7.softNav;r.d(t,["O2",0,{INITIAL_PAGE_LOAD:"",ROUTE_CHANGE:1,UNSPECIFIED:2},"OV",0,"popstate","Qu",0,{INTERACTION:1,AJAX:2,CUSTOM_END:3,CUSTOM_TRACER:4},"TZ",0,n,"ih",0,{IP:"in progress",PF:"pending finish",FIN:"finished",CAN:"cancelled"},"pP",0,"initialPageLoad","tC",0,["click","keydown","submit"]])},4234(e,t,r){"use strict";r.d(t,{W:()=>s});var n=r(1687);class s{constructor(e,t){this.agentRef=e,this.ee=e?.ee,this.featureName=t,this.blocked=!1}deregisterDrain(){(0,n.x3)(this.agentRef,this.featureName)}}},7767(e,t,r){"use strict";var n=r(6154);r.d(t,["V",0,e=>n.RI&&!0===e?.privacy.cookies_enabled])},8362(e,t,r){"use strict";r.d(t,{d:()=>i});var n=r(9566),s=r(1741);class i extends s.W{agentIdentifier=(0,n.LA)(16)}},1741(e,t,r){"use strict";r.d(t,{W:()=>i});var n=r(944),s=r(4261);class i{#e(e,...t){if(this[e]!==i.prototype[e])return this[e](...t);(0,n.R)(35,e)}addPageAction(e,t){return this.#e(s.hG,e,t)}register(e){return this.#e(s.eY,e)}recordCustomEvent(e,t){return this.#e(s.fF,e,t)}setPageViewName(e,t){return this.#e(s.Fw,e,t)}setCustomAttribute(e,t,r){return this.#e(s.cD,e,t,r)}noticeError(e,t){return this.#e(s.o5,e,t)}setUserId(e,t=!1){return this.#e(s.Dl,e,t)}setApplicationVersion(e){return this.#e(s.nb,e)}setErrorHandler(e){return this.#e(s.bt,e)}addRelease(e,t){return this.#e(s.k6,e,t)}log(e,t){return this.#e("log",e,t)}start(){return this.#e("start")}finished(e){return this.#e(s.BL,e)}recordReplay(){return this.#e(s.CH)}pauseReplay(){return this.#e(s.Tb)}addToTrace(e){return this.#e(s.U2,e)}setCurrentRouteName(e){return this.#e(s.PA,e)}interaction(e){return this.#e(s.dT,e)}wrapLogger(e,t,r){return this.#e(s.Wb,e,t,r)}measure(e,t){return this.#e(s.V1,e,t)}consent(e){return this.#e(s.Pv,e)}}},4261(e,t,r){"use strict";r.d(t,["BL",0,"finished","CH",0,"recordReplay","Dl",0,"setUserId","Fw",0,"setPageViewName","PA",0,"setCurrentRouteName","Pv",0,"consent","Tb",0,"pauseReplay","U2",0,"addToTrace","V1",0,"measure","Wb",0,"wrapLogger","bt",0,"setErrorHandler","cD",0,"setCustomAttribute","dT",0,"interaction","eY",0,"register","fF",0,"recordCustomEvent","hG",0,"addPageAction","hw",0,"api-ixn-","k6",0,"addRelease","nb",0,"setApplicationVersion","o5",0,"noticeError"])},1738(e,t,r){"use strict";r.d(t,{U:()=>l,Y:()=>u});var n=r(3241),s=r(9908),i=r(1863),o=r(944),a=r(3969),c=r(8362),d=r(860);function u(e,t,r,i){const u=i||r;!u||u[e]&&u[e]!==c.d.prototype[e]||(u[e]=function(){(0,s.p)(a.xV,["API/"+e+"/called"],void 0,d.K7.metrics,r.ee),(0,n.W)({drained:!!r.runtime?.activatedFeatures,type:"data",name:"api",feature:"api-"+e,data:{}});try{return t.apply(this,arguments)}catch(e){(0,o.R)(23,e)}})}function l(e,t,r,n,o){const a=e.info;null===r?delete a.jsAttributes[t]:a.jsAttributes[t]=r,(o||null===r)&&(0,s.p)("api-"+n,[(0,i.t)(),t,r],void 0,"session",e.ee)}},8374(e,t,r){r.nc=(()=>{try{return document?.currentScript?.nonce}catch(e){}return""})()},860(e,t,r){"use strict";const n="events",s="jserrors",i="browser/blobs",o="browser/logs",a={ajax:"ajax",genericEvents:"generic_events",jserrors:s,logging:"logging",metrics:"metrics",pageAction:"page_action",pageViewEvent:"page_view_event",pageViewTiming:"page_view_timing",sessionReplay:"session_replay",sessionTrace:"session_trace",softNav:"soft_navigations"},c={[a.pageViewEvent]:1,[a.pageViewTiming]:2,[a.metrics]:3,[a.jserrors]:4,[a.softNav]:5,[a.ajax]:6,[a.sessionTrace]:7,[a.sessionReplay]:8,[a.logging]:9,[a.genericEvents]:10},d={[a.pageViewEvent]:"rum",[a.pageViewTiming]:n,[a.ajax]:n,[a.softNav]:n,[a.metrics]:s,[a.jserrors]:s,[a.sessionTrace]:i,[a.sessionReplay]:i,[a.logging]:o,[a.genericEvents]:"ins"};r.d(t,["$J",0,d,"K7",0,a,"P3",0,c,"XX",0,s,"Yy",0,o,"df",0,i])}};const t={};function r(n){const s=t[n];if(void 0!==s)return s.exports;const i=t[n]={exports:{}};return e[n](i,i.exports,r),i.exports}r.m=e,r.d=(e,t)=>{if(Array.isArray(t))for(var n=0;n<t.length;){var s=t[n++],i=t[n++];r.o(e,s)?0===i&&n++:0===i?Object.defineProperty(e,s,{enumerable:!0,value:t[n++]}):Object.defineProperty(e,s,{enumerable:!0,get:i})}else for(var s in t)r.o(t,s)&&!r.o(e,s)&&Object.defineProperty(e,s,{enumerable:!0,get:t[s]})},r.f={},r.e=e=>Promise.all(Object.keys(r.f).reduce((t,n)=>(r.f[n](e,t),t),[])),r.u=e=>({212:"nr-spa-compressor",249:"nr-spa-recorder",478:"nr-spa"}[e]+"-1.319.0.min.js"),r.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),(()=>{const e={},t="NRBA-1.319.0.PROD:";r.l=(n,s,i,o)=>{if(e[n])return void e[n].push(s);let a,c;if(void 0!==i){const e=document.getElementsByTagName("script");for(var d=0;d<e.length;d++){const r=e[d];if(r.getAttribute("src")==n||r.getAttribute("data-webpack")==t+i){a=r;break}}}if(!a){c=!0;var u={478:"sha512-dq8MCH0pxylfoPrUdvWgvfnUaUooTXrYyd67BT880/uJdIonQ6k0/BvrkhnRPc9cPaKh+Hlzi6r4/OYNeCFAEw==",249:"sha512-7wrk3j5JzTYr39CrfsZFRxV3PvNq8HYEAaqxXhor6GI99tX7scYsyWh+TmH1gUywzjeqxZeddmG6fSsCWSL1GA==",212:"sha512-WyFOCOq79iASoGDVkGHlLI2F+4aUnJHxUppxunzIdO9/rJRSwR49QRJbq26sHcY3RIVESwv34MVkSK4giih2lQ=="};a=document.createElement("script"),a.charset="utf-8",r.nc&&a.setAttribute("nonce",r.nc),a.setAttribute("data-webpack",t+i),a.src=n,0!==a.src.indexOf(window.location.origin+"/")&&(a.crossOrigin="anonymous"),u[o]&&(a.integrity=u[o])}e[n]=[s];const l=(t,r)=>{a.onerror=a.onload=null,clearTimeout(f);const s=e[n];if(delete e[n],a.parentNode?.removeChild(a),s?.forEach(e=>e(r)),t)return t(r)},f=setTimeout(l.bind(null,void 0,{type:"timeout",target:a}),12e4);a.onerror=l.bind(null,a.onerror),a.onload=l.bind(null,a.onload),c&&document.head.appendChild(a)}})(),r.r=e=>{Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},r.p="https://js-agent.newrelic.com/",(()=>{const e={38:0,788:0};r.f.j=(t,n)=>{let s=r.o(e,t)?e[t]:void 0;if(0!==s)if(s)n.push(s[2]);else{const i=new Promise((r,n)=>s=e[t]=[r,n]);n.push(s[2]=i);const o=r.p+r.u(t),a=new Error,c=n=>{if(r.o(e,t)&&(s=e[t],0!==s&&(e[t]=void 0),s)){const e=n&&("load"===n.type?"missing":n.type),r=n&&n.target&&n.target.src;a.message="Loading chunk "+t+" failed: ("+e+": "+r+")",a.name="ChunkLoadError",a.type=e,a.request=r,s[1](a)}};r.l(o,c,"chunk-"+t,t)}};const t=(t,n)=>{let[s,i,o]=n;var a,c,d=0;if(s.some(t=>0!==e[t])){for(a in i)r.o(i,a)&&(r.m[a]=i[a]);if(o)o(r)}for(t&&t(n);d<s.length;d++)c=s[d],r.o(e,c)&&e[c]&&e[c][0](),e[c]=0},n=self["webpackChunk:NRBA-1.319.0.PROD"]=self["webpackChunk:NRBA-1.319.0.PROD"]||[];n.forEach(t.bind(null,0)),n.push=t.bind(null,n.push.bind(n))})(),(()=>{"use strict";r(8374);var e=r(8362),t=r(860);const n=Object.values(t.K7);var s=r(384),i=r(1741);var o=r(2555),a=r(3333),c=r(9300);const d=e=>{if(!e||"string"!=typeof e)return!1;try{document.createDocumentFragment().querySelector(e)}catch{return!1}return!0};var u=r(2614),l=r(944),f=r(8122);const h="[data-nr-mask]",p=e=>(0,f.a)(e,(()=>{const e={feature_flags:[],experimental:{register:!1,resources:!1},mask_selector:"*",block_selector:"[data-nr-block]",mask_input_options:{color:!1,date:!1,"datetime-local":!1,email:!1,month:!1,number:!1,range:!1,search:!1,tel:!1,text:!1,time:!1,url:!1,week:!1,textarea:!1,select:!1,password:!0}};return{ajax:{deny_list:void 0,block_internal:!0,enabled:!0,autoStart:!0,capture_payloads:c.mo.NONE},api:{register:{get enabled(){return e.feature_flags.includes(a.$v.REGISTER)||e.experimental.register},set enabled(t){e.experimental.register=t},duplicate_data_to_container:!1}},browser_consent_mode:{enabled:!1},distributed_tracing:{enabled:void 0,exclude_newrelic_header:void 0,cors_use_newrelic_header:void 0,cors_use_tracecontext_headers:void 0,allowed_origins:void 0},get feature_flags(){return e.feature_flags},set feature_flags(t){e.feature_flags=t},generic_events:{enabled:!0,autoStart:!0},harvest:{interval:30},jserrors:{enabled:!0,autoStart:!0},logging:{enabled:!0,autoStart:!0},metrics:{enabled:!0,autoStart:!0},obfuscate:void 0,page_action:{enabled:!0},page_view_event:{enabled:!0,autoStart:!0},page_view_timing:{enabled:!0,autoStart:!0},performance:{capture_marks:!1,capture_measures:!1,capture_detail:!0,resources:{get enabled(){return e.feature_flags.includes(a.$v.RESOURCES)||e.experimental.resources},set enabled(t){e.experimental.resources=t},asset_types:[],first_party_domains:[],ignore_newrelic:!0}},privacy:{cookies_enabled:!0},proxy:{assets:void 0,beacon:void 0},session:{expiresMs:u.wk,inactiveMs:u.BB},session_replay:{autoStart:!0,enabled:!1,preload:!1,sampling_rate:10,error_sampling_rate:100,collect_fonts:!1,inline_images:!1,fix_stylesheets:!0,mask_all_inputs:!0,get mask_text_selector(){return e.mask_selector},set mask_text_selector(t){d(t)?e.mask_selector="".concat(t,",").concat(h):""===t||null===t?e.mask_selector=h:(0,l.R)(5,t)},get block_class(){return"nr-block"},get ignore_class(){return"nr-ignore"},get mask_text_class(){return"nr-mask"},get block_selector(){return e.block_selector},set block_selector(t){d(t)?e.block_selector+=",".concat(t):""!==t&&(0,l.R)(6,t)},get mask_input_options(){return e.mask_input_options},set mask_input_options(t){t&&"object"==typeof t?e.mask_input_options={...t,password:!0}:(0,l.R)(7,t)}},session_trace:{enabled:!0,autoStart:!0},soft_navigations:{enabled:!0,autoStart:!0},ssl:void 0,user_actions:{enabled:!0,elementAttributes:["id","className","tagName","type"]},web_sockets:{enabled:!1}}})());var g=r(6154),m=r(9324);let v=0;const y={buildEnv:"PROD",distMethod:"CDN",version:m.x,originTime:g.WN},b={consented:!1},E={activatedFeatures:void 0,appMetadata:{},configured:!1,get consented(){return this.session?.state?.consent||b.consented},set consented(e){b.consented=e},customTransaction:void 0,denyList:[],disabled:!1,drainRegistry:new Map,harvester:void 0,isolatedBacklog:!1,isRecording:!1,loaderType:void 0,maxBytes:3e4,obfuscator:void 0,onerror:void 0,ptid:void 0,releaseIds:{},session:void 0,timeKeeper:void 0,registeredEntities:[],jsAttributesMetadata:{bytes:0},get harvestCount(){return++v}};var w=r(7836),R=r(3241);const A={accountID:void 0,trustKey:void 0,agentID:void 0,licenseKey:void 0,applicationID:void 0,xpid:void 0};function T(e,t={},n,a){let{init:c,info:d,loader_config:u,runtime:l={},exposed:h=!0}=t;if(!d){const e=(0,s.pV)();c=e.init,d=e.info,u=e.loader_config}var m;e.init=p(c||{}),e.loader_config=(m=u||{},(0,f.a)(m,A)),d.jsAttributes??={},g.bv&&(d.jsAttributes.isWorker=!0),e.info=(0,o.D)(d);const v=e.init;e.runtime??=(e=>{const t=(0,f.a)(e,E),r=Object.keys(y).reduce((e,t)=>(e[t]={value:y[t],writable:!1,configurable:!0,enumerable:!0},e),{});return Object.defineProperties(t,r)})(l),v.proxy.assets&&(e=>{const t=e.startsWith("http");e+="/",r.p=t?e:"https://"+e})(v.proxy.assets),e.runtime.configured||(Object.defineProperty(e,"beacons",{get:()=>[e.info.beacon,e.info.errorBeacon,e.init.proxy.assets,e.init.proxy.beacon].filter(Boolean)}),Object.defineProperty(e.runtime,"denyList",{get:()=>[...e.init.ajax.deny_list||[],...e.init.ajax.block_internal?e.beacons:[]]}),e.runtime.ptid=e.agentIdentifier,function(e){const t=(0,s.pV)();Object.getOwnPropertyNames(i.W.prototype).forEach(r=>{const n=i.W.prototype[r];if("function"!=typeof n||"constructor"===n)return;let s=t[r];e[r]&&!1!==e.exposed&&"micro-agent"!==e.runtime?.loaderType&&(t[r]=(...t)=>{const n=e[r](...t);return s?s(...t):n})})}(e),e.runtime.loaderType=n,e.ee=w.ee.get(e.agentIdentifier),e.exposed=h,(0,R.W)({drained:!!e.runtime.activatedFeatures,type:"lifecycle",name:"initialize",feature:void 0,data:e.config}),e.runtime.configured=!0)}var x=r(9908),S=r(1863),O=r(4261),_=r(1738);var N=r(1687),P=r(4234),L=r(5289),k=r(5270),C=r(7767),j=r(6389),I=r(7699);const D=new WeakSet;class M extends P.W{constructor(e,t){super(e,t),this.abortHandler=void 0,this.featAggregate=void 0,this.loadedSuccessfully=void 0,this.onAggregateImported=new Promise(e=>{this.loadedSuccessfully=e}),this.deferred=Promise.resolve(),!1===e.init[this.featureName].autoStart?this.deferred=new Promise((t,r)=>{this.ee.on("manual-start-all",(0,j.J)(()=>{(0,N.Ak)(e,this.featureName),t()}))}):(0,N.Ak)(e,t)}importAggregator(e,t,n={}){if(this.featAggregate)return;const s=async()=>{if(await this.deferred,this.#t(e),!(0,o.f)(e.info))return(0,l.R)(43),e.ee.abort(),void this.loadedSuccessfully(!1);let s;try{if((0,C.V)(e.init)){const{setupAgentSession:t}=await r.e(478).then(r.bind(r,8766));s=t(e)}}catch(e){(0,l.R)(20,e),this.ee.emit("internal-error",[e]),(0,x.p)(I.qh,[e],void 0,this.featureName,this.ee)}try{if(!this.#r(this.featureName,s,e.init))return(0,N.Ze)(this.agentRef,this.featureName),void this.loadedSuccessfully(!1);const{Aggregate:r}=await t();this.featAggregate=new r(e,n),e.runtime.harvester.initializedAggregates.push(this.featAggregate),this.loadedSuccessfully(!0)}catch(e){(0,l.R)(34,e),this.abortHandler?.(),(0,N.Ze)(this.agentRef,this.featureName,!0),this.loadedSuccessfully(!1),this.ee&&this.ee.abort()}};g.RI?(0,L.GG)(()=>s(),!0):s()}#r(e,r,n){if(this.blocked)return!1;switch(e){case t.K7.sessionReplay:return(0,k.SR)(n)&&!!r;case t.K7.sessionTrace:return!!r;default:return!0}}#t(e){if(!D.has(e)&&(D.add(e),!(0,o.f)(e.info))){const t=(0,s.pV)();let r={...t.info?.jsAttributes};try{r={...r,...e.info?.jsAttributes}}catch(e){}T(e,{...t,info:{...t.info,jsAttributes:r},runtime:e.runtime},e.runtime.loaderType)}}}var B=r(6630);class H extends M{static featureName=B.T;constructor(e){var t;super(e,B.T),this.setupInspectionEvents(),t=e,(0,_.Y)(O.Fw,function(e,r){"string"==typeof e&&("/"!==e.charAt(0)&&(e="/"+e),t.runtime.customTransaction=(r||"http://custom.transaction")+e,(0,x.p)("api-"+O.Fw,[(0,S.t)()],void 0,void 0,t.ee))},t),this.importAggregator(e,()=>r.e(478).then(r.bind(r,5839)))}setupInspectionEvents(){const e=(e,t)=>{e&&(0,R.W)({timeStamp:e.timeStamp,loaded:"complete"===e.target.readyState,type:"window",name:t,data:e.target.location+""})};(0,L.sB)(t=>{e(t,"DOMContentLoaded")}),(0,L.GG)(t=>{e(t,"load")}),(0,L.Qr)(t=>{e(t,"navigate")}),this.ee.on(u.tS.UPDATE,(e,t)=>{(0,R.W)({type:"lifecycle",name:"session",data:t})})}}class W extends e.d{constructor(e){var t;(super(),g.gm)?(this.features={},(0,s.bQ)(this.agentIdentifier,this),this.desiredFeatures=new Set(e.features||[]),this.desiredFeatures.add(H),T(this,e,e.loaderType||"agent"),t=this,(0,_.Y)(O.cD,function(e,r,n=!1){if("string"==typeof e){if(["string","number","boolean"].includes(typeof r)||null===r)return(0,_.U)(t,e,r,O.cD,n);(0,l.R)(40,typeof r)}else(0,l.R)(39,typeof e)},t),function(e){(0,_.Y)(O.Dl,function(t,r=!1){if("string"!=typeof t&&null!==t)return void(0,l.R)(41,typeof t);const n=e.info.jsAttributes["enduser.id"];r&&null!=n&&n!==t?(0,x.p)("api-setUserIdAndResetSession",[t],void 0,"session",e.ee):(0,_.U)(e,"enduser.id",t,O.Dl,!0)},e)}(this),function(e){(0,_.Y)(O.nb,function(t){if("string"==typeof t||null===t)return(0,_.U)(e,"application.version",t,O.nb,!1);(0,l.R)(42,typeof t)},e)}(this),function(e){(0,_.Y)("start",function(){e.ee.emit("manual-start-all")},e)}(this),function(e){(0,_.Y)(O.Pv,function(t=!0){if("boolean"==typeof t){if((0,x.p)("api-"+O.Pv,[t],void 0,"session",e.ee),e.runtime.consented=t,t){const t=e.features.page_view_event;t.onAggregateImported.then(e=>{const r=t.featAggregate;e&&!r.sentRum&&r.sendRum()})}}else(0,l.R)(65,typeof t)},e)}(this),this.run()):(0,l.R)(21)}get config(){return{info:this.info,init:this.init,loader_config:this.loader_config,runtime:this.runtime}}get api(){return this}run(){try{const e=function(e){const t={};return n.forEach(r=>{t[r]=!!e[r]?.enabled}),t}(this.init),r=[...this.desiredFeatures];r.sort((e,r)=>t.P3[e.featureName]-t.P3[r.featureName]),r.forEach(r=>{if(!e[r.featureName]&&r.featureName!==t.K7.pageViewEvent)return;const n=function(e){switch(e){case t.K7.ajax:return[t.K7.jserrors];case t.K7.sessionTrace:return[t.K7.ajax,t.K7.pageViewEvent];case t.K7.sessionReplay:return[t.K7.sessionTrace];case t.K7.pageViewTiming:return[t.K7.pageViewEvent];default:return[]}}(r.featureName).filter(e=>!(e in this.features));n.length>0&&(0,l.R)(36,{targetFeature:r.featureName,missingDependencies:n}),this.features[r.featureName]=new r(this)})}catch(e){(0,l.R)(22,e);for(const e in this.features)this.features[e].abortHandler?.();const t=(0,s.Zm)();delete t.initializedAgents[this.agentIdentifier]?.features,delete this.sharedAggregator;return t.ee.get(this.agentIdentifier).abort(),!1}}}var K=r(2843),F=r(782);class U extends M{static featureName=F.T;constructor(e){super(e,F.T),g.RI&&((0,K.u)(()=>(0,x.p)("docHidden",[(0,S.t)()],void 0,F.T,this.ee),!0),(0,K.G)(()=>(0,x.p)("winPagehide",[(0,S.t)()],void 0,F.T,this.ee)),this.importAggregator(e,()=>r.e(478).then(r.bind(r,9917))))}}var V=r(3969);class z extends M{static featureName=V.TZ;constructor(e){super(e,V.TZ),this.importAggregator(e,()=>r.e(478).then(r.bind(r,6555)))}}var G=r(6774),q=r(3878),Y=r(3304);class Z{constructor(e,t,r,n,s){this.name="UncaughtError",this.message="string"==typeof e?e:(0,Y.A)(e),this.sourceURL=t,this.line=r,this.column=n,this.__newrelic=s}}function Q(e){return ee(e)?e:new Z(void 0!==e?.message?e.message:e,e?.filename||e?.sourceURL,e?.lineno||e?.line,e?.colno||e?.col,e?.__newrelic,e?.cause)}function X(e){const t="Unhandled Promise Rejection: ";if(!e?.reason)return;if(ee(e.reason)){try{e.reason.message.startsWith(t)||(e.reason.message=t+e.reason.message)}catch(e){}return Q(e.reason)}const r=Q(e.reason);return(r.message||"").startsWith(t)||(r.message=t+r.message),r}function J(e){if(e.error instanceof SyntaxError&&!/:\d+$/.test(e.error.stack?.trim())){const t=new Z(e.message,e.filename,e.lineno,e.colno,e.error.__newrelic,e.cause);return t.name=SyntaxError.name,t}return ee(e.error)?e.error:Q(e)}function ee(e){return e instanceof Error&&!!e.stack}function te(e,r,n,s,i=(0,S.t)()){"string"==typeof e&&(e=new Error(e)),(0,x.p)("err",[e,i,!1,r,n.runtime.isRecording,void 0,s],void 0,t.K7.jserrors,n.ee),(0,x.p)("uaErr",[],void 0,t.K7.genericEvents,n.ee)}var re=r(5732),ne=r(993),se=r(3785);function ie(e,{customAttributes:t={},level:r=ne.p_.INFO}={},n,s,i=(0,S.t)()){(0,se.R)(n.ee,e,t,r,!1,s,i)}function oe(e,r,n,s,i=(0,S.t)()){(0,x.p)("api-"+O.hG,[i,e,r,s],void 0,t.K7.genericEvents,n.ee)}function ae(e,r,n,s,i=(0,S.t)()){const{start:o,end:a,customAttributes:c}=r||{},d={customAttributes:c||{}};if("object"!=typeof d.customAttributes||"string"!=typeof e||0===e.length)return void(0,l.R)(57);const u=(e,t)=>null==e?t:"number"==typeof e?e:e instanceof PerformanceMark?e.startTime:Number.NaN;if(d.start=u(o,0),d.end=u(a,i),Number.isNaN(d.start)||Number.isNaN(d.end))(0,l.R)(57);else{if(d.duration=d.end-d.start,!(d.duration<0))return(0,x.p)("api-"+O.V1,[d,e,s],void 0,t.K7.genericEvents,n.ee),d;(0,l.R)(58)}}function ce(e,r={},n,s,i=(0,S.t)()){(0,x.p)("api-"+O.fF,[i,e,r,s],void 0,t.K7.genericEvents,n.ee)}var de=r(7508);const ue=(e,t)=>e?.nrMfeId===t.id&&(!e?.nrMfeObserved||e?.nrMfeObserved===t.instance),le=e=>{try{return g.gm.CSS.escape(e)}catch(t){return e}},fe=(e,t={})=>{const r=t.id,n=t.instance;if(!e||!r||!n)return!1;try{let r=1===e.nodeType?e:e.parentElement;for(;r?.tagName;){if(ue(r.dataset,t))return r.dataset.nrMfeObserved=n,!0;r=r.parentNode}}catch(e){}return!1},he=(e,t)=>{const r=g.gm.document?.querySelectorAll('[data-nr-mfe-id="'.concat(le(e.id),'"]')),n=(Array.from(r)||[]).find(t=>ue(t.dataset,e));let s=!!n;s&&(n.dataset.nrMfeObserved??=e.instance);const i=new g.gm.MutationObserver(r=>{r.forEach(r=>{r.addedNodes.forEach(r=>{const n=1===r.nodeType?r:null;!s&&ue(n?.dataset,e)&&(i.disconnect(),i.observe(n,{childList:!0,subtree:!0}),s=!0,n.dataset.nrMfeObserved=e.instance),(e=>{try{return e?.textContent?.trim()||["img","video","canvas","svg"].includes(e?.nodeName?.toLowerCase())}catch(e){return!1}})(r)&&(s||fe(r,e))&&t(r,i)})})});return i.observe(n||g.gm.document,{childList:!0,subtree:!0}),i},pe=(e,t,r)=>{try{const n=new g.gm.PerformanceObserver(e=>{e.getEntries().forEach(r)});return n.observe(t),e.push(n),n}catch(e){return null}};function ge(e,t){let r,n;const s=e=>null==e?e:e-(t?.scriptStart||t?.registeredAt||0),i={fcp:{get value(){return s(r)}},lcp:{get value(){return s(n)}},cls:{value:void 0},inp:{value:void 0},disconnect:()=>{}};if(!(e&&g.RI&&g.gm.MutationObserver&&g.gm.PerformanceObserver))return i;const o=[];setTimeout(()=>{r||i.disconnect()},1e4);const a=()=>{r??=(0,S.t)(),n??=(0,S.t)(),i.cls.value??=0},c=g.gm.document?.querySelectorAll('[data-nr-mfe-id="'.concat(le(e.id),'"]'));Array.from(c||[]).some(t=>ue(t.dataset,e))&&a();const d=he(e,(e,t)=>{a(),t.disconnect()});o.push(d);let u=0,l=null;const f=new Set;try{l=new g.gm.ResizeObserver(e=>{e.forEach(e=>{try{const t=e.contentRect.width*e.contentRect.height;t>u&&(u=t,n=(0,S.t)()),l.unobserve(e.target)}catch(e){}})})}catch(e){}const h=he(e,e=>{if(a(),l)try{const t=1===e.nodeType?e:e.parentElement;t&&!f.has(t)&&(f.add(t),"IMG"===t.tagName?t.complete?l.observe(t):t.addEventListener("load",()=>{l.observe(t)},{once:!0}):"VIDEO"===t.tagName?t.readyState>=2?l.observe(t):t.addEventListener("loadeddata",()=>{l.observe(t)},{once:!0}):l.observe(t))}catch(e){}});l&&o.push(l),o.push(h),pe(o,{type:"layout-shift",buffered:!0},t=>{t.hadRecentInput||(t.sources||[]).some(r=>!!fe(r.node,e)&&(a(),i.cls.value+=t.value,!0))}),pe(o,{type:"event",buffered:!0,durationThreshold:40},t=>{t.interactionId&&fe(t.target,e)&&(void 0===i.inp.value||t.duration>i.inp.value)&&(a(),i.inp.value=t.duration)});const p=["pointerdown","keydown"],m=()=>{p.forEach(e=>{g.gm.removeEventListener(e,v,{passive:!0})})};i.disconnect=()=>{o.forEach(e=>{try{e?.disconnect()}catch(e){}}),m(),["visibilitychange","pagehide"].forEach(e=>{g.gm.removeEventListener(e,i.disconnect,{once:!0,passive:!0})})};const v=t=>{fe(t?.target,e)&&((()=>{try{h?.disconnect(),l?.disconnect()}catch(e){}})(),m())};return p.forEach(e=>{g.gm.addEventListener(e,v,{passive:!0})}),["visibilitychange","pagehide"].forEach(e=>{g.gm.addEventListener(e,i.disconnect,{once:!0,passive:!0})}),i}var me=r(9566);const ve=["name","id","type"],ye=new Map([[oe,"addPageAction"],[ie,"log"],[ae,"measure"],[te,"noticeError"],[ce,"recordCustomEvent"]]),be={experimental:(0,j.J)(()=>(0,l.R)(54,"newrelic.register")),disabled:(0,j.J)(()=>(0,l.R)(55)),invalidTarget:(0,j.J)(e=>(0,l.R)(48,e)),deregistered:(0,j.J)(()=>(0,l.R)(68))};function Ee(e){(0,_.Y)(O.eY,function(t){return we(e,t)},e)}function we(e,r){be.experimental(),r||={},r.instance=(0,me.LA)(8),r.type=re.fQ.MFE,r.licenseKey||=e.info.licenseKey,r.blocked=!1,("object"!=typeof r.tags||null===r.tags||Array.isArray(r.tags))&&(r.tags={}),r.parent??={get id(){return e.runtime.appMetadata.agents[0].entityGuid},type:re.fQ.BA};const n=(0,de.Qr)(),s=ge(r,n),i={};Object.prototype.hasOwnProperty.call(r,"attributes")||Object.defineProperty(r,"attributes",{get:()=>({...i,"source.id":r.id,"source.name":r.name,"source.type":r.type,"parent.type":r.parent?.type||re.fQ.BA,"parent.id":r.parent?.id})}),Object.entries(r.tags).forEach(([e,t])=>{ve.includes(e)||(i["source.".concat(e)]=t)});let o=()=>{};const a=e.runtime.registeredEntities,c=e=>{r.blocked=!0,o=e};function d(e){return"string"==typeof e&&!!e.trim()&&e.trim().length<501}e.init.api.register.enabled||c(be.disabled),d(r.id)&&d(r.name)||c(()=>be.invalidTarget(r));const u={addPageAction:(t,n={})=>g(oe,[t,{...i,...n},e],r),deregister:()=>{h(),c(be.deregistered)},log:(t,n={})=>g(ie,[t,{...n,customAttributes:{...i,...n.customAttributes||{}}},e],r),measure:(t,n={})=>g(ae,[t,{...n,customAttributes:{...i,...n.customAttributes||{}}},e],r),noticeError:(t,n={})=>g(te,[t,{...i,...n},e],r),recordCustomEvent:(t,n={})=>g(ce,[t,{...i,...n},e],r),setApplicationVersion:e=>p("application.version",e),setCustomAttribute:(e,t)=>p(e,t),setUserId:e=>p("enduser.id",e),metadata:{get customAttributes(){return i},target:r,timings:n,vitals:s}},f=()=>(r.blocked&&o(),r.blocked);function h(){if(n.reportedAt)return;n.reportedAt=(0,S.t)(),s.disconnect(),n.fcp=s.fcp,n.lcp=s.lcp,n.cls=s.cls,n.inp=s.inp;const e=n.fetchEnd-n.fetchStart,t=n.scriptEnd-n.scriptStart,r={assetUrl:n.asset,assetType:n.type,timeAlive:n.reportedAt-n.registeredAt,timeToBeRequested:n.fetchStart,timeToExecute:t,timeToFetch:e,timeToLoad:e+t,timeToRegister:n.registeredAt,...s.fcp.value>=0&&{"nr.vitals.fcp.value":s.fcp.value},...s.lcp.value>=0&&{"nr.vitals.lcp.value":s.lcp.value},...s.cls.value>=0&&{"nr.vitals.cls.value":s.cls.value},...s.inp.value>=0&&{"nr.vitals.inp.value":s.inp.value}};u.recordCustomEvent("MicroFrontEndTiming",r)}f()||(a.push(u),(0,K.G)(h));const p=(e,t)=>{f()||(i[e]=t)},g=(r,n,s)=>{if(f()&&r!==we)return;const i=(0,S.t)(),o=ye.get(r)||"unknown";(0,x.p)(V.xV,["API/register/".concat(o,"/called")],void 0,t.K7.metrics,e.ee);try{return r(...n,s,i)}catch(e){(0,l.R)(50,e)}};return u}class Re extends M{static featureName=G.T;constructor(e){var t;super(e,G.T),t=e,(0,_.Y)(O.o5,(e,r)=>te(e,r,t),t),function(e){(0,_.Y)(O.bt,function(t){e.runtime.onerror=t},e)}(e),function(e){let t=0;(0,_.Y)(O.k6,function(e,r){++t>10||(this.runtime.releaseIds[e.slice(-200)]=(""+r).slice(-200))},e)}(e),Ee(e);try{this.removeOnAbort=new AbortController}catch(e){}this.ee.on("internal-error",(t,r)=>{this.abortHandler&&(0,x.p)("ierr",[Q(t),(0,S.t)(),!0,{},e.runtime.isRecording,r],void 0,this.featureName,this.ee)}),g.gm.addEventListener("unhandledrejection",t=>{this.abortHandler&&(0,x.p)("err",[X(t),(0,S.t)(),!1,{unhandledPromiseRejection:1},e.runtime.isRecording],void 0,this.featureName,this.ee)},(0,q.jT)(!1,this.removeOnAbort?.signal)),g.gm.addEventListener("error",t=>{this.abortHandler&&(0,x.p)("err",[J(t),(0,S.t)(),!1,{},e.runtime.isRecording],void 0,this.featureName,this.ee)},(0,q.jT)(!1,this.removeOnAbort?.signal)),this.abortHandler=this.#n,this.importAggregator(e,()=>r.e(478).then(r.bind(r,9377)))}#n(){this.removeOnAbort?.abort(),this.abortHandler=void 0}}var Ae=r(8990);let Te=1;function xe(e){const t=typeof e;return!e||"object"!==t&&"function"!==t?-1:e===g.gm?0:(0,Ae.I)(e,"nr@id",function(){return Te++})}function Se(e){if("string"==typeof e)return e.length;if("object"==typeof e){if("undefined"!=typeof ArrayBuffer&&e instanceof ArrayBuffer&&e.byteLength)return e.byteLength;if("undefined"!=typeof Blob&&e instanceof Blob&&e.size)return e.size;if(!("undefined"!=typeof FormData&&e instanceof FormData))try{return(0,Y.A)(e).length}catch(e){return}}}var Oe=r(8139),_e=r(3434);const Ne={},Pe=["open","send","setRequestHeader"];function Le(e,t){var r=e||w.ee;const n=function(e){return(e||w.ee).get("xhr")}(r);if(void 0===g.gm.XMLHttpRequest)return n;if(Ne[n.debugId]++)return n;Ne[n.debugId]=1,(0,Oe.u)(r);var s=(0,_e.YM)(n),i=g.gm.XMLHttpRequest,o=g.gm.MutationObserver,a=g.gm.Promise,c=g.gm.setInterval,d="readystatechange",u=["onload","onerror","onabort","onloadstart","onloadend","onprogress","ontimeout"],f=[],h=g.gm.XMLHttpRequest=function(e){const r=new i(e),o=n.context(r);o.targets=(0,re.$5)(t);try{n.emit("new-xhr",[r],o),r.addEventListener(d,(a=o,function(){var e=this;e.readyState>3&&!a.resolved&&(a.resolved=!0,n.emit("xhr-resolved",[],e)),s.inPlace(e,u,"fn-",E)}),(0,q.jT)(!1))}catch(e){(0,l.R)(15,e);try{n.emit("internal-error",[e])}catch(e){}}var a;return r};function p(e,t){s.inPlace(t,["onreadystatechange"],"fn-",E)}if(function(e,t){for(var r in e)t[r]=e[r]}(i,h),h.prototype=i.prototype,s.inPlace(h.prototype,Pe,"-xhr-",E),n.on("send-xhr-start",function(e,t){p(e,t),function(e){f.push(e),o&&(m?m.then(b):c?c(b):(v=-v,y.data=v))}(t)}),n.on("open-xhr-start",p),o){var m=a&&a.resolve();if(!c&&!a){var v=1,y=document.createTextNode(v);new o(b).observe(y,{characterData:!0})}}else r.on("fn-end",function(e){e[0]&&e[0].type===d||b()});function b(){for(var e=0;e<f.length;e++)p(0,f[e]);f.length&&(f=[])}function E(e,t){return t}return n}var ke="fetch-",Ce=ke+"body-",je=["arrayBuffer","blob","json","text","formData"],Ie=g.gm.Request,De=g.gm.Response,Me="prototype";const Be={};function He(e,t){const r=function(e){return(e||w.ee).get("fetch")}(e);if(!(Ie&&De&&g.gm.fetch))return r;if(Be[r.debugId]++)return r;function n(e,n,s){var i=e[n];"function"==typeof i&&(e[n]=function(){var e=[...arguments];const n={},o=(0,re.$5)(t);var a;r.emit(s+"before-start",[e],n),n[w.P]&&n[w.P].dt&&(a=n[w.P].dt);var c=i.apply(this,e);return r.emit(s+"start",[e,a],c),c.then(function(e){return r.emit(s+"end",[null,e,o],c),e},function(e){throw r.emit(s+"end",[e,void 0,o],c),e})})}return Be[r.debugId]=1,je.forEach(e=>{n(Ie[Me],e,Ce),n(De[Me],e,Ce)}),n(g.gm,"fetch",ke),r.on(ke+"end",function(e,t,n){var s=this;if(s.targets=n||[void 0],t){var i=t.headers.get("content-length");null!==i&&(s.rxSize=i),r.emit(ke+"done",[null,t],s)}else r.emit(ke+"done",[e],s)}),r}var We=r(7485);class Ke{constructor(e){this.agentRef=e}generateTracePayload(e){const t=this.agentRef.loader_config;if(!this.shouldGenerateTrace(e)||!t)return null;var r=(t.accountID||"").toString()||null,n=(t.agentID||"").toString()||null,s=(t.trustKey||"").toString()||null;if(!r||!n)return null;var i=(0,me.ZF)(),o=(0,me.el)(),a=Date.now(),c={spanId:i,traceId:o,timestamp:a};return(e.sameOrigin||this.isAllowedOrigin(e)&&this.useTraceContextHeadersForCors())&&(c.traceContextParentHeader=this.generateTraceContextParentHeader(i,o),c.traceContextStateHeader=this.generateTraceContextStateHeader(i,a,r,n,s)),(e.sameOrigin&&!this.excludeNewrelicHeader()||!e.sameOrigin&&this.isAllowedOrigin(e)&&this.useNewrelicHeaderForCors())&&(c.newrelicHeader=this.generateTraceHeader(i,o,a,r,n,s)),c}generateTraceContextParentHeader(e,t){return"00-"+t+"-"+e+"-01"}generateTraceContextStateHeader(e,t,r,n,s){return s+"@nr=0-1-"+r+"-"+n+"-"+e+"----"+t}generateTraceHeader(e,t,r,n,s,i){if(!("function"==typeof g.gm?.btoa))return null;var o={v:[0,1],d:{ty:"Browser",ac:n,ap:s,id:e,tr:t,ti:r}};return i&&n!==i&&(o.d.tk=i),btoa((0,Y.A)(o))}shouldGenerateTrace(e){return this.agentRef.init?.distributed_tracing?.enabled&&this.isAllowedOrigin(e)}isAllowedOrigin(e){var t=!1;const r=this.agentRef.init?.distributed_tracing;if(e.sameOrigin)t=!0;else if(r?.allowed_origins instanceof Array)for(var n=0;n<r.allowed_origins.length;n++){var s=(0,We.D)(r.allowed_origins[n]);if(e.hostname===s.hostname&&e.protocol===s.protocol&&e.port===s.port){t=!0;break}}return t}excludeNewrelicHeader(){var e=this.agentRef.init?.distributed_tracing;return!!e&&!!e.exclude_newrelic_header}useNewrelicHeaderForCors(){var e=this.agentRef.init?.distributed_tracing;return!!e&&!1!==e.cors_use_newrelic_header}useTraceContextHeadersForCors(){var e=this.agentRef.init?.distributed_tracing;return!!e&&!!e.cors_use_tracecontext_headers}}var Fe=r(7295);function Ue(e){return"string"==typeof e?e:e instanceof(0,s.dV)().o.REQ?e.url:g.gm?.URL&&e instanceof URL?e.href:void 0}var Ve=["load","error","abort","timeout"],ze=Ve.length,Ge=(0,s.dV)().o.REQ,qe=(0,s.dV)().o.XHR;const Ye="X-NewRelic-App-Data",Ze="internal-error";class Qe extends M{static featureName=c.TZ;constructor(e){super(e,c.TZ),this.dt=new Ke(e),this.handler=(e,t,r,n)=>(0,x.p)(e,t,r,n,this.ee);try{const e={xmlhttprequest:"xhr",fetch:"fetch",beacon:"beacon"};g.gm?.performance?.getEntriesByType("resource").forEach(r=>{if(r.initiatorType in e&&0!==r.responseStatus){const n={status:r.responseStatus},s={rxSize:r.transferSize,duration:Math.floor(r.duration),cbTime:0};Xe(n,r.name),this.handler("xhr",[n,s,r.startTime,r.responseEnd,e[r.initiatorType]],void 0,t.K7.ajax)}})}catch(e){}He(this.ee,e),Le(this.ee,e),function(e,r,n,s){const i=[c.mo.ALL,c.mo.FAILURES].includes(e.init.ajax?.capture_payloads);function o(e){var t=this;t.totalCbs=0,t.called=0,t.cbTime=0,t.end=T,t.ended=!1,t.xhrGuids={},t.lastSize=null,t.loadCaptureCalled=!1,t.params=this.params||{},t.metrics=this.metrics||{},t.latestLongtaskEnd=0,e.addEventListener("load",function(r){_(t,e)},(0,q.jT)(!1)),g.lR||e.addEventListener("progress",function(e){t.lastSize=e.loaded},(0,q.jT)(!1))}function a(e){this.params={method:e[0]},Xe(this,e[1]),this.metrics={}}function d(t,r){e.loader_config.xpid&&this.sameOrigin&&r.setRequestHeader("X-NewRelic-ID",e.loader_config.xpid);var n=s.generateTracePayload(this.parsedOrigin);if(n){var i=!1;n.newrelicHeader&&(r.setRequestHeader("newrelic",n.newrelicHeader),i=!0),n.traceContextParentHeader&&(r.setRequestHeader("traceparent",n.traceContextParentHeader),n.traceContextStateHeader&&r.setRequestHeader("tracestate",n.traceContextStateHeader),i=!0),i&&(this.dt=n)}}function u(e,t){i&&e.length>=2&&(this.requestHeaders??={},this.requestHeaders[e[0].toLowerCase()]=e[1])}function l(e,t){var n=this.metrics,s=e[0],i=this;if(n&&s){var o=Se(s);o&&(n.txSize=o)}this.startTime=(0,S.t)(),this.requestBody=s,this.listener=function(e){try{"abort"!==e.type||i.loadCaptureCalled||(i.params.aborted=!0),("load"!==e.type||i.called===i.totalCbs&&(i.onloadCalled||"function"!=typeof t.onload)&&"function"==typeof i.end)&&i.end(t)}catch(e){try{r.emit(Ze,[e])}catch(e){}}};for(var a=0;a<ze;a++)t.addEventListener(Ve[a],this.listener,(0,q.jT)(!1))}function f(e,t,r){this.cbTime+=e,t?this.onloadCalled=!0:this.called+=1,this.called!==this.totalCbs||!this.onloadCalled&&"function"==typeof r.onload||"function"!=typeof this.end||this.end(r)}function h(e,t){var r=""+xe(e)+!!t;this.xhrGuids&&!this.xhrGuids[r]&&(this.xhrGuids[r]=!0,this.totalCbs+=1)}function p(e,t){var r=""+xe(e)+!!t;this.xhrGuids&&this.xhrGuids[r]&&(delete this.xhrGuids[r],this.totalCbs-=1)}function m(){this.endTime=(0,S.t)()}function v(e,t){t instanceof qe&&"load"===e[0]&&r.emit("xhr-load-added",[e[1],e[2]],t)}function y(e,t){t instanceof qe&&"load"===e[0]&&r.emit("xhr-load-removed",[e[1],e[2]],t)}function b(e,t,r){t instanceof qe&&("onload"===r&&(this.onload=!0),("load"===(e[0]&&e[0].type)||this.onload)&&(this.xhrCbStart=(0,S.t)()))}function E(e,t){this.xhrCbStart&&r.emit("xhr-cb-time",[(0,S.t)()-this.xhrCbStart,this.onload,t],t)}function w(e){var t,r=e[1]||{};if("string"==typeof e[0]?0===(t=e[0]).length&&g.RI&&(t=""+g.gm.location.href):e[0]&&e[0].url?t=e[0].url:g.gm?.URL&&e[0]&&e[0]instanceof URL?t=e[0].href:"function"==typeof e[0].toString&&(t=e[0].toString()),"string"==typeof t&&0!==t.length){t&&(this.parsedOrigin=(0,We.D)(t),this.sameOrigin=this.parsedOrigin.sameOrigin);var n=s.generateTracePayload(this.parsedOrigin);if(n&&(n.newrelicHeader||n.traceContextParentHeader))if(e[0]&&e[0].headers)a(e[0].headers,n)&&(this.dt=n);else{var i={};for(var o in r)i[o]=r[o];i.headers=new Headers(r.headers||{}),a(i.headers,n)&&(this.dt=n),e.length>1?e[1]=i:e.push(i)}}function a(e,t){var r=!1;return t.newrelicHeader&&(e.set("newrelic",t.newrelicHeader),r=!0),t.traceContextParentHeader&&(e.set("traceparent",t.traceContextParentHeader),t.traceContextStateHeader&&e.set("tracestate",t.traceContextStateHeader),r=!0),r}}function R(e,t){this.params={},this.metrics={},this.startTime=(0,S.t)(),this.dt=t;let[r,n={}]=e;Xe(this,Ue(r));const s=(""+(r&&r instanceof Ge&&r.method||n.method||"GET")).toUpperCase();this.params.method=s,this.txSize=Se(n.body||r?.body)||0;try{var o=n.headers||r?.headers;if(i&&o)if(this.requestHeaders??={},o instanceof Headers)o.forEach(function(e,t){this.requestHeaders[t.toLowerCase()]=e}.bind(this));else if("object"==typeof o)for(var a in o)this.requestHeaders[a.toLowerCase()]=o[a]}catch(e){}this.requestBody=n.body||r?.body}function A(e,t){if(this.endTime=(0,S.t)(),this.params||(this.params={}),(0,Fe.iW)(this.params))return;this.params.status=t?t.status:0;const n=()=>{const e=+this.rxSize,t=null==this.rxSize||isNaN(e)?void 0:e,r={txSize:this.txSize,rxSize:t,duration:this.endTime-this.startTime},n=[this.params,r,this.startTime,this.endTime,"fetch"];this.targets.forEach(e=>O(n,this,e))};t&&i?t.clone().text().then(e=>{this.responseBody=e,this.rxSize&&"0"!==this.rxSize&&0!==this.rxSize||void 0===e||0===this.params.status||(this.rxSize=Se(e)),t?.headers&&(this.responseHeaders={},t.headers.forEach(function(e,t){this.responseHeaders[t.toLowerCase()]=e}.bind(this)))}).catch(e=>{r.emit(Ze,[e])}).finally(()=>{n()}):n()}function T(e){const t=this.params,n=this.metrics;if(this.ended)return;this.ended=!0;for(let t=0;t<ze;t++)e.removeEventListener(Ve[t],this.listener,!1);if(t.aborted)return;if((0,Fe.iW)(t))return;if(n.duration=this.endTime-this.startTime,this.loadCaptureCalled||4!==e.readyState?null==t.status&&(t.status=0):_(this,e),n.cbTime=this.cbTime,i){try{this.responseBody=e.responseText}catch(t){this.responseBody=e.response}if((!n.rxSize||0===n.rxSize)&&void 0!==this.responseBody&&0!==t.status){const e=Se(this.responseBody);void 0!==e&&(n.rxSize=e)}try{this.responseHeaders=function(e){const t={};return e?(e.split("\r\n").forEach(function(e){const r=e.indexOf(": ");if(r>0){const n=e.substring(0,r),s=e.substring(r+2);t[n.toLowerCase()]=s}}),t):t}(e.getAllResponseHeaders())}catch(e){r.emit(Ze,[e])}}const s=[t,n,this.startTime,this.endTime,"xhr"];this.targets.forEach(e=>O(s,this,e))}function O(e,r,s){n("xhr",[...e,s],r,t.K7.ajax)}function _(e,n){e.params.status=n.status;var s=function(e,t){var r=e.responseType;return"json"===r&&null!==t?t:"arraybuffer"===r||"blob"===r||"json"===r?Se(e.response):"text"===r||""===r||void 0===r?Se(e.responseText):void 0}(n,e.lastSize);if(void 0!==s&&0!==n.status&&(e.metrics.rxSize=s),e.sameOrigin&&n.getAllResponseHeaders().indexOf(Ye)>=0){var i=n.getResponseHeader(Ye);i&&((0,x.p)("sm",["Ajax/CrossApplicationTracing/Header/Seen"],void 0,t.K7.metrics,r),e.params.cat=i.split(", ").pop())}e.loadCaptureCalled=!0}r.on("new-xhr",o),r.on("open-xhr-start",a),r.on("open-xhr-end",d),r.on("send-xhr-start",l),r.on("setRequestHeader-xhr-start",u),r.on("xhr-cb-time",f),r.on("xhr-load-added",h),r.on("xhr-load-removed",p),r.on("xhr-resolved",m),r.on("addEventListener-end",v),r.on("removeEventListener-end",y),r.on("fn-end",E),r.on("fetch-before-start",w),r.on("fetch-start",R),r.on("fn-start",b),r.on("fetch-done",A)}(e,this.ee,this.handler,this.dt),this.importAggregator(e,()=>r.e(478).then(r.bind(r,3845)))}}function Xe(e,t){var r=(0,We.D)(t),n=e.params||e;n.hostname=r.hostname,n.port=r.port,n.protocol=r.protocol,n.host=r.hostname+":"+r.port,n.pathname=r.pathname,e.parsedOrigin=r,e.sameOrigin=r.sameOrigin}const $e={},Je=["pushState","replaceState"];function et(e){const t=function(e){return(e||w.ee).get("history")}(e);return!g.RI||$e[t.debugId]++||($e[t.debugId]=1,(0,_e.YM)(t).inPlace(window.history,Je,"-")),t}var tt=r(3738);function rt(e){(0,_.Y)(O.BL,function(r=Date.now()){const n=r-g.WN;n<0&&(0,l.R)(62,r),(0,x.p)(V.XG,[O.BL,{time:n}],void 0,t.K7.metrics,e.ee),e.addToTrace({name:O.BL,start:r,origin:"nr"}),(0,x.p)("api-"+O.hG,[n,O.BL],void 0,t.K7.genericEvents,e.ee)},e)}const{He:nt,bD:st,d3:it,Kp:ot,TZ:at,Lc:ct,uP:dt,Rz:ut}=tt;class lt extends M{static featureName=at;constructor(e){var n;super(e,at),n=e,(0,_.Y)(O.U2,function(e){if(!(e&&"object"==typeof e&&e.name&&e.start))return;const r={n:e.name,s:e.start-g.WN,e:(e.end||e.start)-g.WN,o:e.origin||"",t:"api"};r.s<0||r.e<0||r.e<r.s?(0,l.R)(61,{start:r.s,end:r.e}):(0,x.p)("bstApi",[r],void 0,t.K7.sessionTrace,n.ee)},n),rt(e);if(!(0,C.V)(e.init))return void this.deregisterDrain();const s=this.ee;let i;et(s),this.eventsEE=(0,Oe.u)(s),this.eventsEE.on(dt,function(e,t){this.bstStart=(0,S.t)()}),this.eventsEE.on(ct,function(e,r){(0,x.p)("bst",[e[0],r,this.bstStart,(0,S.t)()],void 0,t.K7.sessionTrace,s)}),s.on(ut+it,function(e){this.time=(0,S.t)(),this.startPath=location.pathname+location.hash}),s.on(ut+ot,function(e){(0,x.p)("bstHist",[location.pathname+location.hash,this.startPath,this.time],void 0,t.K7.sessionTrace,s)});try{i=new PerformanceObserver(e=>{const r=e.getEntries();(0,x.p)(nt,[r],void 0,t.K7.sessionTrace,s)}),i.observe({type:st,buffered:!0})}catch(e){}this.importAggregator(e,()=>r.e(478).then(r.bind(r,6974)),{resourceObserver:i})}}var ft=r(733),ht=r(6344);class pt extends M{static featureName=ht.TZ;#s;recorder;constructor(e){var n;let s;super(e,ht.TZ),n=e,(0,_.Y)(O.CH,function(){(0,x.p)(O.CH,[],void 0,t.K7.sessionReplay,n.ee)},n),function(e){(0,_.Y)(O.Tb,function(){(0,x.p)(O.Tb,[],void 0,t.K7.sessionReplay,e.ee)},e)}(e);const i="".concat(u.Wt).concat((0,ft.Y)(e.info.licenseKey,e.info.applicationID));try{s=JSON.parse(localStorage.getItem(i))}catch(e){}(0,k.SR)(e.init)&&this.ee.on(O.CH,()=>this.#i()),this.#o(s)&&this.importRecorder().then(e=>{e.startRecording(ht.Qb.PRELOAD,s?.sessionReplayMode)}),this.importAggregator(this.agentRef,()=>r.e(478).then(r.bind(r,6167)),this),this.ee.on("err",e=>{this.blocked||this.agentRef.runtime.isRecording&&(this.errorNoticed=!0,(0,x.p)(ht.Vh,[e],void 0,this.featureName,this.ee))})}#o(e){return e&&(e.sessionReplayMode===u.g.FULL||e.sessionReplayMode===u.g.ERROR)||(0,k.Aw)(this.agentRef.init)}importRecorder(){return this.recorder?Promise.resolve(this.recorder):(this.#s??=Promise.all([r.e(478),r.e(249)]).then(r.bind(r,4866)).then(({Recorder:e})=>(this.recorder=new e(this),this.recorder)).catch(e=>{throw this.ee.emit("internal-error",[e]),this.blocked=!0,e}),this.#s)}#i(){this.blocked||(this.featAggregate?this.featAggregate.mode!==u.g.FULL&&this.featAggregate.initializeRecording(u.g.FULL,!0,ht.Qb.API):this.importRecorder().then(()=>{this.recorder.startRecording(ht.Qb.API,u.g.FULL)}))}}var gt=r(3962);class mt extends M{static featureName=gt.TZ;constructor(e){if(super(e,gt.TZ),function(e){const r=e.ee.get("tracer");function n(){}(0,_.Y)(O.dT,function(e){return(new n).get("object"==typeof e?e:{})},e);const s=n.prototype={createTracer:function(n,s){var i={},o=this,a="function"==typeof s;return(0,x.p)(V.xV,["API/createTracer/called"],void 0,t.K7.metrics,e.ee),function(){if(r.emit((a?"":"no-")+"fn-start",[(0,S.t)(),o,a],i),a)try{return s.apply(this,arguments)}catch(e){const t="string"==typeof e?new Error(e):e;throw r.emit("fn-err",[arguments,this,t],i),t}finally{r.emit("fn-end",[(0,S.t)()],i)}}}};["actionText","setName","setAttribute","save","ignore","onEnd","getContext","end","get"].forEach(r=>{_.Y.apply(this,[r,function(){return(0,x.p)(O.hw+r,[performance.now(),...arguments],this,t.K7.softNav,e.ee),this},e,s])}),(0,_.Y)(O.PA,function(){(0,x.p)(O.hw+"routeName",[performance.now(),...arguments],void 0,t.K7.softNav,e.ee)},e)}(e),!g.RI||!(0,s.dV)().o.MO)return;const n=et(this.ee);try{this.removeOnAbort=new AbortController}catch(e){}gt.tC.forEach(e=>{(0,q.sp)(e,e=>{c(e)},!0,this.removeOnAbort?.signal)});const i=()=>(0,x.p)("newURL",[(0,S.t)(),""+window.location],void 0,this.featureName,this.ee);n.on("pushState-end",i),n.on("replaceState-end",i),(0,q.sp)(gt.OV,e=>{c(e),(0,x.p)("newURL",[e.timeStamp,""+window.location],void 0,this.featureName,this.ee)},!0,this.removeOnAbort?.signal);let o=!1;const a=new((0,s.dV)().o.MO)((e,t)=>{o||(o=!0,requestAnimationFrame(()=>{(0,x.p)("newDom",[(0,S.t)()],void 0,this.featureName,this.ee),o=!1}))}),c=(0,j.s)(e=>{"loading"!==document.readyState&&((0,x.p)("newUIEvent",[e],void 0,this.featureName,this.ee),a.observe(document.body,{attributes:!0,childList:!0,subtree:!0,characterData:!0}))},100,{leading:!0});this.abortHandler=function(){this.removeOnAbort?.abort(),a.disconnect(),this.abortHandler=void 0},this.importAggregator(e,()=>r.e(478).then(r.bind(r,4393)),{domObserver:a})}}var vt=r(9119),yt=r(981);const bt={},Et=new Set;function wt(e){return"string"==typeof e?{type:"string",size:(new TextEncoder).encode(e).length}:e instanceof ArrayBuffer?{type:"ArrayBuffer",size:e.byteLength}:e instanceof Blob?{type:"Blob",size:e.size}:e instanceof DataView?{type:"DataView",size:e.byteLength}:ArrayBuffer.isView(e)?{type:"TypedArray",size:e.byteLength}:{type:"unknown",size:0}}class Rt{constructor(e,t){this.timestamp=(0,S.t)(),this.currentUrl=(0,vt.L)(window.location.href),this.socketId=(0,me.LA)(8),this.requestedUrl=(0,vt.L)(e),this.requestedProtocols=Array.isArray(t)?t.join(","):t||"",this.openedAt=void 0,this.protocol=void 0,this.extensions=void 0,this.binaryType=void 0,this.messageOrigin=void 0,this.messageCount=0,this.messageBytes=0,this.messageBytesMin=0,this.messageBytesMax=0,this.messageTypes=void 0,this.sendCount=0,this.sendBytes=0,this.sendBytesMin=0,this.sendBytesMax=0,this.sendTypes=void 0,this.closedAt=void 0,this.closeCode=void 0,this.closeReason="unknown",this.closeWasClean=void 0,this.connectedDuration=0,this.hasErrors=void 0}}class At extends M{static featureName=a.TZ;constructor(e){super(e,a.TZ);const n=e.init.feature_flags.includes("websockets")||e.init.web_sockets?.enabled,i=!e.init.feature_flags.includes("no_spv"),o=[e.init.page_action.enabled,e.init.performance.capture_marks,e.init.performance.capture_measures,e.init.performance.resources.enabled,e.init.user_actions.enabled,n,i];var c;let d;if(c=e,(0,_.Y)(O.hG,(e,t)=>oe(e,t,c),c),function(e){(0,_.Y)(O.fF,(t,r)=>ce(t,r,e),e)}(e),rt(e),Ee(e),function(e){(0,_.Y)(O.V1,(t,r)=>ae(t,r,e),e)}(e),this.removeOnAbort=new AbortController,this.abortHandler=()=>{this.removeOnAbort.abort(),this.abortHandler=void 0},n){const u=function(e,t){if(!(0,s.dV)().o.WS)return e;const r=e.get("websockets");if(bt[r.debugId]++)return r;bt[r.debugId]=1,(0,K.G)(()=>{const e=(0,S.t)();Et.forEach(t=>{t.nrData.closedAt=e,t.nrData.closeCode=1001,t.nrData.closeReason="Page navigating away",t.nrData.closeWasClean=!1,t.nrData.openedAt&&(t.nrData.connectedDuration=e-t.nrData.openedAt),r.emit("ws",[t.nrData,t.targets],t)})});class n extends WebSocket{static name=yt.R.WS;static toString(){return"function WebSocket() { [native code] }"}toString(){return"[object WebSocket]"}get[Symbol.toStringTag](){return n.name}#a(e){(e.__newrelic??={}).socketId=this.nrData.socketId,this.nrData.hasErrors??=!0}constructor(...e){super(...e),this.nrData=new Rt(e[0],e[1]),this.targets=(0,re.$5)(t),this.addEventListener("open",()=>{this.nrData.openedAt=(0,S.t)(),["protocol","extensions","binaryType"].forEach(e=>{this.nrData[e]=this[e]}),Et.add(this)}),this.addEventListener("message",e=>{const{type:t,size:r}=wt(e.data);this.nrData.messageOrigin??=(0,vt.L)(e.origin),this.nrData.messageCount++,this.nrData.messageBytes+=r,this.nrData.messageBytesMin=Math.min(this.nrData.messageBytesMin||1/0,r),this.nrData.messageBytesMax=Math.max(this.nrData.messageBytesMax,r),(this.nrData.messageTypes??"").includes(t)||(this.nrData.messageTypes=this.nrData.messageTypes?"".concat(this.nrData.messageTypes,",").concat(t):t)}),this.addEventListener("close",e=>{this.nrData.closedAt=(0,S.t)(),this.nrData.closeCode=e.code,e.reason&&(this.nrData.closeReason=e.reason),this.nrData.closeWasClean=e.wasClean,this.nrData.connectedDuration=this.nrData.closedAt-this.nrData.openedAt,Et.delete(this),r.emit("ws",[this.nrData,this.targets],this)})}addEventListener(e,t,...r){const n=this,s="function"==typeof t?function(...e){try{return t.apply(this,e)}catch(e){throw n.#a(e),e}}:t?.handleEvent?{handleEvent:function(...e){try{return t.handleEvent.apply(t,e)}catch(e){throw n.#a(e),e}}}:t;return super.addEventListener(e,s,...r)}send(e){if(this.readyState===WebSocket.OPEN){const{type:t,size:r}=wt(e);this.nrData.sendCount++,this.nrData.sendBytes+=r,this.nrData.sendBytesMin=Math.min(this.nrData.sendBytesMin||1/0,r),this.nrData.sendBytesMax=Math.max(this.nrData.sendBytesMax,r),(this.nrData.sendTypes??"").includes(t)||(this.nrData.sendTypes=this.nrData.sendTypes?"".concat(this.nrData.sendTypes,",").concat(t):t)}try{return super.send(e)}catch(e){throw this.#a(e),e}}close(...e){try{super.close(...e)}catch(e){throw this.#a(e),e}}}return g.gm.WebSocket=n,r}(this.ee,e);u.on("ws",(e,t)=>{(0,x.p)("ws-complete",[e,t],void 0,this.featureName,this.ee)})}if(i&&g.gm.addEventListener("securitypolicyviolation",e=>{(0,x.p)("spv",[e],void 0,t.K7.genericEvents,this.ee)},(0,q.jT)(!1,this.removeOnAbort.signal)),g.RI){if(He(this.ee,e),Le(this.ee,e),d=et(this.ee),e.init.user_actions.enabled){function l(t){const r=(0,We.D)(t);return e.beacons.includes(r.hostname+":"+r.port)}function f(){d.emit("navChange")}a.Zp.forEach(e=>(0,q.sp)(e,e=>(0,x.p)("ua",[e],void 0,this.featureName,this.ee),!0)),a.qN.forEach(e=>{const t=(0,j.s)(e=>{(0,x.p)("ua",[e],void 0,this.featureName,this.ee)},500,{leading:!0});(0,q.sp)(e,t)}),g.gm.addEventListener("error",()=>{(0,x.p)("uaErr",[],void 0,t.K7.genericEvents,this.ee)},(0,q.jT)(!1,this.removeOnAbort.signal)),this.ee.on("open-xhr-start",(e,r)=>{l(e[1])||r.addEventListener("readystatechange",()=>{2===r.readyState&&(0,x.p)("uaXhr",[],void 0,t.K7.genericEvents,this.ee)},(0,q.jT)(void 0,this.removeOnAbort.signal))}),this.ee.on("fetch-start",e=>{e.length>=1&&!l(Ue(e[0]))&&(0,x.p)("uaXhr",[],void 0,t.K7.genericEvents,this.ee)}),d.on("pushState-end",f),d.on("replaceState-end",f),window.addEventListener("hashchange",f,(0,q.jT)(!0,this.removeOnAbort.signal)),window.addEventListener("popstate",f,(0,q.jT)(!0,this.removeOnAbort.signal))}if(e.init.performance.resources.enabled&&g.gm.PerformanceObserver?.supportedEntryTypes.includes("resource")){new PerformanceObserver(e=>{e.getEntries().forEach(e=>{(0,x.p)("browserPerformance.resource",[e],void 0,this.featureName,this.ee)})}).observe({type:"resource",buffered:!0})}}o.some(e=>e)?this.importAggregator(e,()=>r.e(478).then(r.bind(r,8019))):this.deregisterDrain()}}var Tt=r(2646);const xt=new Map;function St(e,t,r,n,s=!0,i){if("object"!=typeof t||!t||"string"!=typeof r||!r||"function"!=typeof t[r])return(0,l.R)(29);const o=function(e){return(e||w.ee).get("logger")}(e),a=(0,_e.YM)(o,void 0,i),c=new Tt.y(w.P);c.level=n.level,c.customAttributes=n.customAttributes,c.autoCaptured=s;const d=t[r]?.[_e.Jt]||t[r];return xt.set(d,c),a.inPlace(t,[r],"wrap-logger-",()=>xt.get(d),void 0,!0),o}var Ot=r(1910);class _t extends M{static featureName=ne.TZ;constructor(e){var t;super(e,ne.TZ),t=e,(0,_.Y)("log",(e,r)=>ie(e,r,t),t),function(e){(0,_.Y)(O.Wb,(t,r,{customAttributes:n={},level:s=ne.p_.INFO}={})=>{St(e.ee,t,r,{customAttributes:n,level:s},!1,e)},e)}(e),Ee(e);const n=this.ee;["log","error","warn","info","debug","trace"].forEach(t=>{(0,Ot.i)(g.gm.console[t]),St(n,g.gm.console,t,{level:"log"===t?"info":t},void 0,e)}),this.ee.on("wrap-logger-end",function([e],t,r,s=[]){const{level:i,customAttributes:o,autoCaptured:a}=this;s.forEach(t=>{(0,se.R)(n,e,o,i,a,t)})}),this.importAggregator(e,()=>r.e(478).then(r.bind(r,5288)))}}new W({features:[Qe,H,U,lt,pt,z,Re,At,_t,mt],loaderType:"spa"})})()})();
|
|
</script>
|
|
</head>
|
|
<body class="content">
|
|
|
|
|
|
<div id="top-nav" class="sticky top-0 z-10 bg-tezify-50 px-3 md:px-2 py-3 border-b border-tezify-275">
|
|
<div class="grid grid-cols-4 items-center place-contents-between">
|
|
|
|
<div class="md:pl-2 col-span-2 md:col-span-3">
|
|
<a href="/">Home</a> <span class="text-tezify-500 md:mx-1">/</span> <a class="blue-link" href=/blog/>Tech Posts</a>
|
|
</div>
|
|
<div class="justify-self-end align-middle md:pr-2 col-span-2 md:col-span-1">
|
|
<a class="hidden md:flex px-3 py-0.5 border border-contrast-700 text-contrast-700 rounded-sm hover:bg-contrast-100" href="/services/">
|
|
<img src="/images/hand-waving.png" alt="Hire Me" class="w-4 h-4 m-1"> Hire Me
|
|
</a>
|
|
<a class="flex md:hidden px-3 py-1 border border-contrast-700 text-contrast-700 rounded-sm hover:bg-contrast-100" href="/services/">
|
|
<img src="/images/hand-waving.png" alt="Hire Me" class="w-4 h-4 m-1"> Hire Me
|
|
</a>
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
<style>
|
|
.shadown-on-scrolling {
|
|
box-shadow: rgba(0, 0, 0, 0.1) 0px 0px 15px 0px, rgba(0, 0, 0, 0.04) 0px 10px 10px -5px;
|
|
border-bottom: solid 1px #CCC8D0;
|
|
}
|
|
</style>
|
|
<script>
|
|
window.addEventListener('scroll', function() {
|
|
100 < document.body.scrollTop || 100 < document.documentElement.scrollTop ? document.getElementById("top-nav").classList.add("shadown-on-scrolling") : document.getElementById("top-nav").classList.remove("shadown-on-scrolling")
|
|
});
|
|
|
|
|
|
document.addEventListener('DOMContentLoaded', function() {
|
|
const settingsButton = document.getElementById('settings-menu-button');
|
|
const settingsDropdown = document.getElementById('settings-dropdown');
|
|
const settingsCaret = document.getElementById('settings-caret');
|
|
|
|
if (settingsButton && settingsDropdown && settingsCaret) {
|
|
settingsButton.addEventListener('click', function(e) {
|
|
e.preventDefault();
|
|
e.stopPropagation();
|
|
|
|
|
|
settingsDropdown.classList.toggle('hidden');
|
|
|
|
|
|
if (settingsDropdown.classList.contains('hidden')) {
|
|
settingsCaret.style.transform = 'rotate(0deg)';
|
|
} else {
|
|
settingsCaret.style.transform = 'rotate(180deg)';
|
|
}
|
|
});
|
|
|
|
|
|
document.addEventListener('click', function(e) {
|
|
if (!settingsButton.contains(e.target) && !settingsDropdown.contains(e.target)) {
|
|
settingsDropdown.classList.add('hidden');
|
|
settingsCaret.style.transform = 'rotate(0deg)';
|
|
}
|
|
});
|
|
}
|
|
});
|
|
</script>
|
|
|
|
|
|
|
|
|
|
<div class="bg-tezify-100-w-noise pb-8 lg:min-h-screen">
|
|
<div class="mx-auto top-0 md:top-20 md:pb-16 rounded relative md:max-w-screen-md lg:max-w-screen-lg xl:max-w-screen-xl grid grid-cols-1 xl:grid-cols-12 bg-white xl:bg-tezify-100-w-noise">
|
|
<div class="m-2 p-2 md:py-2 md:px-6 xl:col-span-8 bg-transparent xl:bg-white xl:border xl:border-tezify-350 xl:rounded">
|
|
|
|
|
|
|
|
|
|
<div class="blog-content my-4">
|
|
|
|
<h1 id="how-we-built-observability-with-google-cloud-services">How we built observability with Google Cloud services</h1>
|
|
<p>
|
|
|
|
<div class="mb-8 text-sm text-tezify-700">30th Dec, 2024</div>
|
|
|
|
<div class="mb-8">
|
|
<picture>
|
|
|
|
<source srcset='/images/observability-v3.png' type="image/jpeg">
|
|
|
|
<img loading='eager' src='/images/observability-v3.png' alt='How we built observability with Google Cloud services' title='How we built observability with Google Cloud services' width='100%' class=" mt-2 mx-auto" style='max-width:100%;aspect-ratio: 2.1084;'/>
|
|
|
|
</picture>
|
|
|
|
</div></p>
|
|
<p>A common approach to addressing a production issue is to try to reproduce it in a non-production environment. Once reproduced, the fix can be coded and tested in the non-production environment before applying to production.</p>
|
|
<p>While this approach worked well for most of our issues, it left two vital gaps:</p>
|
|
<ul>
|
|
<li>There were always some issues (functional bugs, slowness, crashes) that we could not reproduce and thus not fix.</li>
|
|
<li>When seeing an issue in production, we could not decipher how wide-spread a given issue was without an in-depth analysis (e.g. - happening for a single user or many users, happening for a moment, or throughout the day).</li>
|
|
</ul>
|
|
<p>To address these two gaps, we decided to build better observability into our production setup.</p>
|
|
<h2 id="observability-tools--why-we-picked-google-cloud-offerings">Observability tools & why we picked Google Cloud Offerings</h2>
|
|
<p>The observability market has various products that could cater to our requirement:</p>
|
|
<ul>
|
|
<li>Paid product suites like Datadog, NewRelic, and Dynatrace are feature-rich and easier to integrate but were out of our contention due to budgetary constraints.</li>
|
|
<li>Grafana + Prometheus + Loki and Kibana APM (ELK stack) are free, but their integration involved a substantial effort (especially with respect to some legacy components of our setup).</li>
|
|
</ul>
|
|
<p>Since our production setup already ran on Google Cloud, we evaluated if we could leverage Google Cloud’s tracing, logging & monitoring features to build observability. And, it turned out that from an effort & pricing vs benefit perspective, it worked out the best among the options for us.</p>
|
|
<p>A downside to choosing Google Cloud was that it would make us more cloud provider dependent. But, since we were not planning to migrate out of Google Cloud for multiple years, we decided to live with this limitation.</p>
|
|
<h2 id="our-existing-google-cloud-logging-and-monitoring-setup">Our existing Google Cloud logging and monitoring setup</h2>
|
|
<p>Our production setup running on Google Cloud consisted of various instance groups with autoscaling rules. A Google Cloud application load balancer received each of our requests and routed them to one of these instance groups. The requests coming to and responses sent by this load balancer were already being monitored via Google Log Explorer and Metrics Explorer.</p>
|
|
<ul>
|
|
<li>The Logs Explorer contained the log entries with details like request URL, response code, backend latency, etc. for each request served via the load balancer.</li>
|
|
<li>The Metrics Explorer was used to monitor requests and responses (# of requests grouped by response code, backend latency, etc.) as well as instance group resources (CPU, memory, network, etc.).</li>
|
|
</ul>
|
|
<h2 id="the-missing-piece-traceability">The missing piece: Traceability</h2>
|
|
<p>Once a request from a user’s device reaches our setup (let’s say, to serve a web page for <code>https://www.mysite.com/products</code>), multiple requests to different sub-systems within our setup are made to formulate the response.</p>
|
|
<div class="mb-8">
|
|
<picture>
|
|
|
|
<source srcset='/images/observability-multiple-services.webp' type="image/webp">
|
|
|
|
<source srcset='/images/observability-multiple-services.png' type="image/jpeg">
|
|
|
|
<img loading='lazy' src='/images/observability-multiple-services.png' alt='Our frontend server invokes multiple service calls to formulate a response' title='Our frontend server invokes multiple service calls to formulate a response' width='100%' class=" mt-2 mx-auto" style='max-width:100%;aspect-ratio: 2.0262;'/>
|
|
|
|
</picture>
|
|
|
|
</div>
|
|
<p>While our logging and metrics system reported each of these requests (the one coming from the user’s device and the ones made within our setup to formulate the response), there was no way to tie these related requests together. So, in a production setup with thousands of requests every minute, if a request for <code>https://www.mysite.com/products</code> errored, there was no way to determine which of the sub-systems caused the error.</p>
|
|
<p>If we could link the first incoming request for <code>https://www.mysite.com/products</code> to all the consecutive sub-system requests made to formulate the response, we could easily identify which sub-system caused an unexpected behavior.</p>
|
|
<p>Setting up traceability would solve this for us.</p>
|
|
<h2 id="our-traceability-implementation">Our Traceability implementation</h2>
|
|
<h3 id="leveraging-the-opentelemetry-integration">Leveraging the OpenTelemetry integration</h3>
|
|
<p>If we could set up <a target="_blank" href="https://cloud.google.com/monitoring/agent/ops-agent">Google Cloud Ops Agent</a> on our instances, we could then use OpenTelemetry (an open standard) to emit traceability information. So, we configured our instances to have the Ops Agent running.</p>
|
|
<p>We found that OpenTelemetry libraries exist for a very large number of languages (<a target="_blank" href="https://www.npmjs.com/package/@opentelemetry/sdk-trace-node">Node</a>, <a target="_blank" href="https://github.com/open-telemetry/opentelemetry-php">PHP</a>, others), frameworks (<a target="_blank" href="https://github.com/opentelemetry-php/contrib-auto-symfony">Symfony</a>, <a target="_blank" href="https://www.npmjs.com/package/@opentelemetry/instrumentation-express">Express</a>, others) and logging libraries (<a target="_blank" href="https://packagist.org/packages/open-telemetry/opentelemetry-auto-psr3">PSR-3</a>, <a target="_blank" href="https://www.npmjs.com/package/@opentelemetry/instrumentation-winston">Winston</a>, others). We integrated these OpenTelemetry libraries within the various services depending on the existing language, framework and logging in use.</p>
|
|
<h3 id="setting-up-sampling-to-keep-traceability-overhead-and-costs-in-check">Setting up sampling to keep traceability overhead and costs in check</h3>
|
|
<p>We wanted to ensure that our traceability setup wouldn’t add notable performance overhead or cause cost overruns. So, we configured our trace libraries to sample a small percent of requests:</p>
|
|
<div class="highlight"><pre tabindex="0" class="chroma"><code class="language-JavaScript" data-lang="JavaScript"><span class="kr">import</span> <span class="p">{</span> <span class="nx">TraceIdRatioBasedSampler</span> <span class="p">}</span> <span class="nx">from</span> <span class="s1">'@opentelemetry/sdk-trace-node'</span><span class="p">;</span>
|
|
|
|
<span class="kr">const</span> <span class="nx">samplePerc</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">;</span>
|
|
|
|
<span class="kr">const</span> <span class="nx">sdk</span> <span class="o">=</span> <span class="k">new</span> <span class="nx">NodeSDK</span><span class="p">({</span>
|
|
<span class="c1">//...
|
|
</span><span class="c1"></span> <span class="c1">//...
|
|
</span><span class="c1"></span> <span class="nx">sampler</span><span class="o">:</span> <span class="k">new</span> <span class="nx">TraceIdRatioBasedSampler</span><span class="p">(</span><span class="nx">samplePerc</span><span class="p">),</span>
|
|
<span class="p">});</span>
|
|
</code></pre></div><p>With the OpenTelemetry libraries integration and Google Cloud Ops agents setup, we could view the trace information within Google Cloud Trace Explorer for the sampled requests.</p>
|
|
<div class="mb-8">
|
|
<picture>
|
|
|
|
<source srcset='/images/google-trace-explorer-screenshot.webp' type="image/webp">
|
|
|
|
<source srcset='/images/google-trace-explorer-screenshot.png' type="image/jpeg">
|
|
|
|
<img loading='lazy' src='/images/google-trace-explorer-screenshot.png' alt='Google Trace Explorer - representative screenshot' title='Google Trace Explorer - representative screenshot' width='100%' class=" mt-2 mx-auto" style='max-width:100%;aspect-ratio: 2.5858;'/>
|
|
|
|
</picture>
|
|
|
|
</div>
|
|
<h3 id="traceability-for-the-requests-not-picked-in-the-trace-sampling">Traceability for the requests not picked in the trace sampling</h3>
|
|
<p>A small percent of our requests were traced via Google Cloud Trace Explorer. However, all the requests reaching our load balancer were already logged and viewable via Google Log Explorer. For these requests not picked in the trace sampling, we still wanted to achieve the following:</p>
|
|
<ul>
|
|
<li>We wanted to see all the related requests within Google Log Explorer in a single view. And we wanted to be able to do this for 100% of our requests.</li>
|
|
<li>Every time a request resulted in an error, we wanted to log the custom error message from our code to Google Log Explorer. We also wanted this custom error message log entry to be tied to the related requests.</li>
|
|
</ul>
|
|
<p>To achieve the above, we needed to ensure traceability for requests excluded by the trace sampling. We achieved this by implementing the following -</p>
|
|
<p>If one of our services makes a request to another service within our setup, it should pass the incoming request’s trace-id to such downstream requests.</p>
|
|
<div class="highlight"><pre tabindex="0" class="chroma"><code class="language-JavaScript" data-lang="JavaScript"><span class="kr">const</span> <span class="nx">traceId</span> <span class="o">=</span> <span class="nx">req</span><span class="p">.</span><span class="nx">headers</span><span class="p">[</span><span class="s1">'x-cloud-trace-context'</span><span class="p">];</span>
|
|
<span class="nx">axios</span><span class="p">.</span><span class="nx">defaults</span><span class="p">.</span><span class="nx">headers</span><span class="p">.</span><span class="nx">common</span><span class="p">[</span><span class="s1">'x-cloud-trace-context'</span><span class="p">]</span> <span class="o">=</span> <span class="nx">traceId</span><span class="p">;</span>
|
|
</code></pre></div><p>With the above, 100% of our requests in Google Log Explorer could be traced at no additional cost or performance overhead.</p>
|
|
<p>And, to tie the custom error messages from our code to this traceability information, we added the incoming request’s trace-id to the log messages.</p>
|
|
<div class="highlight"><pre tabindex="0" class="chroma"><code class="language-JavaScript" data-lang="JavaScript"><span class="c1">//logging to the Google log explorer with @google-cloud/logging-winston
|
|
</span><span class="c1"></span><span class="nx">GCPWinstonLogger</span><span class="p">.</span><span class="nx">error</span><span class="p">(</span><span class="s1">''</span><span class="p">,</span> <span class="p">{</span>
|
|
<span class="nx">statusDetails</span><span class="o">:</span> <span class="nx">err</span><span class="o">?</span><span class="p">.</span><span class="nx">status</span> <span class="o">||</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span>
|
|
<span class="nx">message</span><span class="o">:</span> <span class="nx">err</span><span class="o">?</span><span class="p">.</span><span class="nx">message</span> <span class="o">||</span> <span class="s1">'Error'</span><span class="p">,</span>
|
|
<span class="nx">url</span><span class="o">:</span> <span class="sb">`</span><span class="si">${</span><span class="nx">req</span><span class="p">.</span><span class="nx">hostname</span><span class="si">}${</span><span class="nx">req</span><span class="p">.</span><span class="nx">originalUrl</span><span class="si">}</span><span class="sb">`</span><span class="p">,</span>
|
|
<span class="p">[</span><span class="nx">LoggingWinston</span><span class="p">.</span><span class="nx">LOGGING_TRACE_KEY</span><span class="p">]</span><span class="o">:</span> <span class="nx">traceId</span><span class="p">,</span>
|
|
<span class="p">});</span>
|
|
</code></pre></div><p>With the above, the custom error message in one service could be linked to a custom error message in another service. And all of this could be viewed via Google Log Explorer.</p>
|
|
<h2 id="how-we-use-our-observability-setup">How we use our Observability setup</h2>
|
|
<p>With the above detailed implementation, our Observability requirements are adequately covered as following:</p>
|
|
<ul>
|
|
<li>We leverage Google Cloud Trace to understand the following:
|
|
<ul>
|
|
<li>When a certain kind of requests (example, - <code>/search</code> requests) are slow, the cause of the slowness.</li>
|
|
<li>We can gradually add custom spans to further drill down these common causes of slowness (certain query, certain part of code, etc.).</li>
|
|
</ul>
|
|
</li>
|
|
<li>We leverage Google Log Explorer to understand the following:
|
|
<ul>
|
|
<li>When a specific request errors / is slow - what caused the issue for that specific request.</li>
|
|
</ul>
|
|
</li>
|
|
<li>We leverage Google Metric Explorer to quantify the extent of a custom error behavior:
|
|
<ul>
|
|
<li>The count of log entries with a certain error in comparison to the count of entries with HTTP 500 tells us what percentage of HTTP 500 are caused by a certain error condition.</li>
|
|
<li>We track these via dashboards to allow us to quickly identify the extent of an erroneous / slowness behavior.</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
<h2 id="conclusion">Conclusion</h2>
|
|
<p>With the above detailed observability setup in place, we could achieve the following:</p>
|
|
<ul>
|
|
<li>We could quickly identify costly components of slow-performing requests with Google Trace Explorer.</li>
|
|
<li>With traceability integrated into the log entries visible in Google Log Explorer, we could debug / analyze any issue that occurred in production.</li>
|
|
<li>With gradually built custom dashboards containing various metric charts, we could keep a tab on the health of our overall system at a glance.</li>
|
|
<li>With OpenTelemetry libraries, we limited our cloud provider dependency.</li>
|
|
<li>The above setup could be achieved with limited performance and cost overhead.</li>
|
|
</ul>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div class="mt-12"></div>
|
|
|
|
<div class="bg-tezify-75 px-2 border-t rounded-b border-tezify-275 border-b md:border-b-0 mx-[-1.0rem] mb-[-2.0rem] mt-0 md:mx-[-1.5rem] md:mb-[-1.5rem]">
|
|
<div class="py-8">
|
|
<div class="md:hidden">
|
|
<div class="flex items-center gap-3 mb-4">
|
|
<img src="/images/punit.jpeg" alt="Punit's Photo" class="w-14 h-14 rounded-full">
|
|
<div>
|
|
<p class="font-semibold text-lg">Got a web-dev pain point?</p>
|
|
<p class="text-sm">I'm an independent web dev & architect.</p>
|
|
</div>
|
|
</div>
|
|
<div class="text-left space-y-3 text-sm">
|
|
<div>
|
|
<p class="font-semibold mb-1">What I do:</p>
|
|
<ul class="space-y-1">
|
|
<li>Architecture - Guidance on frameworks, tools, system design, cloud infra</li>
|
|
<li>Fullstack Dev - Frontend, backend, CMS customization, migration, upgrade</li>
|
|
<li>Optimization - Core web vitals, scalability, SEO compliance</li>
|
|
</ul>
|
|
</div>
|
|
<div>
|
|
<p class="font-semibold mb-1">Tech I use:</p>
|
|
<p>Node.js, React, Next.js, Postgres, Strapi, Directus, MySQL, AWS, GCP, Serverless (Lambda, CF Workers), ElasticSearch, Redis, RabbitMQ, Kibana</p>
|
|
</div>
|
|
<div>
|
|
<p class="font-semibold mb-1">My Clients Include:</p>
|
|
<div class="flex items-center justify-center gap-4">
|
|
<img class="grayscale-image max-h-9 h-auto w-auto object-contain filter grayscale opacity-85 hover:opacity-100 transition-opacity duration-200" src="/images/cl_purplle.svg" alt="purplle.com Logo" title="purplle.com Logo" loading="lazy">
|
|
<img class="grayscale-image max-h-9 h-auto w-auto object-contain filter grayscale opacity-85 hover:opacity-100 transition-opacity duration-200" src="/images/cl_mpl_new_2.png" alt="mpl.live Logo" title="mpl.live Logo" loading="lazy">
|
|
<img class="grayscale-image max-h-9 h-auto w-auto object-contain filter grayscale opacity-85 hover:opacity-100 transition-opacity duration-200" src="/images/cl_wedmegood_2.svg" alt="wedmegood.com Logo" title="Wedmegood.com Logo" loading="lazy">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<a href="mailto:punit@tezify.com" class="mt-5 block w-full text-center bg-contrast-500 hover:bg-contrast-600 text-white font-semibold py-2 px-4 rounded">
|
|
Let's solve your tech challenges →
|
|
</a>
|
|
<p class="text-xs text-center mt-2 text-gray-500">Or email: punit@tezify.com</p>
|
|
</div>
|
|
<div class="hidden md:grid grid-cols-12 pt-2">
|
|
<div class="col-span-3 md:col-span-2 px-2 py-3">
|
|
<picture>
|
|
<source srcset="/images/punit.webp" type="image/webp" />
|
|
<img loading="lazy" src="/images/punit.jpeg" alt="Punit Sethi" title="Punit Sethi" class="rounded-full mx-auto mt-2 md:mt-0">
|
|
</picture>
|
|
<div class="text-center mt-2">Punit Sethi</div>
|
|
</div>
|
|
<div class="col-span-9 md:col-span-10 p-2">
|
|
<div class="mb-2 text-slate-300 text-xl"><strong>Got a web-dev pain point?</strong></div>
|
|
<div class="mb-2">
|
|
<p>I'm an independent web dev & architect.</p>
|
|
<p>
|
|
<p class="font-semibold mb-1">
|
|
What I do:
|
|
</p>
|
|
<ul class="list-none p-0 m-0">
|
|
<li class="before:content-['●_'] before:mr-1">Architecture - Guidance on frameworks, tools, system design, cloud infra</li>
|
|
<li class="before:content-['●_'] before:mr-1">Fullstack Dev - Frontend, backend, CMS customization, migration, upgrade</li>
|
|
<li class="before:content-['●_'] before:mr-1">Optimization - Core web vitals, scalability, SEO compliance</li>
|
|
</ul>
|
|
</p>
|
|
<p>
|
|
<p class="font-semibold mb-1">Tech I use:</p>
|
|
<p>Node.js, React, Next.js, Postgres, Strapi, Directus, MySQL, AWS, GCP, Serverless (Lambda, CF Workers), ElasticSearch, Redis, RabbitMQ, Kibana</p>
|
|
</p>
|
|
<p>
|
|
<p class="font-semibold mb-1">My Clients include:</p>
|
|
<div class="flex gap-4">
|
|
<img class="grayscale-image max-h-9 h-auto w-auto object-contain filter grayscale opacity-85 hover:opacity-100 transition-opacity duration-200" src="/images/cl_purplle.svg" alt="purplle.com Logo" title="purplle.com Logo" loading="lazy">
|
|
<img class="grayscale-image max-h-9 h-auto w-auto object-contain filter grayscale opacity-85 hover:opacity-100 transition-opacity duration-200" src="/images/cl_mpl_new_2.png" alt="mpl.live Logo" title="mpl.live Logo" loading="lazy">
|
|
<img class="grayscale-image max-h-9 h-auto w-auto object-contain filter grayscale opacity-85 hover:opacity-100 transition-opacity duration-200" src="/images/cl_wedmegood_2.svg" alt="wedmegood.com Logo" title="Wedmegood.com Logo" loading="lazy">
|
|
</div>
|
|
</p>
|
|
<div class="mt-5">
|
|
<a class="inline-block bg-contrast-500 text-white py-3 px-5 rounded-md no-underline font-bold hover:bg-contrast-600" href="mailto:punit@tezify.com"><span class="flex">Let's solve your technical challenges →</a>
|
|
<p class="mt-3 text-sm">Or email : <strong>punit@tezify.com</strong></p>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
</div>
|
|
|
|
|
|
</div>
|
|
<style>
|
|
h1,h2, h3, h4, h5, h6 {
|
|
scroll-margin-top: 4rem;
|
|
}
|
|
a[aria-current="page"] {
|
|
font-weight: 800;
|
|
color: #332F37;
|
|
}
|
|
a[aria-current="page"] {
|
|
font-weight: 800;
|
|
color: #332F37;
|
|
}
|
|
li[aria-current="page"] {
|
|
background-color: #faf8f9;
|
|
border-left: 5px solid #706779;
|
|
}
|
|
.toc-scrollbar {
|
|
|
|
scrollbar-width: thin;
|
|
scrollbar-color: #b8b1be #faf8f9;
|
|
|
|
|
|
}
|
|
.toc-scrollbar::-webkit-scrollbar {
|
|
width: 8px;
|
|
}
|
|
.toc-scrollbar::-webkit-scrollbar-track {
|
|
background: #faf8f9;
|
|
border-radius: 4px;
|
|
}
|
|
.toc-scrollbar::-webkit-scrollbar-thumb {
|
|
background-color: #faf8f9;
|
|
border-radius: 4px;
|
|
}
|
|
.toc-scrollbar::-webkit-scrollbar-thumb:hover {
|
|
background-color: #faf8f9;
|
|
}
|
|
</style>
|
|
<script>
|
|
document.addEventListener("DOMContentLoaded", () => {
|
|
const sections = document.querySelectorAll("h2[id], h3[id]");
|
|
const tocContainer = document.querySelector('div[type="desktop-toc"]');
|
|
if (!tocContainer) return;
|
|
|
|
const navLinks = tocContainer.querySelectorAll("a");
|
|
|
|
const observer = new IntersectionObserver((entries) => {
|
|
|
|
const visible = entries
|
|
.filter(entry => entry.isIntersecting)
|
|
.sort((a, b) => a.boundingClientRect.top - b.boundingClientRect.top);
|
|
|
|
if (visible.length > 0) {
|
|
const topEntry = visible[0];
|
|
|
|
|
|
navLinks.forEach(link => link.removeAttribute("aria-current"));
|
|
navLinks.forEach(link => link.parentElement.removeAttribute("aria-current"));
|
|
|
|
const activeLink = tocContainer.querySelector(`a[href="#${topEntry.target.id}"]`);
|
|
if (activeLink) {
|
|
activeLink.setAttribute("aria-current", "page");
|
|
activeLink.parentElement.setAttribute("aria-current", "page");
|
|
}
|
|
}
|
|
}, { rootMargin: "-50px 0px -65% 0px", threshold: 0 });
|
|
|
|
sections.forEach(sec => observer.observe(sec));
|
|
});
|
|
</script>
|
|
<script>
|
|
document.addEventListener("DOMContentLoaded", () => {
|
|
const toc = document.querySelector('div[type="desktop-toc"]');
|
|
const footer = document.querySelector('div[type="footer"]');
|
|
if (!toc || !footer) return;
|
|
|
|
const observer = new IntersectionObserver((entries) => {
|
|
entries.forEach(entry => {
|
|
if (entry.isIntersecting) {
|
|
|
|
toc.children[0].classList.remove("sticky");
|
|
toc.style.position = "static";
|
|
} else {
|
|
|
|
toc.children[0].classList.add("sticky");
|
|
toc.style.position = "";
|
|
}
|
|
});
|
|
}, {
|
|
root: null,
|
|
threshold: 0,
|
|
rootMargin: "0px 0px 480px 0px"
|
|
});
|
|
|
|
observer.observe(footer);
|
|
});
|
|
</script>
|
|
<div type="desktop-toc" class="hidden xl:block xl:col-span-4">
|
|
<div class="hidden xl:block px-4 py-2 sticky top-20" style="/*margin-top: 48rem;*/">
|
|
<div class="px-2 py-4 bg-white rounded border border-tezify-350 max-h-[60vh] overflow-y-auto toc-scrollbar">
|
|
<h3 class="text-lg font-semibold mb-3 mx-1"><span class="text-tezify-600">Table of contents</span></h3>
|
|
|
|
<ul class="list-disc px-4 text-sm text-tezify-800 mx-1">
|
|
<li class="py-[3px] pl-1 aria-[current=page]:bg-tezify-25"><a class='nobottomline aria-[current=page]:bg-tezify-250 aria-[current=page]:text-tezify-950' href='#observability-tools--why-we-picked-google-cloud-offerings'>Observability tools & why we picked Google Cloud Offerings</a></li>
|
|
<li class="py-[3px] pl-1 aria-[current=page]:bg-tezify-25"><a class='nobottomline aria-[current=page]:bg-tezify-250 aria-[current=page]:text-tezify-950' href='#our-existing-google-cloud-logging-and-monitoring-setup'>Our existing Google Cloud logging and monitoring setup</a></li>
|
|
<li class="py-[3px] pl-1 aria-[current=page]:bg-tezify-25"><a class='nobottomline aria-[current=page]:bg-tezify-250 aria-[current=page]:text-tezify-950' href='#the-missing-piece-traceability'>The missing piece: Traceability</a></li>
|
|
<li class="py-[3px] pl-1 aria-[current=page]:bg-tezify-25"><a class='nobottomline aria-[current=page]:bg-tezify-250 aria-[current=page]:text-tezify-950' href='#our-traceability-implementation'>Our Traceability implementation</a></li>
|
|
<ul class="list-disc text-sm text-tezify-800 my-1 mx-1.2">
|
|
<li class="ml-2 py-[3px] pl-1 aria-[current=page]:rounded-lg aria-[current=page]:bg-tezify-250"><a class='nobottomline aria-[current=page]:text-tezify-950' href='#leveraging-the-opentelemetry-integration'>Leveraging the OpenTelemetry integration</a></li>
|
|
<li class="ml-2 py-[3px] pl-1 aria-[current=page]:rounded-lg aria-[current=page]:bg-tezify-250"><a class='nobottomline aria-[current=page]:text-tezify-950' href='#setting-up-sampling-to-keep-traceability-overhead-and-costs-in-check'>Setting up sampling to keep traceability overhead and costs in check</a></li>
|
|
<li class="ml-2 py-[3px] pl-1 aria-[current=page]:rounded-lg aria-[current=page]:bg-tezify-250"><a class='nobottomline aria-[current=page]:text-tezify-950' href='#traceability-for-the-requests-not-picked-in-the-trace-sampling'>Traceability for the requests not picked in the trace sampling</a></li>
|
|
</ul>
|
|
<li class="py-[3px] pl-1 aria-[current=page]:bg-tezify-25"><a class='nobottomline aria-[current=page]:bg-tezify-250 aria-[current=page]:text-tezify-950' href='#how-we-use-our-observability-setup'>How we use our Observability setup</a></li>
|
|
<li class="py-[3px] pl-1 aria-[current=page]:bg-tezify-25"><a class='nobottomline aria-[current=page]:bg-tezify-250 aria-[current=page]:text-tezify-950' href='#conclusion'>Conclusion</a></li>
|
|
</ul>
|
|
|
|
|
|
</div>
|
|
|
|
<div class="mt-4 px-2 py-4 border rounded text-sm text-gray-700 bg-white border-tezify-350">
|
|
<div class="mx-1"><img style="height:1em" class="inline" alt="Configure icon" src="/images/svg/icon-server-blue.svg" /> <span>See other <a class="text-blue-600 font-medium hover:text-blue-800" title="Site reliability engineering" href="/site-reliability-engineering/">posts on Site Reliability Engineering</a>.</span></div>
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div class="bg-tezify-100-w-noise">
|
|
<div class="mx-auto rounded relative md:max-w-screen-md flex items-center flex-col">
|
|
<div class="text-lg m-2 py-8 px-2"><i>
|
|
<br />
|
|
</i>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div class="fixed md:hidden bottom-0 left-0 right-0">
|
|
<div class="bg-tezify-75 border-t border-tezify-275 px-3 py-4 "><img style="height:1em" class="inline" alt="Configure icon" src="/images/svg/icon-server-blue.svg" /> <span>More <a class="text-blue-600 font-medium hover:text-blue-800" title="Site reliability engineering" href="/site-reliability-engineering/">posts on Site Reliability Engineering</a></span></div>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<script defer data-domain="punits.dev" src="https://stats.punits.dev/js/script.tagged-events.js"></script>
|
|
<script>
|
|
function getCookie(cname) {
|
|
var name = cname + "=";
|
|
var ca = document.cookie.split(';');
|
|
for(var i = 0; i < ca.length; i++) {
|
|
var c = ca[i];
|
|
while (c.charAt(0) == ' ') {
|
|
c = c.substring(1);
|
|
}
|
|
if (c.indexOf(name) == 0) {
|
|
return c.substring(name.length, c.length);
|
|
}
|
|
}
|
|
return "";
|
|
}
|
|
</script>
|
|
|
|
<div type="footer" class="md:flex md:flex-grow items-center justify-between bg-tezify-800 px-3 py-3 text-sm text-white md:px-12">
|
|
<div>
|
|
Copyright (c) 2017-2026 Tezify All Rights Reserved. Created in India. GSTIN : 24BBQPS3732P1ZW.
|
|
</div>
|
|
<div class="mt-4 md:my-0 flex gap-x-4">
|
|
<a href="/privacy-policy.html" class="nobottomline hover:underline">Privacy Policy</a>
|
|
<a href="/terms-of-service.html" class="nobottomline hover:underline">Terms of Service</a>
|
|
</div>
|
|
</div>
|
|
|
|
</html>
|
|
</body> |