1 line
128 KiB
HTML
1 line
128 KiB
HTML
<!DOCTYPE html><html data-dpl-id="dpl_9i6KweGTYWNasiv9e3XcXFAn1n9r" lang="en-US" data-color-scheme="light" data-resolved-scheme="light"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="image" imageSrcSet="/_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1661860358-image-228.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D80%26w%3D128&w=48&q=75 1x, /_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1661860358-image-228.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D80%26w%3D128&w=96&q=75 2x"/><link rel="preload" as="image" imageSrcSet="/_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=640&q=90 640w, /_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=750&q=90 750w, /_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=828&q=90 828w, /_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=1080&q=90 1080w, /_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=1200&q=90 1200w, /_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=1920&q=90 1920w, /_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=2048&q=90 2048w, /_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=3840&q=90 3840w" imageSizes="(max-width: 767px) 100vw, (max-width: 1439px) 70vw, 1040px"/><link rel="stylesheet" href="/_next/static/immutable/chunks/132trk_zc3ylu.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/immutable/chunks/1bmd2ryn1r-hv.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/immutable/chunks/3qp9-r2fgklws.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/immutable/chunks/43xkgk33-6316.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/immutable/chunks/3w0wy298bmb96.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/immutable/chunks/27k1lbkm9ytw4.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/immutable/chunks/45auz_6fncqnw.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/immutable/chunks/3a0f_y332k2v1.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="/_next/static/immutable/chunks/2iox-02sx9loa.js"/><script src="/_next/static/immutable/chunks/2n1jkab2r5d63.js" async=""></script><script src="/_next/static/immutable/chunks/1pguxqs8oj5s0.js" async=""></script><script src="/_next/static/immutable/chunks/43a_dvcajw8iz.js" async=""></script><script src="/_next/static/immutable/chunks/turbopack-3xk1_gujkngr6.js" async=""></script><script src="/_next/static/immutable/chunks/3sz971vfk095f.js" async="" crossorigin=""></script><script src="/_next/static/immutable/chunks/0czgivygkxa1z.js" async="" crossorigin=""></script><script src="/_next/static/immutable/chunks/2kpu_ktjkwf7q.js" async="" crossorigin=""></script><script src="/_next/static/immutable/chunks/07hv-f84dhd61.js" async="" crossorigin=""></script><script src="/_next/static/immutable/chunks/0639nwpv6l0pm.js" async="" crossorigin=""></script><script src="/_next/static/immutable/chunks/2826pm12av2bw.js" async="" crossorigin=""></script><script src="/_next/static/immutable/chunks/2_swgt_1o6v9a.js" async="" crossorigin=""></script><script src="/_next/static/immutable/chunks/3pjhofejz57jt.js" async=""></script><script src="/_next/static/immutable/chunks/1q-89ajkqbbkw.js" async=""></script><script src="/_next/static/immutable/chunks/2uho2m59t6w8f.js" async=""></script><script src="/_next/static/immutable/chunks/2ti9pdu9p4n5l.js" async=""></script><script src="/_next/static/immutable/chunks/31qqpjo4d7384.js" async=""></script><script src="/_next/static/immutable/chunks/3zb6qebqmln4d.js" async=""></script><meta name="theme-color" content="#fcf9f5"/><link rel="preload" href="/fonts/Roobert-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous"/><link rel="preload" href="/fonts/Reckless-Light.woff2" as="font" type="font/woff2" crossorigin="anonymous"/><link rel="preload" href="/fonts/Reckless-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous"/><title>Thankful for incidents: embracing chaos to find clarity</title><meta name="description" content="How lessons from a recent incident led to improved platform resilience and more comprehensive testing practices."/><meta name="keywords" content="Tines, security workflow automation, No-Code, SOAR"/><link rel="canonical" href="https://www.tines.com/blog/engineering-incidents-improvement/"/><meta property="og:title" content="Thankful for incidents: embracing chaos to find clarity"/><meta property="og:description" content="How lessons from a recent incident led to improved platform resilience and more comprehensive testing practices."/><meta property="og:image" content="https://www.datocms-assets.com/55802/1721460016-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity-1.png"/><meta property="og:type" content="article"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:creator" content="@tines_hq"/><meta name="twitter:title" content="Tines | Intelligent workflow platform"/><meta name="twitter:description" content="Build and orchestrate intelligent workflows and agents, on the platform security and IT teams love."/><meta name="twitter:image" content="https://www.tines.com/v5-assets/images/og/tines-home.png"/><link rel="icon" href="/v5-assets/favicon.png"/><script id="color-scheme-init">(function(){var d=document.documentElement,p=location.pathname,R=["/stories/docs","/stories/whats-new","/workflow-capability-matrix","/3b/examples","/3b/product-updates","/library","/stories/five-minute-flows"],F=["/stories/five-minute-flows"],M=["/workflow-capability-matrix"];function tc(c){var m=document.querySelector('meta[name="theme-color"]');if(m)m.setAttribute("content",c)}function on(L){return L.some(function(r){return p===r||p.indexOf(r+"/")===0})}var ok=on(R);if(!ok){d.removeAttribute("data-dark-mode-supported");d.removeAttribute("data-page-theme");d.setAttribute("data-color-scheme","light");d.setAttribute("data-resolved-scheme","light");tc("#fcf9f5");if(on(["/campaigns"])){d.setAttribute("data-campaign-nav","")}return}d.setAttribute("data-dark-mode-supported","");if(on(F)){d.setAttribute("data-color-scheme","dark");d.setAttribute("data-resolved-scheme","dark");tc("#16131e");return}var pt=document.cookie.match(/page-theme=(\w+)/);var mq=/[?&]matrix=on/.test(location.search);if(on(M)&&((pt&&pt[1]==="matrix")||mq)){d.setAttribute("data-page-theme","matrix");d.setAttribute("data-color-scheme","dark");d.setAttribute("data-resolved-scheme","dark");tc("#0b1a17");return}else{d.removeAttribute("data-page-theme")}var cm=document.cookie.match(/preferred-color-scheme=(\w+)/);var s;if(cm){s=cm[1]}else{var sv=d.getAttribute("data-color-scheme");s=sv&&sv!=="light"?sv:"auto"}var r=s==="dark"?"dark":s==="light"?"light":window.matchMedia("(prefers-color-scheme:dark)").matches?"dark":"light";d.setAttribute("data-color-scheme",s);d.setAttribute("data-resolved-scheme",r);tc(r==="dark"?"#16131e":"#fcf9f5");if(on(["/campaigns"])){d.setAttribute("data-campaign-nav","")}})()</script><script src="/_next/static/immutable/chunks/0c0hxoamwjsbw.js" noModule=""></script></head><body><div hidden=""><!--$--><!--/$--></div><script type="application/ld+json">{"@context":"https://schema.org","@type":"Organization","name":"Tines","url":"https://www.tines.com/","@id":"https://www.tines.com/#org","description":"Tines 3B enables every team to build AI agents, apps, and automation in one safe, secure environment — with the control Security and IT need to govern it at scale.","foundingDate":"2018","founders":[{"@type":"Person","name":"Eoin Hinchy"},{"@type":"Person","name":"Thomas Kinsella"}],"sameAs":["https://www.linkedin.com/company/tines-io/","https://twitter.com/tines_hq","https://www.youtube.com/@TinesHQ","https://github.com/tines"],"contactPoint":[{"@type":"ContactPoint","contactType":"customer support","email":"hello@tines.com","availableLanguage":["en"]},{"@type":"ContactPoint","contactType":"analyst relations","email":"analysts@tines.com","availableLanguage":["en"]},{"@type":"ContactPoint","contactType":"media relations","email":"pr@tines.com","availableLanguage":["en"]}],"address":[{"@type":"PostalAddress","streetAddress":"10 Westland Square, Pearse Street","addressLocality":"Dublin","postalCode":"D02 CX36","addressCountry":"IE","name":"Tines Limited"},{"@type":"PostalAddress","streetAddress":"99 High Street, 14th Floor","addressLocality":"Boston","addressRegion":"MA","postalCode":"02111","addressCountry":"US","name":"Tines Technology Inc."}],"areaServed":"Global"}</script><div class="PageBackdrop-module-scss-module__KKf-uW__pageShell"><div id="page-backdrop-portal" class="PageBackdrop-module-scss-module__KKf-uW__slot" aria-hidden="true"></div><a href="#main-content" class="skip-link">Skip to content</a><header id="site-nav" class="SiteNav26-module-scss-module__OUXeaq__header"><div class="SiteNav26-module-scss-module__OUXeaq__blur" aria-hidden="true"></div><div class="SiteNav26-module-scss-module__OUXeaq__backdrop" aria-hidden="true"></div><div class="SiteNav26-module-scss-module__OUXeaq__bar"><div class="SiteNav26-module-scss-module__OUXeaq__left"><div class="SiteNav26-module-scss-module__OUXeaq__logoWrap"><a class="SiteNav26-module-scss-module__OUXeaq__logo" title="Tines" href="/"><svg class="TinesLogo" width="91" height="28" viewBox="0 0 103 32" fill="currentColor"><title>Tines</title><path d="M42.9689 10.5676H40V13.6565H42.9689V20.4041C42.9689 23.7629 45.1282 25.6822 48.427 25.6822H50.8561V22.3534H48.8468C47.4673 22.3534 46.6576 21.5737 46.6576 20.0442V13.6565H50.8561V10.5676H46.6576V6.00928H42.9689V10.5676Z"></path><path d="M52.5751 8.19849H56.6536V4H52.5751V8.19849ZM56.4437 25.6822V10.5676H52.755V25.6822H56.4437Z"></path><path d="M62.3935 25.6822V17.4352C62.3935 15.216 63.833 13.5966 65.9023 13.5966C67.7916 13.5966 69.0511 15.036 69.0511 17.1053V25.6822H72.7398V16.3256C72.7398 12.7868 70.5206 10.3277 67.1018 10.3277C64.9426 10.3277 63.2332 11.2574 62.3935 12.9068V10.5676H58.7048V25.6822H62.3935Z"></path><path d="M85.399 20.644C85.0092 22.0835 83.7796 22.8632 81.8903 22.8632C79.5511 22.8632 77.9917 21.3038 77.7218 18.8747H88.8778C88.9078 18.5748 88.9377 18.1249 88.9377 17.6151C88.9377 14.0164 86.6586 10.3277 81.5904 10.3277C76.6422 10.3277 74.183 14.0764 74.183 18.065C74.183 22.0235 76.9421 25.9221 81.8903 25.9221C85.6089 25.9221 88.4279 23.8529 88.9977 20.644H85.399ZM81.5904 13.0867C83.7796 13.0867 85.0991 14.5262 85.2491 16.4455H77.8117C78.2016 14.1664 79.5211 13.0867 81.5904 13.0867Z"></path><path d="M96.3428 25.9221C99.8815 25.9221 102.191 24.0628 102.191 21.2138C102.191 14.3763 93.7937 17.975 93.7937 14.7062C93.7937 13.6865 94.6634 13.0867 95.9829 13.0867C97.1525 13.0867 98.442 13.8065 98.6519 15.216H102.011C101.861 12.277 99.4316 10.3277 95.8929 10.3277C92.774 10.3277 90.4649 12.1871 90.4649 14.8261C90.4649 21.2138 98.7419 18.005 98.7419 21.3638C98.7419 22.2634 97.7822 22.9832 96.3428 22.9832C94.6933 22.9832 93.5837 22.0835 93.4338 20.644H90.075C90.2549 23.8829 92.6841 25.9221 96.3428 25.9221Z"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M11.8018 0C8.01458 0 4.66599 2.45749 3.53258 6.06868L0.415527 16L3.53258 25.9313C4.66599 29.5425 8.01458 32 11.8018 32H20.1981C23.9853 32 27.3339 29.5425 28.4673 25.9313L31.5844 16L28.4673 6.06868C27.3339 2.45749 23.9853 0 20.1981 0H11.8018ZM20.1982 2.49634C22.8938 2.49634 25.2772 4.24548 26.0839 6.81577L26.8481 9.25062C25.3107 7.98154 23.3639 7.26723 21.3292 7.26707L10.648 7.26679C8.62691 7.26694 6.69264 7.97168 5.16015 9.22481L5.91625 6.81577C6.72297 4.24548 9.10635 2.49634 11.8019 2.49634H20.1982ZM5.73674 12.1986L3.79587 14.7519L28.1811 14.7519L26.2404 12.1989C25.0741 10.6646 23.2571 9.76356 21.329 9.76341H10.5898C8.68349 9.78153 6.89125 10.6798 5.73674 12.1986ZM28.1771 17.2482L26.2403 19.7989C25.0739 21.3349 23.2555 22.237 21.326 22.2368L10.6509 22.2366C8.72137 22.2367 6.90298 21.3346 5.73661 19.7986L3.79996 17.2482L28.1771 17.2482ZM5.9161 25.1842C6.72282 27.7545 9.1062 29.5037 11.8018 29.5037H20.1981C22.8936 29.5037 25.277 27.7545 26.0837 25.1842L26.8485 22.7476C25.3104 24.0182 23.3622 24.7333 21.3258 24.7332L10.651 24.7329C8.6283 24.7331 6.69244 24.0274 5.15921 22.7727L5.9161 25.1842Z"></path></svg></a></div><nav class="SiteNav26-module-scss-module__OUXeaq__navList" aria-label="Primary"><button type="button" class="SiteNav26-module-scss-module__OUXeaq__navItem" aria-expanded="false">Product<span class="SiteNav26-module-scss-module__OUXeaq__navChevron" aria-hidden="true"></span></button><button type="button" class="SiteNav26-module-scss-module__OUXeaq__navItem" aria-expanded="false">Solutions<span class="SiteNav26-module-scss-module__OUXeaq__navChevron" aria-hidden="true"></span></button><a class="SiteNav26-module-scss-module__OUXeaq__navItem" href="/customers/">Customers</a><button type="button" class="SiteNav26-module-scss-module__OUXeaq__navItem" aria-expanded="false">Discover<span class="SiteNav26-module-scss-module__OUXeaq__navChevron" aria-hidden="true"></span></button><a class="SiteNav26-module-scss-module__OUXeaq__navItem" href="/pricing/">Pricing</a></nav></div><div class="SiteNav26-module-scss-module__OUXeaq__right"><a href="https://login.tines.com" target="_blank" rel="noopener noreferrer" class="SiteNav26-module-scss-module__OUXeaq__cta">Log in</a><a href="https://login.tines.com/saml_idp/signup" target="_blank" rel="noopener noreferrer" class="SiteNav26-module-scss-module__OUXeaq__cta">Sign up</a><a class="SiteNav26-module-scss-module__OUXeaq__cta SiteNav26-module-scss-module__OUXeaq__ctaFilled" href="/book-a-demo/">Book a demo</a><div class="SiteNav26-module-scss-module__OUXeaq__searchSlot"><button type="button" class="NavSearch-module-scss-module__vLhI_W__pillButton" aria-label="Open search"><svg viewBox="0 0 18 18" fill="none" aria-hidden="true"><path fill-rule="evenodd" clip-rule="evenodd" d="M8 3C10.7614 3 13 5.23858 13 8C13 9.11033 12.6367 10.1351 12.0244 10.9648C12.0262 10.9666 12.0285 10.9679 12.0303 10.9697L15.5303 14.4697C15.8232 14.7626 15.8232 15.2374 15.5303 15.5303C15.2374 15.8232 14.7626 15.8232 14.4697 15.5303L10.9697 12.0303C10.9679 12.0285 10.9666 12.0262 10.9648 12.0244C10.1351 12.6367 9.11033 13 8 13C5.23858 13 3 10.7614 3 8C3 5.23858 5.23858 3 8 3ZM8 4.5C6.067 4.5 4.5 6.067 4.5 8C4.5 9.933 6.067 11.5 8 11.5C9.933 11.5 11.5 9.933 11.5 8C11.5 6.067 9.933 4.5 8 4.5Z" fill="currentColor"></path></svg></button></div><button type="button" class="SiteNav26-module-scss-module__OUXeaq__hamburger" aria-label="Toggle menu" aria-expanded="false"><svg viewBox="0 0 18 18" fill="none" aria-hidden="true"><path d="M3 5h12M3 9h12M3 13h12" stroke="currentColor" stroke-width="1.6" stroke-linecap="round"></path></svg></button></div></div><div class="SiteNav26-module-scss-module__OUXeaq__panel" hidden=""></div><div class="SiteNav26-module-scss-module__OUXeaq__panel" hidden=""></div><div class="SiteNav26-module-scss-module__OUXeaq__panel" hidden=""></div></header><main id="main-content"><div><div class="AreaNav-module-scss-module__oPsmja__areaNav"><div class="AreaNav-module-scss-module__oPsmja__bar"><div class="AreaNav-module-scss-module__oPsmja__inner AreaNav-module-scss-module__oPsmja__preMeasure"><span class="AreaNav-module-scss-module__oPsmja__pillWrap"><a class="AreaNav-module-scss-module__oPsmja__pill" href="/blog/">Blog</a></span><div class="AreaNav-module-scss-module__oPsmja__right AreaNav-module-scss-module__oPsmja__flushEnd"><span class="AreaNav-module-scss-module__oPsmja__sectionsZone"><nav class="AreaNav-module-scss-module__oPsmja__sections"><a class="AreaNav-module-scss-module__oPsmja__sectionTrigger" href="/blog/">Home</a><span class="AreaNav-module-scss-module__oPsmja__flatGroup"><a class="AreaNav-module-scss-module__oPsmja__sectionTrigger" href="/blog/news/">News</a><a class="AreaNav-module-scss-module__oPsmja__sectionTrigger" href="/blog/tutorials/">Tutorials</a><a class="AreaNav-module-scss-module__oPsmja__sectionTrigger" href="/blog/engineering/">Engineering</a><a class="AreaNav-module-scss-module__oPsmja__sectionTrigger" href="/blog/product-updates/">Product updates</a><a class="AreaNav-module-scss-module__oPsmja__sectionTrigger" href="/blog/inside-tines/">Inside Tines</a><a class="AreaNav-module-scss-module__oPsmja__sectionTrigger" href="/blog/tines-by-tines/">Tines × Tines</a></span><div class="AreaNav-module-scss-module__oPsmja__section AreaNav-module-scss-module__oPsmja__collapsible"><button type="button" class="AreaNav-module-scss-module__oPsmja__sectionTrigger" aria-expanded="false">Categories<span class="AreaNav-module-scss-module__oPsmja__caret" aria-hidden="true"></span></button></div><a class="AreaNav-module-scss-module__oPsmja__sectionTrigger" href="/blog/guides/">Guides</a><a class="AreaNav-module-scss-module__oPsmja__sectionTrigger" href="/blog/articles/">All articles</a></nav></span><div class="AreaNav-module-scss-module__oPsmja__rightFixed"></div></div></div></div></div><div class="blog-module-scss-module__RlMr6a__contentColumn"><script type="application/ld+json">{"@context":"https://schema.org","@type":"BlogPosting","headline":"Thankful for incidents: embracing chaos to find clarity","description":"How lessons from a recent incident led to improved platform resilience and more comprehensive testing practices.","datePublished":"2024-07-19T23:00:00+00:00","dateModified":"2024-08-13T13:45:13+00:00","url":"https://www.tines.com/blog/engineering-incidents-improvement/","image":"https://www.datocms-assets.com/55802/1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png?auto=format%2Ccompress&fit=max&q=90&w=2400","author":[{"@type":"Person","name":"Shayon Mukherjee"}],"publisher":{"@type":"Organization","name":"Tines","url":"https://www.tines.com"}}</script><div style="display:block;width:var(--xl);height:var(--xl);flex:0 0 var(--xl);user-select:none"></div><header><h1 class="ArticlePageHeader-module-scss-module__szLeSa__title">Thankful for incidents: embracing chaos to find clarity</h1><div class="ArticlePageHeader-module-scss-module__szLeSa__authorInfo"><div class="ArticlePageHeader-module-scss-module__szLeSa__authors"><div class="ArticlePageHeader-module-scss-module__szLeSa__authorPhotos"><img alt="" loading="eager" width="42" height="42" decoding="async" data-nimg="1" class="ArticlePageHeader-module-scss-module__szLeSa__authorPhoto" style="color:transparent" srcSet="/_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1661860358-image-228.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D80%26w%3D128&w=48&q=75 1x, /_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1661860358-image-228.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D80%26w%3D128&w=96&q=75 2x" src="/_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1661860358-image-228.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D80%26w%3D128&w=96&q=75"/></div><div><span class="ArticlePageHeader-module-scss-module__szLeSa__authorName"><span class="ArticlePageHeader-module-scss-module__szLeSa__writtenBy">Written by </span><span>Shayon Mukherjee</span></span><span class="ArticlePageHeader-module-scss-module__szLeSa__authorWorkTitle">Staff Software Engineer, Tines</span></div></div><div class="ArticlePageHeader-module-scss-module__szLeSa__meta"><time dateTime="2024-07-19T23:00:00.000Z">Published on <!-- -->July 19, 2024</time><div class="ArticlePageHeader-module-scss-module__szLeSa__categoryPills"><a class="ArticlePageHeader-module-scss-module__szLeSa__categoryPill" href="/blog/news/">News</a><a class="ArticlePageHeader-module-scss-module__szLeSa__categoryPill" href="/blog/engineering/">Engineering</a></div></div></div></header><div style="display:block;width:var(--lg);height:var(--lg);flex:0 0 var(--lg);user-select:none"></div><div class="page-module-scss-module__Z5AZta__articleLayout page-module-scss-module__Z5AZta__noToc"><aside class="page-module-scss-module__Z5AZta__articleAside"><a href="https://login.tines.com/saml_idp/signup" class="DefaultSidebarContent-module-scss-module__nlQFtG__card"><div class="DefaultSidebarContent-module-scss-module__nlQFtG__text"><h3 class="DefaultSidebarContent-module-scss-module__nlQFtG__heading">Sign up for Tines 3B Explore Edition</h3><p class="DefaultSidebarContent-module-scss-module__nlQFtG__linkArrow">Get started for free</p></div><img alt="" aria-hidden="true" loading="lazy" width="90" height="90" decoding="async" data-nimg="1" class="DefaultSidebarContent-module-scss-module__nlQFtG__icon" style="color:transparent" src="/v5-assets/images/usp/blossoming-transformation.svg?dpl=dpl_9i6KweGTYWNasiv9e3XcXFAn1n9r"/></a></aside><div class="page-module-scss-module__Z5AZta__articleMain"><div class="page-module-scss-module__Z5AZta__coverImage"><img alt="Cover image for Thankful for incidents: embracing chaos to find clarity" loading="eager" width="2400" height="1260" decoding="async" data-nimg="1" style="color:transparent" sizes="(max-width: 767px) 100vw, (max-width: 1439px) 70vw, 1040px" srcSet="/_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=640&q=90 640w, /_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=750&q=90 750w, /_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=828&q=90 828w, /_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=1080&q=90 1080w, /_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=1200&q=90 1200w, /_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=1920&q=90 1920w, /_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=2048&q=90 2048w, /_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=3840&q=90 3840w" src="/_next/image/?url=https%3A%2F%2Fwww.datocms-assets.com%2F55802%2F1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png%3Fauto%3Dformat%252Ccompress%26fit%3Dmax%26q%3D90%26w%3D2400&w=3840&q=90"/></div><div class="page-module-scss-module__Z5AZta__oldArticleWarning"><div class="InfoBox-module-scss-module__K7dC3a__infoBox"><svg width="22" height="22" viewBox="0 0 22 22" fill="currentColor" aria-hidden="true"><path fill-rule="evenodd" clip-rule="evenodd" d="M11 0C4.925 0 0 4.925 0 11C0 17.075 4.925 22 11 22C17.075 22 22 17.075 22 11C22 4.925 17.075 0 11 0ZM10.5 5C10.2348 5 9.98043 5.10536 9.79289 5.29289C9.60536 5.48043 9.5 5.73478 9.5 6C9.5 6.26522 9.60536 6.51957 9.79289 6.70711C9.98043 6.89464 10.2348 7 10.5 7H11C11.2652 7 11.5196 6.89464 11.7071 6.70711C11.8946 6.51957 12 6.26522 12 6C12 5.73478 11.8946 5.48043 11.7071 5.29289C11.5196 5.10536 11.2652 5 11 5H10.5ZM9 9C8.73478 9 8.48043 9.10536 8.29289 9.29289C8.10536 9.48043 8 9.73478 8 10C8 10.2652 8.10536 10.5196 8.29289 10.7071C8.48043 10.8946 8.73478 11 9 11H10V14H9C8.73478 14 8.48043 14.1054 8.29289 14.2929C8.10536 14.4804 8 14.7348 8 15C8 15.2652 8.10536 15.5196 8.29289 15.7071C8.48043 15.8946 8.73478 16 9 16H13C13.2652 16 13.5196 15.8946 13.7071 15.7071C13.8946 15.5196 14 15.2652 14 15C14 14.7348 13.8946 14.4804 13.7071 14.2929C13.5196 14.1054 13.2652 14 13 14H12V10C12 9.73478 11.8946 9.48043 11.7071 9.29289C11.5196 9.10536 11.2652 9 11 9H9Z"></path></svg><div class="InfoBox-module-scss-module__K7dC3a__text">This article was posted more than 18 months ago.</div></div></div><article class="Article-module-scss-module__ZE_SmG__article page-module-scss-module__Z5AZta__articleContent"><p><em><strong>In this blog post, Tines software engineer Shayon Mukherjee shares how lessons from a recent incident led to improved platform resilience and more comprehensive testing practices.</strong></em></p><p>It was a typical late June afternoon when we embarked on what seemed a routine Redis cluster upgrade across approximately 40 customer stacks. The upgrade was essential, influenced by a previous outage that highlighted the risks of not using more robust instances and better networking support on those instance types. </p><p>This wasn't the first time we performed such an upgrade, nor is it going to be the last time. But little did we know that this upgrade would soon reveal an unseen issue that lay dormant, undetected, and ready to teach us a valuable lesson.</p><p>A few moments into the upgrade process, our monitoring systems flagged the Toolkit API Down alarm.<a href="/blog/introducing-the-tines-toolkit/"> <u>Tines Toolkit</u></a> is a Tines<a href="/docs/actions/types/webhook#response-enabled-webhooks"> <u>response-enabled webhooks</u></a>-powered service. This alert was the first signal of something amiss. The webhooks were timing out,<a href="https://status.tines.com/incidents/jgXSym"> <u>affecting crucial customer workflows</u></a>. The immediate response was swift - our engineers triggered a force deployment to flush out any bad state from our containers, bringing things back online within a few minutes. But this was just the beginning. Now we needed to understand what had actually happened.</p><h3 id="how-does-it-all-work"><a href="#how-does-it-all-work"><span><strong>How does it all work?</strong></span><svg aria-hidden="true" data-heading-link-icon="true" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M13.5438 10.456C12.7247 9.63723 11.614 9.17728 10.4558 9.17728C9.29766 9.17728 8.18692 9.63723 7.36781 10.456L4.27881 13.544C3.45969 14.3631 2.99951 15.4741 2.99951 16.6325C2.99951 17.7909 3.45969 18.9019 4.27881 19.721C5.09793 20.5401 6.2089 21.0003 7.36731 21.0003C8.52572 21.0003 9.63669 20.5401 10.4558 19.721L11.9998 18.177" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path><path d="M10.4561 13.544C11.2752 14.3628 12.3859 14.8227 13.5441 14.8227C14.7022 14.8227 15.8129 14.3628 16.6321 13.544L19.7211 10.456C20.5402 9.63687 21.0004 8.52591 21.0004 7.36749C21.0004 6.20908 20.5402 5.09811 19.7211 4.27899C18.9019 3.45987 17.791 2.99969 16.6326 2.99969C15.4741 2.99969 14.3632 3.45987 13.5441 4.27899L12.0001 5.82299" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a></h3><p>In our system,<a href="/docs/actions/types/webhook#response-enabled-webhooks"> <u>response-enabled webhooks</u></a> play a critical role, especially in how they interact with Redis Pub/Sub to manage real-time data flows. Before we dive further into the bug, let's take a quick minute to understand the system first.</p><p>Our application is built using<a href="https://rubyonrails.org/"> <u>Ruby on Rails</u></a>, with<a href="https://github.com/puma/puma"> <u>Puma</u></a> as our web server and<a href="https://github.com/rack/rack"> <u>Rack</u></a> as the web server interface. For<a href="/docs/actions/types/webhook#response-enabled-webhooks"> <u>response-enabled webhooks</u></a>, we use a technique known as Socket Hijacking to make async webhooks function like synchronous ones. </p><p>When a webhook request is received, Rack performs what's known as “Full Socket Hijacking.” This process closes the Rails response object, which frees up the request handling thread (inside Puma) to return to the pool of available workers while maintaining the socket connection to the client. This allows the Puma server to accept new web requests without blocking the response for the synchronous webhook, all the while keeping the socket connection open with the client to return a response once it is available.</p><p>Another important part of this system is a simple Redis Pub/Sub implementation running from a dedicated listener thread inside the web server. When a webhook is received, we enqueue the relevant Tines Action Run and open a Redis subscription (from the thread) to listen for the event received when the Tines story exit action generates the response. We then proxy it back to the client.</p><figure class="ImageBlock-module-scss-module__0Ef4dq__imageBlock"><img src="https://www.datocms-assets.com/55802/1722000769-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity-infographic-1.jpg?auto=format%2Ccompress&fit=max&q=90&w=2000&auto=format,compress&q=90&w=2000&fit=max" alt="" width="2400" height="1260" loading="lazy"/></figure><h2 id="the-hidden-bug"><a href="#the-hidden-bug"><span><strong>The hidden bug</strong></span><svg aria-hidden="true" data-heading-link-icon="true" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M13.5438 10.456C12.7247 9.63723 11.614 9.17728 10.4558 9.17728C9.29766 9.17728 8.18692 9.63723 7.36781 10.456L4.27881 13.544C3.45969 14.3631 2.99951 15.4741 2.99951 16.6325C2.99951 17.7909 3.45969 18.9019 4.27881 19.721C5.09793 20.5401 6.2089 21.0003 7.36731 21.0003C8.52572 21.0003 9.63669 20.5401 10.4558 19.721L11.9998 18.177" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path><path d="M10.4561 13.544C11.2752 14.3628 12.3859 14.8227 13.5441 14.8227C14.7022 14.8227 15.8129 14.3628 16.6321 13.544L19.7211 10.456C20.5402 9.63687 21.0004 8.52591 21.0004 7.36749C21.0004 6.20908 20.5402 5.09811 19.7211 4.27899C18.9019 3.45987 17.791 2.99969 16.6326 2.99969C15.4741 2.99969 14.3632 3.45987 13.5441 4.27899L12.0001 5.82299" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a></h2><p>Normally, this dedicated listener thread maintains a persistent connection with Redis to receive and process messages. However, during the upgrade, it encountered connectivity issues, and without any system alerts or logs, the Redis persistent connection died. This failure is one of those classic cases of "the server is running but nothing is working,” just a quiet stoppage that gradually led to service degradation as new webhook requests could no longer be processed.</p><p>It took us a while to reach this conclusion. It required an examination of logs, reading the code, formulating hypotheses, and running some controlled failure cases. We were eventually able to recreate the scenario both locally and in production, and felt confident that this failure mode was the primary contributing factor to the incident.</p><p>This incident highlighted something in our system’s architecture: a lack of a good error handling or recovery mechanism for the webhook's listener thread during Redis connectivity or similar issues. </p><blockquote class="PullQuote-module-scss-module__xEM2FG__pullQuote" data-design="Featured quote"><div class="PullQuote-module-scss-module__xEM2FG__pullQuoteInner"><div class="PullQuote-module-scss-module__xEM2FG__pullQuoteContent"><span class="PullQuote-module-scss-module__xEM2FG__pullQuoteMark"><svg width="36" height="36" viewBox="0 0 36 36" fill="none" aria-hidden="true"><rect width="36" height="36" rx="18" fill="currentColor" fill-opacity="0.18"></rect><path d="M16.864 17.28C16.192 17.28 15.76 17.784 15.76 18.576V21.384H13V18.336C13 16.368 14.272 15 16.168 15H17.296V17.28H16.864ZM18.184 21.384V18.336C18.184 16.368 19.456 15 21.352 15H22.48V17.28H22.048C21.376 17.28 20.944 17.784 20.944 18.576V21.384H18.184Z" fill="currentColor"></path></svg></span><p>This oversight meant that a single point of failure could lead to disruptions in our production environment, a lesson that was both surprising and invaluable for our ongoing efforts to build a more resilient platform.</p></div></div></blockquote><h3 id="lessons-from-the-incident"><a href="#lessons-from-the-incident"><span><strong>Lessons from the incident</strong></span><svg aria-hidden="true" data-heading-link-icon="true" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M13.5438 10.456C12.7247 9.63723 11.614 9.17728 10.4558 9.17728C9.29766 9.17728 8.18692 9.63723 7.36781 10.456L4.27881 13.544C3.45969 14.3631 2.99951 15.4741 2.99951 16.6325C2.99951 17.7909 3.45969 18.9019 4.27881 19.721C5.09793 20.5401 6.2089 21.0003 7.36731 21.0003C8.52572 21.0003 9.63669 20.5401 10.4558 19.721L11.9998 18.177" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path><path d="M10.4561 13.544C11.2752 14.3628 12.3859 14.8227 13.5441 14.8227C14.7022 14.8227 15.8129 14.3628 16.6321 13.544L19.7211 10.456C20.5402 9.63687 21.0004 8.52591 21.0004 7.36749C21.0004 6.20908 20.5402 5.09811 19.7211 4.27899C18.9019 3.45987 17.791 2.99969 16.6326 2.99969C15.4741 2.99969 14.3632 3.45987 13.5441 4.27899L12.0001 5.82299" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a></h3><p>This incident, albeit stressful, was a blessing in disguise. It exposed a critical vulnerability in how our webhook system was designed. The reliance on a single listener thread without a robust fail open mechanism was something that we had overlooked. But thanks to this incident, it was brought to light under circumstances that allowed us to respond without major consequences.</p><p>We learned that our preparations for zero-downtime capabilities during Redis updates were incomplete. We had not considered<a href="/docs/actions/types/webhook#response-enabled-webhooks"> <u>response-enabled webhooks</u></a> in our chaos testing scenarios. This incident was a good reminder of the importance of comprehensive testing and the need for a holistic view of system dependencies and resilience, particularly for non-traditional parts of our system that might not be well-understood.</p><h3 id="embracing-the-incident"><a href="#embracing-the-incident"><span><strong>Embracing the incident</strong></span><svg aria-hidden="true" data-heading-link-icon="true" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M13.5438 10.456C12.7247 9.63723 11.614 9.17728 10.4558 9.17728C9.29766 9.17728 8.18692 9.63723 7.36781 10.456L4.27881 13.544C3.45969 14.3631 2.99951 15.4741 2.99951 16.6325C2.99951 17.7909 3.45969 18.9019 4.27881 19.721C5.09793 20.5401 6.2089 21.0003 7.36731 21.0003C8.52572 21.0003 9.63669 20.5401 10.4558 19.721L11.9998 18.177" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path><path d="M10.4561 13.544C11.2752 14.3628 12.3859 14.8227 13.5441 14.8227C14.7022 14.8227 15.8129 14.3628 16.6321 13.544L19.7211 10.456C20.5402 9.63687 21.0004 8.52591 21.0004 7.36749C21.0004 6.20908 20.5402 5.09811 19.7211 4.27899C18.9019 3.45987 17.791 2.99969 16.6326 2.99969C15.4741 2.99969 14.3632 3.45987 13.5441 4.27899L12.0001 5.82299" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a></h3><p>Incidents happen to everyone, but how you deal with them and learn from them is what counts. What stands out about incidents like these is not just the immediate impact or the technical breakdowns but the invaluable insight they provide into our systems. </p><blockquote class="PullQuote-module-scss-module__xEM2FG__pullQuote" data-design="Featured quote"><div class="PullQuote-module-scss-module__xEM2FG__pullQuoteInner"><div class="PullQuote-module-scss-module__xEM2FG__pullQuoteContent"><span class="PullQuote-module-scss-module__xEM2FG__pullQuoteMark"><svg width="36" height="36" viewBox="0 0 36 36" fill="none" aria-hidden="true"><rect width="36" height="36" rx="18" fill="currentColor" fill-opacity="0.18"></rect><path d="M16.864 17.28C16.192 17.28 15.76 17.784 15.76 18.576V21.384H13V18.336C13 16.368 14.272 15 16.168 15H17.296V17.28H16.864ZM18.184 21.384V18.336C18.184 16.368 19.456 15 21.352 15H22.48V17.28H22.048C21.376 17.28 20.944 17.784 20.944 18.576V21.384H18.184Z" fill="currentColor"></path></svg></span><p>Each incident, especially one that uncovers fundamental flaws, is an opportunity to improve, harden our systems, and ensure better service for our users. </p></div></div></blockquote><p>They push us to think creatively about solutions and to fortify areas we hadn't even considered before.</p><h3 id="action-items"><a href="#action-items"><span><strong>Action items</strong></span><svg aria-hidden="true" data-heading-link-icon="true" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M13.5438 10.456C12.7247 9.63723 11.614 9.17728 10.4558 9.17728C9.29766 9.17728 8.18692 9.63723 7.36781 10.456L4.27881 13.544C3.45969 14.3631 2.99951 15.4741 2.99951 16.6325C2.99951 17.7909 3.45969 18.9019 4.27881 19.721C5.09793 20.5401 6.2089 21.0003 7.36731 21.0003C8.52572 21.0003 9.63669 20.5401 10.4558 19.721L11.9998 18.177" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path><path d="M10.4561 13.544C11.2752 14.3628 12.3859 14.8227 13.5441 14.8227C14.7022 14.8227 15.8129 14.3628 16.6321 13.544L19.7211 10.456C20.5402 9.63687 21.0004 8.52591 21.0004 7.36749C21.0004 6.20908 20.5402 5.09811 19.7211 4.27899C18.9019 3.45987 17.791 2.99969 16.6326 2.99969C15.4741 2.99969 14.3632 3.45987 13.5441 4.27899L12.0001 5.82299" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a></h3><p>While we’re happy that our automated monitoring caught the issue and we were able to mitigate the incident in a very short period, we still want to strive for more. We’ve already ensured the singleton thread acts in a reconciliation loop so that it can withstand similar issues in the future. </p><p>We’re committed to improving our resilience across the board. To achieve this, we’ll be conducting periodic and controlled chaos testing, which will include actions like taking down both our primary and ephemeral data stores to ensure that critical Tines workflows remain unaffected. This process should not only allow us to uncover more hidden unknowns like the ones we saw above but also catch regressions in our tooling and practices as we scale.</p><h3 id="reflecting-on-the-incident"><a href="#reflecting-on-the-incident"><span><strong>Reflecting on the incident</strong></span><svg aria-hidden="true" data-heading-link-icon="true" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor"><path d="M13.5438 10.456C12.7247 9.63723 11.614 9.17728 10.4558 9.17728C9.29766 9.17728 8.18692 9.63723 7.36781 10.456L4.27881 13.544C3.45969 14.3631 2.99951 15.4741 2.99951 16.6325C2.99951 17.7909 3.45969 18.9019 4.27881 19.721C5.09793 20.5401 6.2089 21.0003 7.36731 21.0003C8.52572 21.0003 9.63669 20.5401 10.4558 19.721L11.9998 18.177" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path><path d="M10.4561 13.544C11.2752 14.3628 12.3859 14.8227 13.5441 14.8227C14.7022 14.8227 15.8129 14.3628 16.6321 13.544L19.7211 10.456C20.5402 9.63687 21.0004 8.52591 21.0004 7.36749C21.0004 6.20908 20.5402 5.09811 19.7211 4.27899C18.9019 3.45987 17.791 2.99969 16.6326 2.99969C15.4741 2.99969 14.3632 3.45987 13.5441 4.27899L12.0001 5.82299" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></svg></a></h3><blockquote class="PullQuote-module-scss-module__xEM2FG__pullQuote" data-design="Featured quote"><div class="PullQuote-module-scss-module__xEM2FG__pullQuoteInner"><div class="PullQuote-module-scss-module__xEM2FG__pullQuoteContent"><span class="PullQuote-module-scss-module__xEM2FG__pullQuoteMark"><svg width="36" height="36" viewBox="0 0 36 36" fill="none" aria-hidden="true"><rect width="36" height="36" rx="18" fill="currentColor" fill-opacity="0.18"></rect><path d="M16.864 17.28C16.192 17.28 15.76 17.784 15.76 18.576V21.384H13V18.336C13 16.368 14.272 15 16.168 15H17.296V17.28H16.864ZM18.184 21.384V18.336C18.184 16.368 19.456 15 21.352 15H22.48V17.28H22.048C21.376 17.28 20.944 17.784 20.944 18.576V21.384H18.184Z" fill="currentColor"></path></svg></span><p>As we continue to build and maintain complex software systems, incidents like these are not just inevitable, they’re necessary. </p></div></div></blockquote><p>They teach us about our systems' resilience, the effectiveness of our responses, and the areas that need more attention. They force us to confront the realities of our designs and our decisions. </p><p>So, in a way, we should be thankful for each incident. After all, each one gives us a chance to be better than we were the day before. Let's be thankful for these incidents - they are, indeed, our best friends in the ever-evolving landscape of complex systems.</p></article></div></div><div style="display:block;width:var(--sectionPadding);height:var(--sectionPadding);flex:0 0 var(--sectionPadding);user-select:none"></div></div><!--$--><!--/$--><section class="WildCodeCTASection-module-scss-module__eN7prW__section"><div class="InteractiveCursor-module-scss-module__DEDOvG__region WildCodeCTASection-module-scss-module__eN7prW__wordmark"><div class="WildCodeField-module-scss-module__N5x1Gq__wordmark" role="img" aria-label="Start today"></div></div><div class="WildCodeCTASection-module-scss-module__eN7prW__text"><p class="WildCodeCTASection-module-scss-module__eN7prW__tagline">Sign up today to get started or schedule time with our team to learn more.</p><div class="CtaButtons-module-scss-module__okO8Ua__actions"><a href="https://login.tines.com/saml_idp/signup"><span class="Button-module-scss-module__KA9GbG__buttonWrapper Button-module-scss-module__KA9GbG__fullWidth" style="width:100%"><span class="Button-module-scss-module__KA9GbG__buttonInner Button-module-scss-module__KA9GbG__filled Button-module-scss-module__KA9GbG__mono" style="--buttonColor:var(--pageForegroundColor);--buttonTextColor:var(--pageBg, var(--lightest))"><span class="Button-module-scss-module__KA9GbG__buttonText">Sign up free</span></span></span></a><a href="/book-a-demo/"><span class="Button-module-scss-module__KA9GbG__buttonWrapper Button-module-scss-module__KA9GbG__fullWidth CtaButtons-module-scss-module__okO8Ua__secondary" style="width:100%"><span class="Button-module-scss-module__KA9GbG__buttonInner Button-module-scss-module__KA9GbG__outlined Button-module-scss-module__KA9GbG__mono" style="--buttonColor:var(--pageForegroundColor);--buttonTextColor:var(--pageBg, var(--lightest))"><span class="Button-module-scss-module__KA9GbG__buttonText">Book a demo</span></span></span></a></div></div></section></div></main><footer id="site-footer" class="SiteFooter26-module-scss-module__LzK7FG__footer"><div class="SiteFooter26-module-scss-module__LzK7FG__sections"><div class="SiteFooter26-module-scss-module__LzK7FG__section"><button type="button" class="SiteFooter26-module-scss-module__LzK7FG__sectionToggle" aria-expanded="false"><span class="SiteFooter26-module-scss-module__LzK7FG__sectionTitle">Product</span><svg width="10" height="7" viewBox="0 0 10 7" fill="none" class="SiteFooter26-module-scss-module__LzK7FG__chevron"><path d="M1.04785 1L5.04785 5L9.04785 1" stroke="currentColor" stroke-width="1.5"></path></svg></button><div class="SiteFooter26-module-scss-module__LzK7FG__sectionContent"><div class="SiteFooter26-module-scss-module__LzK7FG__sectionContentInner"><div class="SiteFooter26-module-scss-module__LzK7FG__group"><a class="SiteFooter26-module-scss-module__LzK7FG__groupTitleLink" href="/3b/">Tines 3B</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/3b/examples/">Examples gallery</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/3b/product-updates/">Product updates</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="https://docs.3b.tines.com/" target="_blank" rel="noopener noreferrer">Docs<span class="SiteFooter26-module-scss-module__LzK7FG__externalMark" aria-hidden="true">↗</span></a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="https://status.tines.com/" target="_blank" rel="noopener noreferrer">Status<span class="SiteFooter26-module-scss-module__LzK7FG__externalMark" aria-hidden="true">↗</span></a></div><div class="SiteFooter26-module-scss-module__LzK7FG__group"><a class="SiteFooter26-module-scss-module__LzK7FG__groupTitleLink" href="/stories/">Tines Stories</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/stories/storyboard/">Storyboard</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/stories/cases/">Cases</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/stories/workbench/">Workbench</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/stories/agents/">Agents</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/stories/monitoring/">Monitoring</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/workflow-capability-matrix/">Workflow capability matrix</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/stories/templates/">Templates</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/library/">Library</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/stories/five-minute-flows/">Five-minute flows</a><a href="/university/" class="SiteFooter26-module-scss-module__LzK7FG__leaf">University</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/stories/whats-new/">What’s new</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/stories/docs/">Docs and API</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="https://status.tines.com/" target="_blank" rel="noopener noreferrer">Status<span class="SiteFooter26-module-scss-module__LzK7FG__externalMark" aria-hidden="true">↗</span></a></div><div class="SiteFooter26-module-scss-module__LzK7FG__group"><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/pricing/">Pricing</a></div></div></div></div><div class="SiteFooter26-module-scss-module__LzK7FG__section"><button type="button" class="SiteFooter26-module-scss-module__LzK7FG__sectionToggle" aria-expanded="false"><span class="SiteFooter26-module-scss-module__LzK7FG__sectionTitle">Solutions</span><svg width="10" height="7" viewBox="0 0 10 7" fill="none" class="SiteFooter26-module-scss-module__LzK7FG__chevron"><path d="M1.04785 1L5.04785 5L9.04785 1" stroke="currentColor" stroke-width="1.5"></path></svg></button><div class="SiteFooter26-module-scss-module__LzK7FG__sectionContent"><div class="SiteFooter26-module-scss-module__LzK7FG__sectionContentInner"><div class="SiteFooter26-module-scss-module__LzK7FG__group"><a class="SiteFooter26-module-scss-module__LzK7FG__groupTitleLink" href="/solutions/it/">IT</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/solutions/it/it-as-a-business-enabler/">IT as a business enabler </a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/solutions/it/infrastructure-management/">Infrastructure management</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/solutions/it/networking/">Networking</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/solutions/it/application-management/">Application management</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/solutions/it/it-service-delivery-and-support/">IT service delivery and support</a></div><div class="SiteFooter26-module-scss-module__LzK7FG__group"><a class="SiteFooter26-module-scss-module__LzK7FG__groupTitleLink" href="/solutions/security/">Security</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/solutions/security/ai-soc/">AI SOC</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/solutions/security/soar/">SOAR</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/solutions/security/governance-risk-and-compliance/">GRC</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/solutions/security/threat-intelligence/">Threat intelligence</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/solutions/security/vulnerability-management/">Vulnerability management</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/solutions/security/network-security/">Network security</a></div><div class="SiteFooter26-module-scss-module__LzK7FG__group"><a class="SiteFooter26-module-scss-module__LzK7FG__leaf SiteFooter26-module-scss-module__LzK7FG__leafBold" href="/public-sector/">Public sector</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf SiteFooter26-module-scss-module__LzK7FG__leafBold" href="/financial-services/">Financial services</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf SiteFooter26-module-scss-module__LzK7FG__leafBold" href="/mssps/">MSSPs</a></div><div class="SiteFooter26-module-scss-module__LzK7FG__group"><a href="/professional-services/" class="SiteFooter26-module-scss-module__LzK7FG__leaf">Professional services</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/ai-in-tines/">AI in Tines</a><a href="/enterprise/" class="SiteFooter26-module-scss-module__LzK7FG__leaf">Enterprise</a></div></div></div></div><div class="SiteFooter26-module-scss-module__LzK7FG__section"><button type="button" class="SiteFooter26-module-scss-module__LzK7FG__sectionToggle" aria-expanded="false"><span class="SiteFooter26-module-scss-module__LzK7FG__sectionTitle">Discover</span><svg width="10" height="7" viewBox="0 0 10 7" fill="none" class="SiteFooter26-module-scss-module__LzK7FG__chevron"><path d="M1.04785 1L5.04785 5L9.04785 1" stroke="currentColor" stroke-width="1.5"></path></svg></button><div class="SiteFooter26-module-scss-module__LzK7FG__sectionContent"><div class="SiteFooter26-module-scss-module__LzK7FG__sectionContentInner"><div class="SiteFooter26-module-scss-module__LzK7FG__group"><a class="SiteFooter26-module-scss-module__LzK7FG__groupTitleLink" href="/customers/">Customers</a></div><div class="SiteFooter26-module-scss-module__LzK7FG__group"><a class="SiteFooter26-module-scss-module__LzK7FG__groupTitleLink" href="/blog/">Blog</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/blog/series/">Series</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/blog/guides/">Guides</a><a href="/podcast/" class="SiteFooter26-module-scss-module__LzK7FG__leaf">Podcast</a></div><div class="SiteFooter26-module-scss-module__LzK7FG__group"><a class="SiteFooter26-module-scss-module__LzK7FG__groupTitleLink" href="/events/">Events</a><a href="/webinars/" class="SiteFooter26-module-scss-module__LzK7FG__leaf">Webinars</a><a href="/bootcamps/" class="SiteFooter26-module-scss-module__LzK7FG__leaf">Bootcamps</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/events/builder-connect/">Builder Connect</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="https://www.workflow.live/" target="_blank" rel="noopener noreferrer">Workflow.live<span class="SiteFooter26-module-scss-module__LzK7FG__externalMark" aria-hidden="true">↗</span></a></div><div class="SiteFooter26-module-scss-module__LzK7FG__group"><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="https://community.tines.com/" target="_blank" rel="noopener noreferrer">Community<span class="SiteFooter26-module-scss-module__LzK7FG__externalMark" aria-hidden="true">↗</span></a><a href="/you-did-what-with-tines/" class="SiteFooter26-module-scss-module__LzK7FG__leaf">YDWWT</a></div><div class="SiteFooter26-module-scss-module__LzK7FG__group"><a href="/customer-center/" class="SiteFooter26-module-scss-module__LzK7FG__leaf">Customer center</a></div></div></div></div><div class="SiteFooter26-module-scss-module__LzK7FG__stackedColumn"><div class="SiteFooter26-module-scss-module__LzK7FG__section"><button type="button" class="SiteFooter26-module-scss-module__LzK7FG__sectionToggle" aria-expanded="false"><span class="SiteFooter26-module-scss-module__LzK7FG__sectionTitle">Company</span><svg width="10" height="7" viewBox="0 0 10 7" fill="none" class="SiteFooter26-module-scss-module__LzK7FG__chevron"><path d="M1.04785 1L5.04785 5L9.04785 1" stroke="currentColor" stroke-width="1.5"></path></svg></button><div class="SiteFooter26-module-scss-module__LzK7FG__sectionContent"><div class="SiteFooter26-module-scss-module__LzK7FG__sectionContentInner"><div class="SiteFooter26-module-scss-module__LzK7FG__group"><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/about/">About us</a><a href="/careers/" class="SiteFooter26-module-scss-module__LzK7FG__leaf">Careers</a><a href="/newsroom/" class="SiteFooter26-module-scss-module__LzK7FG__leaf">Newsroom</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/partners/">Partners</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="/contact/">Contact</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="https://store.tines.com/" target="_blank" rel="noopener noreferrer">Store<span class="SiteFooter26-module-scss-module__LzK7FG__externalMark" aria-hidden="true">↗</span></a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="https://live.standards.site/tines" target="_blank" rel="noopener noreferrer">Brand assets<span class="SiteFooter26-module-scss-module__LzK7FG__externalMark" aria-hidden="true">↗</span></a></div><div class="SiteFooter26-module-scss-module__LzK7FG__group"><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="https://www.linkedin.com/company/tines-io/" target="_blank" rel="noopener noreferrer">LinkedIn<span class="SiteFooter26-module-scss-module__LzK7FG__externalMark" aria-hidden="true">↗</span></a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="https://twitter.com/tines_hq" target="_blank" rel="noopener noreferrer">X<span class="SiteFooter26-module-scss-module__LzK7FG__externalMark" aria-hidden="true">↗</span></a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="https://www.youtube.com/@TinesHQ" target="_blank" rel="noopener noreferrer">YouTube<span class="SiteFooter26-module-scss-module__LzK7FG__externalMark" aria-hidden="true">↗</span></a></div></div></div></div><div class="SiteFooter26-module-scss-module__LzK7FG__section"><button type="button" class="SiteFooter26-module-scss-module__LzK7FG__sectionToggle" aria-expanded="false"><span class="SiteFooter26-module-scss-module__LzK7FG__sectionTitle">Trust and security</span><svg width="10" height="7" viewBox="0 0 10 7" fill="none" class="SiteFooter26-module-scss-module__LzK7FG__chevron"><path d="M1.04785 1L5.04785 5L9.04785 1" stroke="currentColor" stroke-width="1.5"></path></svg></button><div class="SiteFooter26-module-scss-module__LzK7FG__sectionContent"><div class="SiteFooter26-module-scss-module__LzK7FG__sectionContentInner"><div class="SiteFooter26-module-scss-module__LzK7FG__group"><a href="/security/" class="SiteFooter26-module-scss-module__LzK7FG__groupTitleLink">Security</a><a href="/tines-security-best-practices/" class="SiteFooter26-module-scss-module__LzK7FG__leaf">Security best practices</a><a class="SiteFooter26-module-scss-module__LzK7FG__leaf" href="https://trust.tines.com/" target="_blank" rel="noopener noreferrer">Trust center<span class="SiteFooter26-module-scss-module__LzK7FG__externalMark" aria-hidden="true">↗</span></a></div><div class="SiteFooter26-module-scss-module__LzK7FG__group"><a href="/legal/" class="SiteFooter26-module-scss-module__LzK7FG__groupTitleLink">Legal</a><a href="/privacy/" class="SiteFooter26-module-scss-module__LzK7FG__leaf">Privacy</a><a href="/terms/" class="SiteFooter26-module-scss-module__LzK7FG__leaf">Terms</a><a href="/dpa/" class="SiteFooter26-module-scss-module__LzK7FG__leaf">DPA</a><a href="/cookies-policy/" class="SiteFooter26-module-scss-module__LzK7FG__leaf">Cookies policy</a></div></div></div></div></div></div><div class="SiteFooter26-module-scss-module__LzK7FG__bottomBar"><span>© Tines</span><a class="SiteFooter26-module-scss-module__LzK7FG__bottomLogo" title="Tines" href="/"><svg class="tines-logo-icon " width="28" height="28" viewBox="0 0 48 48" fill="currentColor"><path fill-rule="evenodd" clip-rule="evenodd" d="M17.6641 0C11.9485 0 6.89478 3.68623 5.18424 9.10302L0.47998 24L5.18424 38.897C6.89479 44.3138 11.9485 48 17.6641 48H30.3358C36.0515 48 41.1052 44.3138 42.8157 38.897L47.52 24L42.8157 9.10302C41.1052 3.68623 36.0515 0 30.3358 0H17.6641ZM30.336 3.74451C34.4042 3.74451 38.0012 6.36822 39.2187 10.2237L40.372 13.8759C38.0518 11.9723 35.1137 10.9008 32.0429 10.9006L15.9228 10.9002C12.8726 10.9004 9.9534 11.9575 7.64056 13.8372L8.78167 10.2237C9.99917 6.36822 13.5962 3.74451 17.6643 3.74451H30.336ZM8.51076 18.2979L5.58159 22.1278L42.3838 22.1278L39.4549 18.2983C37.6948 15.9969 34.9524 14.6453 32.0426 14.6451H15.835C12.958 14.6723 10.2531 16.0197 8.51076 18.2979ZM42.3778 25.8723L39.4547 29.6983C37.6944 32.0023 34.9501 33.3555 32.0381 33.3553L15.9271 33.3548C13.0152 33.3551 10.2708 32.0019 8.51056 29.6979L5.58776 25.8723L42.3778 25.8723ZM8.78145 37.7763C9.99894 41.6318 13.5959 44.2555 17.6641 44.2555H30.3358C34.404 44.2555 38.001 41.6318 39.2185 37.7763L40.3726 34.1214C38.0514 36.0273 35.111 37.1 32.0378 37.0998L15.9274 37.0994C12.8747 37.0996 9.9531 36.0412 7.63915 34.159L8.78145 37.7763Z"></path></svg></a><button class="SiteFooter26-module-scss-module__LzK7FG__cookieSettings">Cookie settings</button></div></footer></div><script src="/_next/static/immutable/chunks/2iox-02sx9loa.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n4:I[11926,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"default\"]\n5:I[550880,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"default\"]\n7:I[311235,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"OutletBoundary\"]\n8:\"$Sreact.suspense\"\nb:I[311235,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"ViewportBoundary\"]\nd:I[311235,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"MetadataBoundary\"]\nf:I[940405,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"default\",1]\n:HL[\"/_next/static/immutable/chunks/132trk_zc3ylu.css\",\"style\"]\n:HL[\"/_next/static/immutable/chunks/1bmd2ryn1r-hv.css\",\"style\"]\n:HL[\"/_next/static/immutable/chunks/3qp9-r2fgklws.css\",\"style\"]\n:HL[\"/_next/static/immutable/chunks/43xkgk33-6316.css\",\"style\"]\n:HL[\"/_next/static/immutable/chunks/3w0wy298bmb96.css\",\"style\"]\n:HL[\"/_next/static/immutable/chunks/27k1lbkm9ytw4.css\",\"style\"]\n:HL[\"/_next/static/immutable/chunks/45auz_6fncqnw.css\",\"style\"]\n:HL[\"/_next/static/immutable/chunks/3a0f_y332k2v1.css\",\"style\"]\na:X\n0:{\"P\":null,\"c\":[\"\",\"blog\",\"engineering-incidents-improvement\",\"\"],\"q\":\"\",\"i\":false,\"f\":[[[\"\",{\"children\":[\"blog\",{\"children\":[[\"segments\",\"engineering-incidents-improvement\",\"c\",[\"guides\",\"series\"]],{\"children\":[\"__PAGE__\",{},\"$undefined\",\"$undefined\",4096]},\"$undefined\",\"$undefined\",4096]},\"$undefined\",\"$undefined\",4096]},\"$undefined\",\"$undefined\",4112],[[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/132trk_zc3ylu.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"1\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/1bmd2ryn1r-hv.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"2\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/3qp9-r2fgklws.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"3\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/43xkgk33-6316.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"4\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/3w0wy298bmb96.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-1\",{\"src\":\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-2\",{\"src\":\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-3\",{\"src\":\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-4\",{\"src\":\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-5\",{\"src\":\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-6\",{\"src\":\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\",\"async\":true,\"nonce\":\"$undefined\"}]],\"$L2\"]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/27k1lbkm9ytw4.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"1\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/45auz_6fncqnw.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/immutable/chunks/3pjhofejz57jt.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-1\",{\"src\":\"/_next/static/immutable/chunks/1q-89ajkqbbkw.js\",\"async\":true,\"nonce\":\"$undefined\"}]],\"$L3\"]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[null,[\"$\",\"$L4\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L5\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]]}],{\"children\":[[\"$\",\"$1\",\"c\",{\"children\":[\"$L6\",[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/3a0f_y332k2v1.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-0\",{\"src\":\"/_next/static/immutable/chunks/2uho2m59t6w8f.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-1\",{\"src\":\"/_next/static/immutable/chunks/2ti9pdu9p4n5l.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-2\",{\"src\":\"/_next/static/immutable/chunks/31qqpjo4d7384.js\",\"async\":true,\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-3\",{\"src\":\"/_next/static/immutable/chunks/3zb6qebqmln4d.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"$L7\",null,{\"children\":[\"$\",\"$8\",null,{\"name\":\"Next.MetadataOutlet\",\"children\":\"$@9\"}]}]]}],{},null,false,null]},null,false,\"$a\"]},null,false,null]},null,false,null],[\"$\",\"$1\",\"h\",{\"children\":[null,[\"$\",\"$Lb\",null,{\"children\":\"$Lc\"}],[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$Ld\",null,{\"children\":[\"$\",\"$8\",null,{\"name\":\"Next.Metadata\",\"children\":\"$Le\"}]}]}],null]}],false]],\"m\":\"$undefined\",\"G\":[\"$f\",[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/132trk_zc3ylu.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"1\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/1bmd2ryn1r-hv.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"2\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/3qp9-r2fgklws.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"3\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/43xkgk33-6316.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"link\",\"4\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/3w0wy298bmb96.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}]]],\"S\":false,\"h\":null,\"r\":\"$undefined\",\"s\":\"$undefined\",\"a\":\"$undefined\",\"l\":\"$undefined\",\"p\":\"$undefined\",\"d\":\"$undefined\"}\na:C\nc:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}]]\n15:I[62200,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\",\"/_next/static/immutable/chunks/3pjhofejz57jt.js\",\"/_next/static/immutable/chunks/1q-89ajkqbbkw.js\",\"/_next/static/immutable/chunks/2uho2m59t6w8f.js\",\"/_next/static/immutable/chunks/2ti9pdu9p4n5l.js\",\"/_next/static/immutable/chunks/31qqpjo4d7384.js\",\"/_next/static/immutable/chunks/3zb6qebqmln4d.js\"],\"AreaNavActiveProvider\"]\n16:I[380637,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\",\"/_next/static/immutable/chunks/3pjhofejz57jt.js\",\"/_next/static/immutable/chunks/1q-89ajkqbbkw.js\"],\"AreaNav\"]\n17:I[323752,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\",\"/_next/static/immutable/chunks/3pjhofejz57jt.js\",\"/_next/static/immutable/chunks/1q-89ajkqbbkw.js\"],\"WildCodeCTASection\"]\n:HL[\"/fonts/Roobert-Regular.woff2\",\"font\",{\"crossOrigin\":\"anonymous\",\"type\":\"font/woff2\"}]\n:HL[\"/fonts/Reckless-Light.woff2\",\"font\",{\"crossOrigin\":\"anonymous\",\"type\":\"font/woff2\"}]\n:HL[\"/fonts/Reckless-Regular.woff2\",\"font\",{\"crossOrigin\":\"anonymous\",\"type\":\"font/woff2\"}]\n10:T699,(function(){var d=document.documentElement,p=location.pathname,R=[\"/stories/docs\",\"/stories/whats-new\",\"/workflow-capability-matrix\",\"/3b/examples\",\"/3b/product-updates\",\"/library\",\"/stories/five-minute-flows\"],F=[\"/stories/five-minute-flows\"],M=[\"/workflow-capability-matrix\"];function tc(c){var m=document.querySelector('meta[name=\"theme-color\"]');if(m)m.setAttribute(\"content\",c)}function on(L){return L.some(function(r){return p===r||p.indexOf(r+\"/\")===0})}var ok=on(R);if(!ok){d.removeAttribute(\"data-dark-mode-supported\");d.removeAttribute(\"data-page-theme\");d.setAttribute(\"data-color-scheme\",\"light\");d.setAttribute(\"data-resolved-scheme\",\"light\");tc(\"#fcf9f5\");if(on([\"/campaigns\"])){d.setAttribute(\"data-campaign-nav\",\"\")}return}d.setAttribute(\"data-dark-mode-supported\",\"\");if(on(F)){d.setAttribute(\"data-color-scheme\",\"dark\");d.setAttribute(\"data-resolved-scheme\",\"dark\");tc(\"#16131e\");return}var pt=document.cookie.match(/page-theme=(\\w+)/);var mq=/[?\u0026]matrix=on/.test(location.search);if(on(M)\u0026\u0026((pt\u0026\u0026pt[1]===\"matrix\")||mq)){d.setAttribute(\"data-page-theme\",\"matrix\");d.setAttribute(\"data-color-scheme\",\"dark\");d.setAttribute(\"data-resolved-scheme\",\"dark\");tc(\"#0b1a17\");return}else{d.removeAttribute(\"data-page-theme\")}var cm=document.cookie.match(/preferred-color-scheme=(\\w+)/);var s;if(cm){s=cm[1]}else{var sv=d.getAttribute(\"data-color-scheme\");s=sv\u0026\u0026sv!==\"light\"?sv:\"auto\"}var r=s===\"dark\"?\"dark\":s===\"light\"?\"light\":window.matchMedia(\"(prefers-color-scheme:dark)\").matches?\"dark\":\"light\";d.setAttribute(\"data-color-scheme\",s);d.setAttribute(\"data-resolved-scheme\",r);tc(r===\"dark\"?\"#16131e\":\"#fcf9f5\");if(on([\"/campaigns\"])){d.setAttribute(\"data-campaign-nav\",\"\")}})()11:T53a,{\"@context\":\"https://schema.org\",\"@type\":\"Organization\",\"name\":\"Tines\",\"url\":\"https://www.tines.com/\",\"@id\":\"https://www.tines.com/#org\",\"description\":\"Tines 3B enables every team to build AI agents, apps, and automation in one safe, secure environment — with the control Security and IT need to govern it at scale.\",\"foundingDate\":\"2018\",\"founders\":[{\"@type\":\"Person\",\"name\":\"Eoin Hinchy\"},{\"@type\":\"Person\",\"name\":\"Thomas Kinsella\"}],\"sameAs\":[\"https://www.linkedin.com/company/tines-io/\",\"https://twitter.com/tines_hq\",\"https://www.youtube.com/@TinesHQ\",\"https://github.com/tines\"],\"contactPoint\":[{\"@type\":\"ContactPoint\",\"contactType\":\"customer support\",\"email\":\"hello@tines.com\",\"availableLanguage\":[\"en\"]},{\"@type\":\"ContactPoint\",\"contactType\":\"analyst relations\",\"email\":\"analysts@tines.com\",\"availableLanguage\":[\"en\"]},{\"@type\":\"ContactPoint\",\"contactType\":\"media relations\",\"email\":\"pr@tines.com\",\"availableLanguage\":[\"en\"]}],\"address\":[{\"@type\":\"PostalAddress\",\"streetAddress\":\"10 Westland Square, Pearse Street\",\"addressLocality\":\"Dublin\",\"postalCode\":\"D02 CX36\",\"addressCountry\":\"IE\",\"name\":\"Tines Limited\"},{\"@type\":\"PostalAddress\",\"streetAddress\":\"99 High Street, 14th Floor\",\"addressLocality\":\"Boston\",\"addressRegion\":\"MA\",\"postalCode\":\"02111\",\"addressCountry\":\"US\",\"name\":\"Tines Technology Inc.\"}],\"areaServed\":\"Global\"}2:[\"$\",\"html\",null,{\"lang\":\"en-US\",\"data-color-scheme\":\"light\",\"data-resolved-scheme\":\"light\",\"suppressHydrationWarning\":true,\"children\":[[\"$\",\"head\",null,{\"children\":[[\"$\",\"script\",null,{\"id\":\"color-scheme-init\",\"dangerouslySetInnerHTML\":{\"__html\":\"$10\"}}],[\"$\",\"meta\",null,{\"name\":\"theme-color\",\"content\":\"#fcf9f5\"}],[\"$\",\"link\",null,{\"rel\":\"preload\",\"href\":\"/fonts/Roobert-Regular.woff2\",\"as\":\"font\",\"type\":\"font/woff2\",\"crossOrigin\":\"anonymous\"}],[\"$\",\"link\",null,{\"rel\":\"preload\",\"href\":\"/fonts/Reckless-Light.woff2\",\"as\":\"font\",\"type\":\"font/woff2\",\"crossOrigin\":\"anonymous\"}],[\"$\",\"link\",null,{\"rel\":\"preload\",\"href\":\"/fonts/Reckless-Regular.woff2\",\"as\":\"font\",\"type\":\"font/woff2\",\"crossOrigin\":\"anonymous\"}]]}],[\"$\",\"body\",null,{\"children\":[[\"$\",\"script\",null,{\"type\":\"application/ld+json\",\"dangerouslySetInnerHTML\":{\"__html\":\"$11\"}}],\"$L12\",\"$L13\",\"$L14\"]}]]}]\n3:[\"$\",\"div\",null,{\"children\":[null,[\"$\",\"$L15\",null,{\"children\":[[\"$\",\"$L16\",null,{\"nav\":{\"id\":\"blog\",\"prefix\":\"/blog\",\"pill\":{\"label\":\"Blog\",\"href\":\"/blog\"},\"sections\":[{\"label\":\"Home\",\"href\":\"/blog\"},{\"label\":\"Categories\",\"flattenWhenWide\":true,\"links\":[{\"label\":\"News\",\"href\":\"/blog/news\"},{\"label\":\"Tutorials\",\"href\":\"/blog/tutorials\"},{\"label\":\"Engineering\",\"href\":\"/blog/engineering\"},{\"label\":\"Product updates\",\"href\":\"/blog/product-updates\"},{\"label\":\"Inside Tines\",\"href\":\"/blog/inside-tines\"},{\"label\":\"Tines × Tines\",\"href\":\"/blog/tines-by-tines\"}]},{\"label\":\"Guides\",\"href\":\"/blog/guides\"},{\"label\":\"All articles\",\"href\":\"/blog/articles\"}]}}],[\"$\",\"$L4\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L5\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":\"$undefined\",\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}],[\"$\",\"$L17\",null,{}]]}]]}]\n18:I[12579,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"default\"]\n19:I[757183,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"AppCookiesProvider\"]\n1a:I[44679,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"CookieConsentInit\"]\n1b:I[209507,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"PageViews\"]\n1c:I[102237,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"PageThemeRouteSync\"]\n1d:I[373871,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"default\"]\n1e:I[871026,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"SiteNav26\"]\n1f:I[414512,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"SiteNavHeightTracker\"]\n20:I[841241,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\",\"/_next/static/immutable/chunks/3pjhofejz57jt.js\",\"/_next/static/immutable/chunks/1q-89ajkqbbkw.js\",\"/_next/static/immutable/chunks/2uho2m59t6w8f.js\",\"/_next/static/immutable/chunks/2ti9pdu9p4n5l.js\",\"/_next/static/immutable/chunks/31qqpjo4d7384.js\",\"/_next/static/immutable/chunks/3zb6qebqmln4d.js\"],\"Image\"]\n21:I[980849,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\",\"/_next/static/immutable/chunks/3pjhofejz57jt.js\",\"/_next/static/immutable/chunks/1q-89ajkqbbkw.js\",\"/_next/static/immutable/chunks/2uho2m59t6w8f.js\",\"/_next/static/immutable/chunks/2ti9pdu9p4n5l.js\",\"/_next/static/immutable/chunks/31qqpjo4d7384.js\",\"/_next/static/immutable/chunks/3zb6qebqmln4d.js\"],\"default\"]\n23:I[20190,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"DevGridOverlay\"]\n24:I[70011,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"DevPanel\"]\n25:I[535897,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"PreviewModeDevTag\"]\n26:I[868430,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"\"]\n:HL[\"/_next/static/immutable/chunks/3_-011rjao1d9.css\",\"style\"]\n12:[\"$\",\"$L18\",null,{}]\n13:[\"$\",\"$L19\",null,{\"cookies\":{\"emailAddress\":null,\"lastUsedTenantDomain\":\"\"},\"children\":[[\"$\",\"$L1a\",null,{}],[\"$\",\"$L1b\",null,{}],[\"$\",\"$L1c\",null,{}],[\"$\",\"$L1d\",null,{\"children\":[[\"$\",\"div\",null,{\"className\":\"PageBackdrop-module-scss-module__KKf-uW__pageShell\",\"children\":[[\"$\",\"div\",null,{\"id\":\"page-backdrop-portal\",\"className\":\"PageBackdrop-module-scss-module__KKf-uW__slot\",\"aria-hidden\":\"true\"}],[\"$\",\"a\",null,{\"href\":\"#main-content\",\"className\":\"skip-link\",\"children\":\"Skip to content\"}],[\"$\",\"$L1e\",null,{\"featured\":{\"blog\":{\"title\":\"We gave all our employees access to Tines 3B. Here's what happened next.\",\"href\":\"/blog/tines-3b-first-90/\",\"imageUrl\":\"https://www.datocms-assets.com/55802/1788825912-2026-09-we-gave-employees-3b-header.png?auto=format%2Ccompress\u0026fit=crop\u0026h=420\u0026q=70\u0026w=800\"},\"event\":{\"title\":\"Wild code: You can’t govern what you can’t see\",\"href\":\"/webinars/wild-code-you-cant-govern-what-you-cant-see/\",\"imageUrl\":\"https://www.datocms-assets.com/55802/1785849013-techadvicewebinar-dato-1200-x-630-2.png?auto=format%2Ccompress\u0026fit=crop\u0026h=420\u0026q=70\u0026w=800\"}}}],[\"$\",\"$L1f\",null,{}],[\"$\",\"main\",null,{\"id\":\"main-content\",\"children\":[\"$\",\"$L4\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L5\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[null,[\"$\",\"div\",null,{\"className\":\"not-found-module-scss-module__DyuKYG__wrap\",\"children\":[[\"$\",\"$L20\",null,{\"className\":\"not-found-module-scss-module__DyuKYG__illustration\",\"src\":\"/v5-assets/images/tines-404-illustration-ufo.png\",\"alt\":\"\",\"width\":120,\"height\":120}],[\"$\",\"h1\",null,{\"className\":\"not-found-module-scss-module__DyuKYG__heading\",\"children\":\"Page not found\"}],[\"$\",\"p\",null,{\"className\":\"not-found-module-scss-module__DyuKYG__subtext\",\"children\":\"We couldn’t find what you were looking for.\"}],[\"$\",\"$L21\",null,{\"to\":\"/\",\"color\":\"purple\",\"shade\":700,\"children\":\"Go home\"}]]}]],[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/immutable/chunks/3_-011rjao1d9.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}]]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}]}],\"$L22\"]}],[\"$\",\"$L23\",null,{}],[\"$\",\"$L24\",null,{}],[\"$\",\"$L25\",null,{}]]}]]}]\n14:[[\"$\",\"$L26\",null,{\"id\":\"gtag-shim\",\"strategy\":\"afterInteractive\",\"data-cookieconsent\":\"ignore\",\"dangerouslySetInnerHTML\":{\"__html\":\"window.dataLayer=window.dataLayer||[];window.gtag=window.gtag||function(){dataLayer.push(arguments)};\"}}],[\"$\",\"$L26\",null,{\"id\":\"gtm-loader\",\"strategy\":\"afterInteractive\",\"data-cookieconsent\":\"ignore\",\"dangerouslySetInnerHTML\":{\"__html\":\"(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({\\\"gtm.start\\\":new Date().getTime(),event:\\\"gtm.js\\\"});var f=d.getElementsByTagName(s)[0],j=d.createElement(s),dl=l!=\\\"dataLayer\\\"?\\\"\u0026l=\\\"+l:\\\"\\\";j.async=true;j.src=\\\"https://www.googletagmanager.com/gtm.js?id=\\\"+i+dl;f.parentNode.insertBefore(j,f)})(window,document,\\\"script\\\",\\\"dataLayer\\\",\\\"GTM-5Z8VZQ5\\\");\"}}]]\n27:I[586848,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"FooterSection\"]\n28:I[693529,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\",\"/_next/static/immutable/chunks/3pjhofejz57jt.js\",\"/_next/static/immutable/chunks/1q-89ajkqbbkw.js\",\"/_next/static/immutable/chunks/2uho2m59t6w8f.js\",\"/_next/static/immutable/chunks/2ti9pdu9p4n5l.js\",\"/_next/static/immutable/chunks/31qqpjo4d7384.js\",\"/_next/static/immutable/chunks/3zb6qebqmln4d.js\"],\"\"]\n22:[\"$\",\"footer\",null,{\"id\":\"site-footer\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__footer\",\"children\":[[\"$\",\"div\",null,{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__sections\",\"children\":[[[\"$\",\"$L27\",\"Product\",{\"title\":\"Product\",\"children\":[[\"$\",\"div\",\"0\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__group\",\"children\":[[\"$\",\"$L28\",null,{\"href\":\"/3b/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__groupTitleLink\",\"children\":\"Tines 3B\"}],[[\"$\",\"$L28\",\"Examples gallery\",{\"href\":\"/3b/examples/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Examples gallery\"}],[\"$\",\"$L28\",\"Product updates\",{\"href\":\"/3b/product-updates/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Product updates\"}],[\"$\",\"a\",\"Docs\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"href\":\"https://docs.3b.tines.com/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"Docs\",[\"$\",\"span\",null,{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__externalMark\",\"aria-hidden\":\"true\",\"children\":\"↗\"}]]}],[\"$\",\"a\",\"Status\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"href\":\"https://status.tines.com/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"Status\",[\"$\",\"span\",null,{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__externalMark\",\"aria-hidden\":\"true\",\"children\":\"↗\"}]]}]]]}],[\"$\",\"div\",\"1\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__group\",\"children\":[[\"$\",\"$L28\",null,{\"href\":\"/stories/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__groupTitleLink\",\"children\":\"Tines Stories\"}],[[\"$\",\"$L28\",\"Storyboard\",{\"href\":\"/stories/storyboard/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Storyboard\"}],[\"$\",\"$L28\",\"Cases\",{\"href\":\"/stories/cases/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Cases\"}],[\"$\",\"$L28\",\"Workbench\",{\"href\":\"/stories/workbench/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Workbench\"}],[\"$\",\"$L28\",\"Agents\",{\"href\":\"/stories/agents/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Agents\"}],[\"$\",\"$L28\",\"Monitoring\",{\"href\":\"/stories/monitoring/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Monitoring\"}],[\"$\",\"$L28\",\"Workflow capability matrix\",{\"href\":\"/workflow-capability-matrix/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Workflow capability matrix\"}],[\"$\",\"$L28\",\"Templates\",{\"href\":\"/stories/templates/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Templates\"}],[\"$\",\"$L28\",\"Library\",{\"href\":\"/library/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Library\"}],[\"$\",\"$L28\",\"Five-minute flows\",{\"href\":\"/stories/five-minute-flows/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Five-minute flows\"}],[\"$\",\"a\",\"University\",{\"href\":\"/university/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"University\"}],[\"$\",\"$L28\",\"What’s new\",{\"href\":\"/stories/whats-new/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"What’s new\"}],[\"$\",\"$L28\",\"Docs and API\",{\"href\":\"/stories/docs/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Docs and API\"}],[\"$\",\"a\",\"Status\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"href\":\"https://status.tines.com/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"Status\",[\"$\",\"span\",null,{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__externalMark\",\"aria-hidden\":\"true\",\"children\":\"↗\"}]]}]]]}],[\"$\",\"div\",\"2\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__group\",\"children\":[null,[[\"$\",\"$L28\",\"Pricing\",{\"href\":\"/pricing/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Pricing\"}]]]}]]}],[\"$\",\"$L27\",\"Solutions\",{\"title\":\"Solutions\",\"children\":[[\"$\",\"div\",\"0\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__group\",\"children\":[[\"$\",\"$L28\",null,{\"href\":\"/solutions/it/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__groupTitleLink\",\"children\":\"IT\"}],[[\"$\",\"$L28\",\"IT as a business enabler \",{\"href\":\"/solutions/it/it-as-a-business-enabler/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"IT as a business enabler \"}],\"$L29\",\"$L2a\",\"$L2b\",\"$L2c\"]]}],\"$L2d\",\"$L2e\",\"$L2f\"]}],\"$L30\"],\"$L31\"]}],\"$L32\"]}]\n34:I[986992,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"CookieSettingsButton\"]\n29:[\"$\",\"$L28\",\"Infrastructure management\",{\"href\":\"/solutions/it/infrastructure-management/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Infrastructure management\"}]\n2a:[\"$\",\"$L28\",\"Networking\",{\"href\":\"/solutions/it/networking/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Networking\"}]\n2b:[\"$\",\"$L28\",\"Application management\",{\"href\":\"/solutions/it/application-management/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Application management\"}]\n2c:[\"$\",\"$L28\",\"IT service delivery and support\",{\"href\":\"/solutions/it/it-service-delivery-and-support/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"IT service delivery and support\"}]\n2d:[\"$\",\"div\",\"1\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__group\",\"children\":[[\"$\",\"$L28\",null,{\"href\":\"/solutions/security/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__groupTitleLink\",\"children\":\"Security\"}],[[\"$\",\"$L28\",\"AI SOC\",{\"href\":\"/solutions/security/ai-soc/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"AI SOC\"}],[\"$\",\"$L28\",\"SOAR\",{\"href\":\"/solutions/security/soar/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"SOAR\"}],[\"$\",\"$L28\",\"GRC\",{\"href\":\"/solutions/security/governance-risk-and-compliance/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"GRC\"}],[\"$\",\"$L28\",\"Threat intelligence\",{\"href\":\"/solutions/security/threat-intelligence/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Threat intelligence\"}],[\"$\",\"$L28\",\"Vulnerability management\",{\"href\":\"/solutions/security/vulnerability-management/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Vulnerability management\"}],[\"$\",\"$L28\",\"Network security\",{\"href\":\"/solutions/security/network-security/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Network security\"}]]]}]\n2e:[\"$\",\"div\",\"2\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__group\",\"children\":[null,[[\"$\",\"$L28\",\"Public sector\",{\"href\":\"/public-sector/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf SiteFooter26-module-scss-module__LzK7FG__leafBold\",\"children\":\"Public sector\"}],[\"$\",\"$L28\",\"Financial services\",{\"href\":\"/financial-services/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf SiteFooter26-module-scss-module__LzK7FG__leafBold\",\"children\":\"Financial services\"}],[\"$\",\"$L28\",\"MSSPs\",{\"href\":\"/mssps/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf SiteFooter26-module-scss-module__LzK7FG__leafBold\",\"children\":\"MSSPs\"}]]]}]\n2f:[\"$\",\"div\",\"3\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__group\",\"children\":[null,[[\"$\",\"a\",\"Professional services\",{\"href\":\"/professional-services/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Professional services\"}],[\"$\",\"$L28\",\"AI in Tines\",{\"href\":\"/ai-in-tines/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"AI in Tines\"}],[\"$\",\"a\",\"Enterprise\",{\"href\":\"/enterprise/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Enterprise\"}]]]}]\n30:[\"$\",\"$L27\",\"Discover\",{\"title\":\"Discover\",\"children\":[[\"$\",\"div\",\"0\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__group\",\"children\":[[\"$\",\"$L28\",null,{\"href\":\"/customers/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__groupTitleLink\",\"children\":\"Customers\"}],[]]}],[\"$\",\"div\",\"1\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__group\",\"children\":[[\"$\",\"$L28\",null,{\"href\":\"/blog/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__groupTitleLink\",\"children\":\"Blog\"}],[[\"$\",\"$L28\",\"Series\",{\"href\":\"/blog/series/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Series\"}],[\"$\",\"$L28\",\"Guides\",{\"href\":\"/blog/guides/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Guides\"}],[\"$\",\"a\",\"Podcast\",{\"href\":\"/podcast/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Podcast\"}]]]}],[\"$\",\"div\",\"2\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__group\",\"children\":[[\"$\",\"$L28\",null,{\"href\":\"/events/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__groupTitleLink\",\"children\":\"Events\"}],[[\"$\",\"a\",\"Webinars\",{\"href\":\"/webinars/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Webinars\"}],[\"$\",\"a\",\"Bootcamps\",{\"href\":\"/bootcamps/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Bootcamps\"}],[\"$\",\"$L28\",\"Builder Connect\",{\"href\":\"/events/builder-connect/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Builder Connect\"}],[\"$\",\"a\",\"Workflow.live\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"href\":\"https://www.workflow.live/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"Workflow.live\",[\"$\",\"span\",null,{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__externalMark\",\"aria-hidden\":\"true\",\"children\":\"↗\"}]]}]]]}],[\"$\",\"div\",\"3\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__group\",\"children\":[null,[[\"$\",\"a\",\"Community\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"href\":\"https://community.tines.com/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"Community\",[\"$\",\"span\",null,{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__externalMark\",\"aria-hidden\":\"true\",\"children\":\"↗\"}]]}],[\"$\",\"a\",\"YDWWT\",{\"href\":\"/you-did-what-with-tines/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"YDWWT\"}]]]}],[\"$\",\"div\",\"4\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__group\",\"children\":[null,[[\"$\",\"a\",\"Customer center\",{\"href\":\"/customer-center/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Customer center\"}]]]}]]}]\n31:[\"$\",\"div\",null,{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__stackedColumn\",\"children\":[[\"$\",\"$L27\",\"Company\",{\"title\":\"Company\",\"children\":[[\"$\",\"div\",\"0\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__group\",\"children\":[null,[[\"$\",\"$L28\",\"About us\",{\"href\":\"/about/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"About us\"}],[\"$\",\"a\",\"Careers\",{\"href\":\"/careers/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Careers\"}],[\"$\",\"a\",\"Newsroom\",{\"href\":\"/newsroom/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Newsroom\"}],[\"$\",\"$L28\",\"Partners\",{\"href\":\"/partners/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Partners\"}],[\"$\",\"$L28\",\"Contact\",{\"href\":\"/contact/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Contact\"}],[\"$\",\"a\",\"Store\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"href\":\"https://store.tines.com/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"Store\",[\"$\",\"span\",null,{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__externalMark\",\"aria-hidden\":\"true\",\"children\":\"↗\"}]]}],[\"$\",\"a\",\"Brand assets\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"href\":\"https://live.standards.site/tines\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"Brand assets\",[\"$\",\"span\",null,{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__externalMark\",\"aria-hidden\":\"true\",\"children\":\"↗\"}]]}]]]}],[\"$\",\"div\",\"1\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__group\",\"children\":[null,[[\"$\",\"a\",\"LinkedIn\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"href\":\"https://www.linkedin.com/company/tines-io/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"LinkedIn\",[\"$\",\"span\",null,{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__externalMark\",\"aria-hidden\":\"true\",\"children\":\"↗\"}]]}],[\"$\",\"a\",\"X\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"href\":\"https://twitter.com/tines_hq\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"X\",[\"$\",\"span\",null,{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__externalMark\",\"aria-hidden\":\"true\",\"children\":\"↗\"}]]}],[\"$\",\"a\",\"YouTube\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"href\":\"https://www.youtube.com/@TinesHQ\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"YouTube\",[\"$\",\"span\",null,{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__externalMark\",\"aria-hidden\":\"true\",\"children\":\"↗\"}]]}]]]}]]}],[\"$\",\"$L27\",\"Trust and security\",{\"title\":\"Trust and security\",\"children\":[[\"$\",\"div\",\"0\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__group\",\"children\":[[\"$\",\"a\",null,{\"href\":\"/security/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__groupTitleLink\",\"children\":\"Security\"}],[[\"$\",\"a\",\"Security best practices\",{\"href\":\"/tines-security-best-practices/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Security best practices\"}],[\"$\",\"a\",\"Trust center\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"href\":\"https://trust.tines.com/\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":[\"Trust center\",[\"$\",\"span\",null,{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__externalMark\",\"aria-hidden\":\"true\",\"children\":\"↗\"}]]}]]]}],[\"$\",\"div\",\"1\",{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__group\",\"children\":[[\"$\",\"a\",null,{\"href\":\"/legal/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__groupTitleLink\",\"children\":\"Legal\"}],[[\"$\",\"a\",\"Privacy\",{\"href\":\"/privacy/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Privacy\"}],[\"$\",\"a\",\"Terms\",{\"href\":\"/terms/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Terms\"}],[\"$\",\"a\",\"DPA\",{\"href\":\"/dpa/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"DPA\"}],[\"$\",\"a\",\"Cookies policy\",{\"href\":\"/cookies-policy/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__leaf\",\"children\":\"Cookies policy\"}]]]}]]}]]}]\n33:T44a,M17.6641 0C11.9485 0 6.89478 3.68623 5.18424 9.10302L0.47998 24L5.18424 38.897C6.89479 44.3138 11.9485 48 17.6641 48H30.3358C36.0515 48 41.1052 44.3138 42.8157 38.897L47.52 24L42.8157 9.10302C41.1052 3.68623 36.0515 0 30.3358 0H17.6641ZM30.336 3.74451C34.4042 3.74451 38.0012 6.36822 39.2187 10.2237L40.372 13.8759C38.0518 11.9723 35.1137 10.9008 32.0429 10.9006L15.9228 10.9002C12.8726 10.9004 9.9534 11.9575 7.64056 13.8372L8.78167 10.2237C9.99917 6.36822 13.5962 3.74451 17.6643 3.74451H30.336ZM8.51076 18.2979L5.58159 22.1278L42.3838 22.1278L39.4549 18.2983C37.6948 15.9969 34.9524 14.6453 32.0426 14.6451H15.835C12.958 14.6723 10.2531 16.0197 8.51076 18.2979ZM42.3778 25.8723L39.4547 29.6983C37.6944 32.0023 34.9501 33.3555 32.0381 33.3553L15.9271 33.3548C13.0152 33.3551 10.2708 32.0019 8.51056 29.6979L5.58776 25.8723L42.3778 25.8723ZM8.78145 37.7763C9.99894 41.6318 13.5959 44.2555 17.6641 44.2555H30.3358C34.404 44.2555 38.001 41.6318 39.2185 37.7763L40.3726 34.1214C38.0514 36.0273 35.111 37.1 32.0378 37.0998L15.9274 37.0994C12.8747 37.0996 9.9531 36.0412 7.63915 34.159L8.78145 37.7763Z32:[\"$\",\"div\",null,{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__bottomBar\",\"children\":[[\"$\",\"span\",null,{\"children\":\"© Tines\"}],[\"$\",\"$L28\",null,{\"href\":\"/\",\"className\":\"SiteFooter26-module-scss-module__LzK7FG__bottomLogo\",\"title\":\"Tines\",\"children\":[\"$\",\"svg\",null,{\"className\":\"tines-logo-icon \",\"width\":28,\"height\":28,\"viewBox\":\"0 0 48 48\",\"fill\":\"currentColor\",\"children\":[\"$\",\"path\",null,{\"fillRule\":\"evenodd\",\"clipRule\":\"evenodd\",\"d\":\"$33\"}]}]}],[\"$\",\"$L34\",null,{\"className\":\"SiteFooter26-module-scss-module__LzK7FG__cookieSettings\"}]]}]\n36:I[875931,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\"],\"IconMark\"]\n6:[\"$\",\"div\",null,{\"className\":\"blog-module-scss-module__RlMr6a__contentColumn\",\"children\":\"$L35\"}]\n9:null\ne:[[\"$\",\"title\",\"0\",{\"children\":\"Thankful for incidents: embracing chaos to find clarity\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"How lessons from a recent incident led to improved platform resilience and more comprehensive testing practices.\"}],[\"$\",\"meta\",\"2\",{\"name\":\"keywords\",\"content\":\"Tines, security workflow automation, No-Code, SOAR\"}],[\"$\",\"link\",\"3\",{\"rel\":\"canonical\",\"href\":\"https://www.tines.com/blog/engineering-incidents-improvement/\"}],[\"$\",\"meta\",\"4\",{\"property\":\"og:title\",\"content\":\"Thankful for incidents: embracing chaos to find clarity\"}],[\"$\",\"meta\",\"5\",{\"property\":\"og:description\",\"content\":\"How lessons from a recent incident led to improved platform resilience and more comprehensive testing practices.\"}],[\"$\",\"meta\",\"6\",{\"property\":\"og:image\",\"content\":\"https://www.datocms-assets.com/55802/1721460016-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity-1.png\"}],[\"$\",\"meta\",\"7\",{\"property\":\"og:type\",\"content\":\"article\"}],[\"$\",\"meta\",\"8\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"9\",{\"name\":\"twitter:creator\",\"content\":\"@tines_hq\"}],[\"$\",\"meta\",\"10\",{\"name\":\"twitter:title\",\"content\":\"Tines | Intelligent workflow platform\"}],[\"$\",\"meta\",\"11\",{\"name\":\"twitter:description\",\"content\":\"Build and orchestrate intelligent workflows and agents, on the platform security and IT teams love.\"}],[\"$\",\"meta\",\"12\",{\"name\":\"twitter:image\",\"content\":\"https://www.tines.com/v5-assets/images/og/tines-home.png\"}],[\"$\",\"link\",\"13\",{\"rel\":\"icon\",\"href\":\"/v5-assets/favicon.png\"}],[\"$\",\"$L36\",\"14\",{}]]\n37:I[383156,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\",\"/_next/static/immutable/chunks/3pjhofejz57jt.js\",\"/_next/static/immutable/chunks/1q-89ajkqbbkw.js\",\"/_next/static/immutable/chunks/2uho2m59t6w8f.js\",\"/_next/static/immutable/chunks/2ti9pdu9p4n5l.js\",\"/_next/static/immutable/chunks/31qqpjo4d7384.js\",\"/_next/static/immutable/chunks/3zb6qebqmln4d.js\"],\"HashScroller\"]\n38:I[62200,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\",\"/_next/static/immutable/chunks/3pjhofejz57jt.js\",\"/_next/static/immutable/chunks/1q-89ajkqbbkw.js\",\"/_next/static/immutable/chunks/2uho2m59t6w8f.js\",\"/_next/static/immutable/chunks/2ti9pdu9p4n5l.js\",\"/_next/static/immutable/chunks/31qqpjo4d7384.js\",\"/_next/static/immutable/chunks/3zb6qebqmln4d.js\"],\"SetAreaNavActive\"]\n35:[[\"$\",\"$L37\",null,{}],[\"$\",\"$L38\",null,{\"href\":\"/blog/news/\"}],[\"$\",\"script\",null,{\"type\":\"application/ld+json\",\"dangerouslySetInnerHTML\":{\"__html\":\"{\\\"@context\\\":\\\"https://schema.org\\\",\\\"@type\\\":\\\"BlogPosting\\\",\\\"headline\\\":\\\"Thankful for incidents: embracing chaos to find clarity\\\",\\\"description\\\":\\\"How lessons from a recent incident led to improved platform resilience and more comprehensive testing practices.\\\",\\\"datePublished\\\":\\\"2024-07-19T23:00:00+00:00\\\",\\\"dateModified\\\":\\\"2024-08-13T13:45:13+00:00\\\",\\\"url\\\":\\\"https://www.tines.com/blog/engineering-incidents-improvement/\\\",\\\"image\\\":\\\"https://www.datocms-assets.com/55802/1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png?auto=format%2Ccompress\u0026fit=max\u0026q=90\u0026w=2400\\\",\\\"author\\\":[{\\\"@type\\\":\\\"Person\\\",\\\"name\\\":\\\"Shayon Mukherjee\\\"}],\\\"publisher\\\":{\\\"@type\\\":\\\"Organization\\\",\\\"name\\\":\\\"Tines\\\",\\\"url\\\":\\\"https://www.tines.com\\\"}}\"}}],[\"$\",\"div\",null,{\"className\":\"$undefined\",\"style\":{\"display\":\"block\",\"width\":\"var(--xl)\",\"height\":\"var(--xl)\",\"flex\":\"0 0 var(--xl)\",\"userSelect\":\"none\"}}],[\"$\",\"header\",null,{\"children\":[null,[\"$\",\"h1\",null,{\"className\":\"ArticlePageHeader-module-scss-module__szLeSa__title\",\"children\":\"Thankful for incidents: embracing chaos to find clarity\"}],[\"$\",\"div\",null,{\"className\":\"ArticlePageHeader-module-scss-module__szLeSa__authorInfo\",\"children\":[[\"$\",\"div\",null,{\"className\":\"ArticlePageHeader-module-scss-module__szLeSa__authors\",\"children\":[[\"$\",\"div\",null,{\"className\":\"ArticlePageHeader-module-scss-module__szLeSa__authorPhotos\",\"children\":[[\"$\",\"$L20\",\"129183137\",{\"src\":\"https://www.datocms-assets.com/55802/1661860358-image-228.png?auto=format%2Ccompress\u0026fit=max\u0026q=80\u0026w=128\",\"alt\":\"\",\"className\":\"ArticlePageHeader-module-scss-module__szLeSa__authorPhoto\",\"width\":42,\"height\":42,\"loading\":\"eager\"}]]}],[\"$\",\"div\",null,{\"children\":[[\"$\",\"span\",null,{\"className\":\"ArticlePageHeader-module-scss-module__szLeSa__authorName\",\"children\":[[\"$\",\"span\",null,{\"className\":\"ArticlePageHeader-module-scss-module__szLeSa__writtenBy\",\"children\":\"Written by \"}],[[\"$\",\"span\",\"129183137\",{\"children\":[\"Shayon Mukherjee\",\"\"]}]]]}],[\"$\",\"span\",null,{\"className\":\"ArticlePageHeader-module-scss-module__szLeSa__authorWorkTitle\",\"children\":\"Staff Software Engineer, Tines\"}]]}]]}],[\"$\",\"div\",null,{\"className\":\"ArticlePageHeader-module-scss-module__szLeSa__meta\",\"children\":[[\"$\",\"time\",null,{\"dateTime\":\"2024-07-19T23:00:00.000Z\",\"children\":[\"Published on \",\"July 19, 2024\"]}],[\"$\",\"div\",null,{\"className\":\"ArticlePageHeader-module-scss-module__szLeSa__categoryPills\",\"children\":[[\"$\",\"$L28\",\"news\",{\"href\":\"/blog/news/\",\"className\":\"ArticlePageHeader-module-scss-module__szLeSa__categoryPill\",\"children\":\"News\"}],[\"$\",\"$L28\",\"engineering\",{\"href\":\"/blog/engineering/\",\"className\":\"ArticlePageHeader-module-scss-module__szLeSa__categoryPill\",\"children\":\"Engineering\"}]]}]]}]]}]]}],null,[\"$\",\"div\",null,{\"className\":\"$undefined\",\"style\":{\"display\":\"block\",\"width\":\"var(--lg)\",\"height\":\"var(--lg)\",\"flex\":\"0 0 var(--lg)\",\"userSelect\":\"none\"}}],[\"$\",\"div\",null,{\"className\":\"page-module-scss-module__Z5AZta__articleLayout page-module-scss-module__Z5AZta__noToc\",\"children\":[[\"$\",\"aside\",null,{\"className\":\"page-module-scss-module__Z5AZta__articleAside\",\"children\":[\"$\",\"a\",null,{\"href\":\"https://login.tines.com/saml_idp/signup\",\"className\":\"DefaultSidebarContent-module-scss-module__nlQFtG__card\",\"children\":[[\"$\",\"div\",null,{\"className\":\"DefaultSidebarContent-module-scss-module__nlQFtG__text\",\"children\":[[\"$\",\"h3\",null,{\"className\":\"DefaultSidebarContent-module-scss-module__nlQFtG__heading\",\"children\":\"Sign up for Tines 3B Explore Edition\"}],[\"$\",\"p\",null,{\"className\":\"DefaultSidebarContent-module-scss-module__nlQFtG__linkArrow\",\"children\":\"Get started for free\"}]]}],[\"$\",\"$L20\",null,{\"className\":\"DefaultSidebarContent-module-scss-module__nlQFtG__icon\",\"src\":\"/v5-assets/images/usp/blossoming-transformation.svg\",\"alt\":\"\",\"aria-hidden\":\"true\",\"height\":90,\"width\":90}]]}]}],[\"$\",\"div\",null,{\"className\":\"page-module-scss-module__Z5AZta__articleMain\",\"children\":[\"$L39\",\"$L3a\",\"$L3b\",null,null]}]]}],null,\"$L3c\"]\n3d:I[729731,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\",\"/_next/static/immutable/chunks/3pjhofejz57jt.js\",\"/_next/static/immutable/chunks/1q-89ajkqbbkw.js\",\"/_next/static/immutable/chunks/2uho2m59t6w8f.js\",\"/_next/static/immutable/chunks/2ti9pdu9p4n5l.js\",\"/_next/static/immutable/chunks/31qqpjo4d7384.js\",\"/_next/static/immutable/chunks/3zb6qebqmln4d.js\"],\"SmartLink\"]\n3e:I[383156,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\",\"/_next/static/immutable/chunks/3pjhofejz57jt.js\",\"/_next/static/immutable/chunks/1q-89ajkqbbkw.js\",\"/_next/static/immutable/chunks/2uho2m59t6w8f.js\",\"/_next/static/immutable/chunks/2ti9pdu9p4n5l.js\",\"/_next/static/immutable/chunks/31qqpjo4d7384.js\",\"/_next/static/immutable/chunks/3zb6qebqmln4d.js\"],\"LinkedHeading\"]\n39:[\"$\",\"div\",null,{\"className\":\"page-module-scss-module__Z5AZta__coverImage\",\"children\":[\"$\",\"$L20\",null,{\"src\":\"https://www.datocms-assets.com/55802/1721460077-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity.png?auto=format%2Ccompress\u0026fit=max\u0026q=90\u0026w=2400\",\"width\":2400,\"height\":1260,\"alt\":\"Cover image for Thankful for incidents: embracing chaos to find clarity\",\"sizes\":\"(max-width: 767px) 100vw, (max-width: 1439px) 70vw, 1040px\",\"quality\":90,\"loading\":\"eager\"}]}]\n3a:[\"$\",\"div\",null,{\"className\":\"page-module-scss-module__Z5AZta__oldArticleWarning\",\"children\":[\"$\",\"div\",null,{\"className\":\"InfoBox-module-scss-module__K7dC3a__infoBox\",\"children\":[[\"$\",\"svg\",null,{\"width\":\"22\",\"height\":\"22\",\"viewBox\":\"0 0 22 22\",\"fill\":\"currentColor\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"path\",null,{\"fillRule\":\"evenodd\",\"clipRule\":\"evenodd\",\"d\":\"M11 0C4.925 0 0 4.925 0 11C0 17.075 4.925 22 11 22C17.075 22 22 17.075 22 11C22 4.925 17.075 0 11 0ZM10.5 5C10.2348 5 9.98043 5.10536 9.79289 5.29289C9.60536 5.48043 9.5 5.73478 9.5 6C9.5 6.26522 9.60536 6.51957 9.79289 6.70711C9.98043 6.89464 10.2348 7 10.5 7H11C11.2652 7 11.5196 6.89464 11.7071 6.70711C11.8946 6.51957 12 6.26522 12 6C12 5.73478 11.8946 5.48043 11.7071 5.29289C11.5196 5.10536 11.2652 5 11 5H10.5ZM9 9C8.73478 9 8.48043 9.10536 8.29289 9.29289C8.10536 9.48043 8 9.73478 8 10C8 10.2652 8.10536 10.5196 8.29289 10.7071C8.48043 10.8946 8.73478 11 9 11H10V14H9C8.73478 14 8.48043 14.1054 8.29289 14.2929C8.10536 14.4804 8 14.7348 8 15C8 15.2652 8.10536 15.5196 8.29289 15.7071C8.48043 15.8946 8.73478 16 9 16H13C13.2652 16 13.5196 15.8946 13.7071 15.7071C13.8946 15.5196 14 15.2652 14 15C14 14.7348 13.8946 14.4804 13.7071 14.2929C13.5196 14.1054 13.2652 14 13 14H12V10C12 9.73478 11.8946 9.48043 11.7071 9.29289C11.5196 9.10536 11.2652 9 11 9H9Z\"}]}],[\"$\",\"div\",null,{\"className\":\"InfoBox-module-scss-module__K7dC3a__text\",\"children\":\"This article was posted more than 18 months ago.\"}]]}]}]\n3b:[\"$\",\"article\",null,{\"className\":\"Article-module-scss-module__ZE_SmG__article page-module-scss-module__Z5AZta__articleContent\",\"children\":[[[\"$\",\"p\",\"0\",{\"children\":[[\"$\",\"em\",\"0\",{\"children\":[\"$\",\"strong\",null,{\"children\":\"In this blog post, Tines software engineer Shayon Mukherjee shares how lessons from a recent incident led to improved platform resilience and more comprehensive testing practices.\"}]}]]}],[\"$\",\"p\",\"1\",{\"children\":[\"It was a typical late June afternoon when we embarked on what seemed a routine Redis cluster upgrade across approximately 40 customer stacks. The upgrade was essential, influenced by a previous outage that highlighted the risks of not using more robust instances and better networking support on those instance types. \"]}],[\"$\",\"p\",\"2\",{\"children\":[\"This wasn't the first time we performed such an upgrade, nor is it going to be the last time. But little did we know that this upgrade would soon reveal an unseen issue that lay dormant, undetected, and ready to teach us a valuable lesson.\"]}],[\"$\",\"p\",\"3\",{\"children\":[\"A few moments into the upgrade process, our monitoring systems flagged the Toolkit API Down alarm.\",[\"$\",\"$L3d\",\"1\",{\"to\":\"https://www.tines.com/blog/introducing-the-tines-toolkit\",\"children\":[\" \",[\"$\",\"u\",\"1\",{\"children\":\"Tines Toolkit\"}]]}],\" is a Tines\",[\"$\",\"$L3d\",\"3\",{\"to\":\"https://www.tines.com/docs/actions/types/webhook#response-enabled-webhooks\",\"children\":[\" \",[\"$\",\"u\",\"1\",{\"children\":\"response-enabled webhooks\"}]]}],\"-powered service. This alert was the first signal of something amiss. The webhooks were timing out,\",[\"$\",\"$L3d\",\"5\",{\"to\":\"https://status.tines.com/incidents/jgXSym\",\"children\":[\" \",[\"$\",\"u\",\"1\",{\"children\":\"affecting crucial customer workflows\"}]]}],\". The immediate response was swift - our engineers triggered a force deployment to flush out any bad state from our containers, bringing things back online within a few minutes. But this was just the beginning. Now we needed to understand what had actually happened.\"]}],[\"$\",\"$L3e\",\"4\",{\"level\":3,\"id\":\"how-does-it-all-work\",\"children\":[[\"$\",\"strong\",\"0\",{\"children\":\"How does it all work?\"}]]}],[\"$\",\"p\",\"5\",{\"children\":[\"In our system,\",[\"$\",\"$L3d\",\"1\",{\"to\":\"https://www.tines.com/docs/actions/types/webhook#response-enabled-webhooks\",\"children\":[\" \",[\"$\",\"u\",\"1\",{\"children\":\"response-enabled webhooks\"}]]}],\" play a critical role, especially in how they interact with Redis Pub/Sub to manage real-time data flows. Before we dive further into the bug, let's take a quick minute to understand the system first.\"]}],[\"$\",\"p\",\"6\",{\"children\":[\"Our application is built using\",[\"$\",\"$L3d\",\"1\",{\"to\":\"https://rubyonrails.org/\",\"children\":[\" \",[\"$\",\"u\",\"1\",{\"children\":\"Ruby on Rails\"}]]}],\", with\",[\"$\",\"$L3d\",\"3\",{\"to\":\"https://github.com/puma/puma\",\"children\":[\" \",[\"$\",\"u\",\"1\",{\"children\":\"Puma\"}]]}],\" as our web server and\",[\"$\",\"$L3d\",\"5\",{\"to\":\"https://github.com/rack/rack\",\"children\":[\" \",[\"$\",\"u\",\"1\",{\"children\":\"Rack\"}]]}],\" as the web server interface. For\",[\"$\",\"$L3d\",\"7\",{\"to\":\"https://www.tines.com/docs/actions/types/webhook#response-enabled-webhooks\",\"children\":[\" \",[\"$\",\"u\",\"1\",{\"children\":\"response-enabled webhooks\"}]]}],\", we use a technique known as Socket Hijacking to make async webhooks function like synchronous ones. \"]}],[\"$\",\"p\",\"7\",{\"children\":[\"When a webhook request is received, Rack performs what's known as “Full Socket Hijacking.” This process closes the Rails response object, which frees up the request handling thread (inside Puma) to return to the pool of available workers while maintaining the socket connection to the client. This allows the Puma server to accept new web requests without blocking the response for the synchronous webhook, all the while keeping the socket connection open with the client to return a response once it is available.\"]}],\"$L3f\",\"$L40\",\"$L41\",\"$L42\",\"$L43\",\"$L44\",\"$L45\",\"$L46\",\"$L47\",\"$L48\",\"$L49\",\"$L4a\",\"$L4b\",\"$L4c\",\"$L4d\",\"$L4e\",\"$L4f\",\"$L50\",\"$L51\",\"$L52\",\"$L53\"],\"$L54\"]}]\n3c:[\"$\",\"div\",null,{\"className\":\"$undefined\",\"style\":{\"display\":\"block\",\"width\":\"var(--sectionPadding)\",\"height\":\"var(--sectionPadding)\",\"flex\":\"0 0 var(--sectionPadding)\",\"userSelect\":\"none\"}}]\n55:I[285528,[\"/_next/static/immutable/chunks/3sz971vfk095f.js\",\"/_next/static/immutable/chunks/0czgivygkxa1z.js\",\"/_next/static/immutable/chunks/2kpu_ktjkwf7q.js\",\"/_next/static/immutable/chunks/07hv-f84dhd61.js\",\"/_next/static/immutable/chunks/0639nwpv6l0pm.js\",\"/_next/static/immutable/chunks/2826pm12av2bw.js\",\"/_next/static/immutable/chunks/2_swgt_1o6v9a.js\",\"/_next/static/immutable/chunks/3pjhofejz57jt.js\",\"/_next/static/immutable/chunks/1q-89ajkqbbkw.js\",\"/_next/static/immutable/chunks/2uho2m59t6w8f.js\",\"/_next/static/immutable/chunks/2ti9pdu9p4n5l.js\",\"/_next/static/immutable/chunks/31qqpjo4d7384.js\",\"/_next/static/immutable/chunks/3zb6qebqmln4d.js\"],\"CodeBlockCopy\"]\n3f:[\"$\",\"p\",\"8\",{\"children\":[\"Another important part of this system is a simple Redis Pub/Sub implementation running from a dedicated listener thread inside the web server. When a webhook is received, we enqueue the relevant Tines Action Run and open a Redis subscription (from the thread) to listen for the event received when the Tines story exit action generates the response. We then proxy it back to the client.\"]}]\n40:[\"$\",\"figure\",\"9\",{\"className\":\"ImageBlock-module-scss-module__0Ef4dq__imageBlock\",\"children\":[[\"$\",\"img\",null,{\"src\":\"https://www.datocms-assets.com/55802/1722000769-2024-07-thankful-for-incidents-embracing-chaos-to-find-clarity-infographic-1.jpg?auto=format%2Ccompress\u0026fit=max\u0026q=90\u0026w=2000\u0026auto=format,compress\u0026q=90\u0026w=2000\u0026fit=max\",\"alt\":\"\",\"width\":2400,\"height\":1260,\"loading\":\"lazy\"}],null]}]\n41:[\"$\",\"$L3e\",\"10\",{\"level\":2,\"id\":\"the-hidden-bug\",\"children\":[[\"$\",\"strong\",\"0\",{\"children\":\"The hidden bug\"}]]}]\n42:[\"$\",\"p\",\"11\",{\"children\":[\"Normally, this dedicated listener thread maintains a persistent connection with Redis to receive and process messages. However, during the upgrade, it encountered connectivity issues, and without any system alerts or logs, the Redis persistent connection died. This failure is one of those classic cases of \\\"the server is running but nothing is working,” just a quiet stoppage that gradually led to service degradation as new webhook requests could no longer be processed.\"]}]\n43:[\"$\",\"p\",\"12\",{\"children\":[\"It took us a while to reach this conclusion. It required an examination of logs, reading the code, formulating hypotheses, and running some controlled failure cases. We were eventually able to recreate the scenario both locally and in production, and felt confident that this failure mode was the primary contributing factor to the incident.\"]}]\n44:[\"$\",\"p\",\"13\",{\"children\":[\"This incident highlighted something in our system’s architecture: a lack of a good error handling or recovery mechanism for the webhook's listener thread during Redis connectivity or similar issues. \"]}]\n45:[\"$\",\"blockquote\",\"14\",{\"className\":\"PullQuote-module-scss-module__xEM2FG__pullQuote\",\"data-design\":\"Featured quote\",\"children\":[\"$\",\"div\",null,{\"className\":\"PullQuote-module-scss-module__xEM2FG__pullQuoteInner\",\"children\":[[\"$\",\"div\",null,{\"className\":\"PullQuote-module-scss-module__xEM2FG__pullQuoteContent\",\"children\":[[\"$\",\"span\",null,{\"className\":\"PullQuote-module-scss-module__xEM2FG__pullQuoteMark\",\"children\":[\"$\",\"svg\",null,{\"width\":\"36\",\"height\":\"36\",\"viewBox\":\"0 0 36 36\",\"fill\":\"none\",\"aria-hidden\":\"true\",\"children\":[[\"$\",\"rect\",null,{\"width\":\"36\",\"height\":\"36\",\"rx\":\"18\",\"fill\":\"currentColor\",\"fillOpacity\":0.18}],[\"$\",\"path\",null,{\"d\":\"M16.864 17.28C16.192 17.28 15.76 17.784 15.76 18.576V21.384H13V18.336C13 16.368 14.272 15 16.168 15H17.296V17.28H16.864ZM18.184 21.384V18.336C18.184 16.368 19.456 15 21.352 15H22.48V17.28H22.048C21.376 17.28 20.944 17.784 20.944 18.576V21.384H18.184Z\",\"fill\":\"currentColor\"}]]}]}],[[\"$\",\"p\",\"p-0\",{\"children\":\"This oversight meant that a single point of failure could lead to disruptions in our production environment, a lesson that was both surprising and invaluable for our ongoing efforts to build a more resilient platform.\"}]]]}],null]}]}]\n46:[\"$\",\"$L3e\",\"15\",{\"level\":3,\"id\":\"lessons-from-the-incident\",\"children\":[[\"$\",\"strong\",\"0\",{\"children\":\"Lessons from the incident\"}]]}]\n47:[\"$\",\"p\",\"16\",{\"children\":[\"This incident, albeit stressful, was a blessing in disguise. It exposed a critical vulnerability in how our webhook system was designed. The reliance on a single listener thread without a robust fail open mechanism was something that we had overlooked. But thanks to this incident, it was brought to light under circumstances that allowed us to respond without major consequences.\"]}]\n48:[\"$\",\"p\",\"17\",{\"children\":[\"We learned that our preparations for zero-downtime capabilities during Redis updates were incomplete. We had not considered\",[\"$\",\"$L3d\",\"1\",{\"to\":\"https://www.tines.com/docs/actions/types/webhook#response-enabled-webhooks\",\"children\":[\" \",[\"$\",\"u\",\"1\",{\"children\":\"response-enabled webhooks\"}]]}],\" in our chaos testing scenarios. This incident was a good reminder of the importance of comprehensive testing and the need for a holistic view of system dependencies and resilience, particularly for non-traditional parts of our system that might not be well-understood.\"]}]\n49:[\"$\",\"$L3e\",\"18\",{\"level\":3,\"id\":\"embracing-the-incident\",\"children\":[[\"$\",\"strong\",\"0\",{\"children\":\"Embracing the incident\"}]]}]\n4a:[\"$\",\"p\",\"19\",{\"children\":[\"Incidents happen to everyone, but how you deal with them and learn from them is what counts. What stands out about incidents like these is not just the immediate impact or the technical breakdowns but the invaluable insight they provide into our systems. \"]}]\n4b:[\"$\",\"blockquote\",\"20\",{\"className\":\"PullQuote-module-scss-module__xEM2FG__pullQuote\",\"data-design\":\"Featured quote\",\"children\":[\"$\",\"div\",null,{\"className\":\"PullQuote-module-scss-module__xEM2FG__pullQuoteInner\",\"children\":[[\"$\",\"div\",null,{\"className\":\"PullQuote-module-scss-module__xEM2FG__pullQuoteContent\",\"children\":[[\"$\",\"span\",null,{\"className\":\"PullQuote-module-scss-module__xEM2FG__pullQuoteMark\",\"children\":[\"$\",\"svg\",null,{\"width\":\"36\",\"height\":\"36\",\"viewBox\":\"0 0 36 36\",\"fill\":\"none\",\"aria-hidden\":\"true\",\"children\":[[\"$\",\"rect\",null,{\"width\":\"36\",\"height\":\"36\",\"rx\":\"18\",\"fill\":\"currentColor\",\"fillOpacity\":0.18}],[\"$\",\"path\",null,{\"d\":\"M16.864 17.28C16.192 17.28 15.76 17.784 15.76 18.576V21.384H13V18.336C13 16.368 14.272 15 16.168 15H17.296V17.28H16.864ZM18.184 21.384V18.336C18.184 16.368 19.456 15 21.352 15H22.48V17.28H22.048C21.376 17.28 20.944 17.784 20.944 18.576V21.384H18.184Z\",\"fill\":\"currentColor\"}]]}]}],[[\"$\",\"p\",\"p-0\",{\"children\":\"Each incident, especially one that uncovers fundamental flaws, is an opportunity to improve, harden our systems, and ensure better service for our users. \"}]]]}],null]}]}]\n4c:[\"$\",\"p\",\"21\",{\"children\":[\"They push us to think creatively about solutions and to fortify areas we hadn't even considered before.\"]}]\n4d:[\"$\",\"$L3e\",\"22\",{\"level\":3,\"id\":\"action-items\",\"children\":[[\"$\",\"strong\",\"0\",{\"children\":\"Action items\"}]]}]\n4e:[\"$\",\"p\",\"23\",{\"children\":[\"While we’re happy that our automated monitoring caught the issue and we were able to mitigate the incident in a very short period, we still want to strive for more. We’ve already ensured the singleton thread acts in a reconciliation loop so that it can withstand similar issues in the future. \"]}]\n4f:[\"$\",\"p\",\"24\",{\"children\":[\"We’re committed to improving our resilience across the board. To achieve this, we’ll be conducting periodic and controlled chaos testing, which will include actions like taking down both our primary and ephemeral data stores to ensure that critical Tines workflows remain unaffected. This process should not only allow us to uncover more hidden unknowns like the ones we saw above but also catch regressions in our tooling and practices as we scale.\"]}]\n50:[\"$\",\"$L3e\",\"25\",{\"level\":3,\"id\":\"reflecting-on-the-incident\",\"children\":[[\"$\",\"strong\",\"0\",{\"children\":\"Reflecting on the incident\"}]]}]\n51:[\"$\",\"blockquote\",\"26\",{\"className\":\"PullQuote-module-scss-module__xEM2FG__pullQuote\",\"data-design\":\"Featured quote\",\"children\":[\"$\",\"div\",null,{\"className\":\"PullQuote-module-scss-module__xEM2FG__pullQuoteInner\",\"children\":[[\"$\",\"div\",null,{\"className\":\"PullQuote-module-scss-module__xEM2FG__pullQuoteContent\",\"children\":[[\"$\",\"span\",null,{\"className\":\"PullQuote-module-scss-module__xEM2FG__pullQuoteMark\",\"children\":[\"$\",\"svg\",null,{\"width\":\"36\",\"height\":\"36\",\"viewBox\":\"0 0 36 36\",\"fill\":\"none\",\"aria-hidden\":\"true\",\"children\":[[\"$\",\"rect\",null,{\"width\":\"36\",\"height\":\"36\",\"rx\":\"18\",\"fill\":\"currentColor\",\"fillOpacity\":0.18}],[\"$\",\"path\",null,{\"d\":\"M16.864 17.28C16.192 17.28 15.76 17.784 15.76 18.576V21.384H13V18.336C13 16.368 14.272 15 16.168 15H17.296V17.28H16.864ZM18.184 21.384V18.336C18.184 16.368 19.456 15 21.352 15H22.48V17.28H22.048C21.376 17.28 20.944 17.784 20.944 18.576V21.384H18.184Z\",\"fill\":\"currentColor\"}]]}]}],[[\"$\",\"p\",\"p-0\",{\"children\":\"As we continue to build and maintain complex software systems, incidents like these are not just inevitable, they’re necessary. \"}]]]}],null]}]}]\n52:[\"$\",\"p\",\"27\",{\"children\":[\"They teach us about our systems' resilience, the effectiveness of our responses, and the areas that need more attention. They force us to confront the realities of our designs and our decisions. \"]}]\n53:[\"$\",\"p\",\"28\",{\"children\":[\"So, in a way, we should be thankful for each incident. After all, each one gives us a chance to be better than we were the day before. Let's be thankful for these incidents - they are, indeed, our best friends in the ever-evolving landscape of complex systems.\"]}]\n54:[\"$\",\"$L55\",null,{}]\n"])</script></body></html> |