Files
nexus/sreweekly/articles/342/01-video-observability.html
2026-09-12 17:23:01 +08:00

39 lines
22 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html><html lang="en"><head><!-- Global Metadata --><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><link rel="icon" href="/favicon.svg"><!-- Canonical URL --><link rel="canonical" href="https://evertz.io/blog/2022-09-22-video-observability"><!-- Primary Meta Tags --><title>evertz.io | Blog | Video Observability</title><meta name="title" content="evertz.io | Blog | Video Observability"><meta name="description" content="Combining Observability with statistical content matching to monitor video streams."><!-- Open Graph / Facebook --><meta property="og:type" content="article"><meta property="og:url" content="https://evertz.io/blog/2022-09-22-video-observability"><meta property="og:title" content="evertz.io | Blog | Video Observability"><meta property="og:description" content="Combining Observability with statistical content matching to monitor video streams."><meta property="og:image" content="https://evertz.io/open-graph/blog/2022-09-22-video-observability.png"><meta property="og:site_name" content="evertz.io"><!-- Twitter --><meta property="twitter:card" content="summary_large_image"><meta property="twitter:url" content="https://evertz.io/blog/2022-09-22-video-observability"><meta property="twitter:title" content="evertz.io | Blog | Video Observability"><meta property="twitter:description" content="Combining Observability with statistical content matching to monitor video streams."><meta property="twitter:image" content="https://evertz.io/open-graph/blog/2022-09-22-video-observability.png"><meta property="twitter:creator" content="@evertzio"><meta property="twitter:site" content="@evertzio"><!-- Links --><link rel="sitemap" href="/sitemap-index.xml"><meta name="author" content="Evertz Microsystems"><link rel="alternate" type="application/rss+xml" href="https://evertz.io/feed.xml" title="evertz.io Blog"><!-- PWA --><link rel="apple-touch-icon" href="/assets/img/favicons/evertz-io-apple-touch-icon.png"><meta name="apple-mobile-web-app-title" content="evertz.io"><link rel="mask-icon" href="/assets/img/favicons/evertz-io-icon-silhouette-pinned-tab.svg" color="#FFFFFF"><meta name="theme-color" content="#000000"><!-- Termly --><script type="text/javascript" src="https://app.termly.io/resource-blocker/60ca59ca-58c9-4005-bb53-2b4869e3a142?autoBlock=on"></script><!-- Fonts --><link rel="preconnect" href="https://fonts.googleapis.com"><link rel="preconnect" href="https://fonts.gstatic.com" crossorigin><link href="https://fonts.googleapis.com/css2?family=Montserrat:ital,wght@0,400;0,500;0,700;1,400;1,500;1,700&display=swap" rel="stylesheet"><!-- Google Analytics --><script async src="https://www.googletagmanager.com/gtag/js?id=G-63372Z3LDM"></script><script>
window.dataLayer = window.dataLayer || [];
function gtag() {
dataLayer.push(arguments);
}
gtag('js', new Date());
gtag('config', 'G-63372Z3LDM');
</script><script async src="https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js?client=ca-pub-1811571878037573" crossorigin="anonymous"></script><link rel="stylesheet" href="/_astro/Layout.FzceOJwj.css"></head><body><div class="container-fluid"><header class="fixed-top" data-astro-cid-yxete6jp><!-- NAVIGATION --><nav class="navbar navbar-dark navbar-expand-lg static-top" data-astro-cid-yxete6jp><div class="container-xl" data-astro-cid-yxete6jp><a class="navbar-brand" href="/" title="Home" aria-label="Go home" data-astro-cid-yxete6jp></a><button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarResponsive" aria-controls="navbarResponsive" aria-expanded="false" aria-label="Toggle navigation" data-astro-cid-yxete6jp><span class="navbar-toggler-icon" data-astro-cid-yxete6jp></span></button><div class="collapse navbar-collapse" id="navbarResponsive" data-astro-cid-yxete6jp><ul class="navbar-nav ml-auto menu-left" data-astro-cid-yxete6jp><li class="nav-item dropdown" data-astro-cid-yxete6jp><a class="nav-link dropdown-toggle" href="#" id="servicesDropdownMenuLink" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" data-astro-cid-yxete6jp>Services</a><div class="dropdown-menu m-top" aria-labelledby="servicesDropdownMenuLink" data-astro-cid-yxete6jp><a href="/stream" class="dropdown-item" data-astro-cid-yxete6jp="true" data-astro-cid-3o5q7yvo>Stream</a><a href="/transform" class="dropdown-item" data-astro-cid-yxete6jp="true" data-astro-cid-3o5q7yvo>Transform</a><a href="/xchange" class="dropdown-item" data-astro-cid-yxete6jp="true" data-astro-cid-3o5q7yvo>XChange</a></div></li><li class="nav-item active" data-astro-cid-yxete6jp><a class="nav-link" href="/blog" role="button" data-astro-cid-yxete6jp>Blog</a></li><li class="nav-item dropdown" data-astro-cid-yxete6jp><a class="nav-link dropdown-toggle" href="#" id="AboutDropdownMenuLink" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" data-astro-cid-yxete6jp>About</a><div class="dropdown-menu m-top" aria-labelledby="AboutDropdownMenuLink" data-astro-cid-yxete6jp><a href="/about" class="dropdown-item" data-astro-cid-yxete6jp="true" data-astro-cid-3o5q7yvo>About Us</a><a href="/careers" class="dropdown-item" data-astro-cid-yxete6jp="true" data-astro-cid-3o5q7yvo>Careers</a></div></li><li data-astro-cid-yxete6jp><span class="btn-log" data-astro-cid-yxete6jp><div class="nav-item right-side" data-astro-cid-yxete6jp><a class="btn btn-primary btn-lightblue" href="https://app.evertz.io" role="button" data-astro-cid-yxete6jp>Log In</a></div><div class="nav-item right-side" data-astro-cid-yxete6jp><a class="btn btn-primary btn-lightblue" href="mailto:%73%61%6C%65%73%40%65%76%65%72%74%7A%2E%63%6F%6D" role="button" data-astro-cid-yxete6jp>Request Access</a></div></span></li></ul></div></div></nav></header><script type="module">$(function(){$(`a.nav-link.dropdown-toggle`).on(`click`,function(){$(this).toggleClass(`turner`),$(`.turner`).not(this).removeClass(`turner`)}),$(document).on(`click`,function(e){let t=$(`a.nav-link.dropdown-toggle`),n=e.target;!t.is(n)&&t.has(n).length===0&&t.removeClass(`turner`)})});</script><main><div class="row sections m-s test-sec post-cont"><div class="container-xl"><div class="row justify-content-md-center"><div class="col-md-8 col-sm-12"><div><a href="/blog">&lt; Back to Blog</a></div><header class="mb-4"><!-- Post title--><h2 class="fw-bolder mb-1">Video Observability</h2><!-- Post meta content--><div class="text-muted fst-italic mb-2">Posted on<time datetime="2022-09-22T00:00:00.000Z">Sep 22, 2022</time>by Jeremy Blythe</div><div class="moma"><h3><i class="fa-solid fa-quote-left"></i>As a television broadcaster, how do I ensure that my channels are playing out the right thing for my viewers?<i class="fa-solid fa-quote-right"></i></h3><img src="/assets/img/blog/b1.webp" alt="Abstract colorful lines" class="blog_bck eio-no-select" aria-hidden="true"></div></header><section class="mb-5"><p>As a television broadcaster, how do I ensure that my channels are playing out the right thing for my viewers? Well, usually by watching and listening to it!</p>
<p>In the old days you would have a handful of channels, each broadcasting video and audio in a single language. You would hire an operator to sit in front of a few monitors and watch them all simultaneously.</p>
<blockquote>
<h3 id="whats-in-a-stream">What’s in a stream?</h3>
<p>A television stream is made of multiple spatiotemporal elements. Temporally: frames of video are rendered one-by-one in front of your eyes typically between 24 and 60 frames per second. The audio samples run even faster and create the continuous sound you hear. Spatially: within each frame there are visible elements such as overlaid channel branding logos, closed-captions, pop-up graphics to entice you to watch more on the channel and many more. The complexity doesn’t end there. These days there are multiple audio and captions languages streamed at the same time so as a viewer you can choose to watch a show with Spanish audio and English captions for example. Finally, there’s a world of data encoded into non-visible parts of the signal that inform downstream components to perform some action, like deliver local advertisements when they decode a trigger.</p>
<p><img src="/assets/img/blog/posts/spatiotemporal.png" alt="A diagram showing the spatiotemporal characteristic of video"/></p>
<p>You can imagine the task of orchestrating all this complexity to come together into a seamless experience for the viewer! Playout automation services have been taking care of this problem for many years. Given a schedule describing the desired output, the automation controls a myriad of software services and hardware devices in real-time to produce the output stream.</p>
</blockquote>
<p>As stations have grown to hundreds of channels, and each channel might have station logos, closed-captions, pop-up graphics and half a dozen audio languages, monitoring has become a much more daunting task. The screens have become a huge video wall, with hundreds of streams visible simultaneously. Operators can select a stream to listen to one-by-one, but without being an amazing polyglot they can at best make an educated guess that what they are listening to sounds Portuguese-ish. But is it Portuguese or Brazilian-Portuguese? You can see how this doesn’t scale.</p>
<p><img src="/assets/img/blog/posts/facility.webp" alt="A broadcast facility with hundreds of monitors"/></p>
<p>To help the operators, automated monitoring tools have been developed. These check that the pictures have not degraded in some way or frozen or simply showing black. They’ll check that the audio is not silent and that the captions are present. But, not a great deal more than that.</p>
<p>These monitoring tools, like most monitoring, are only checking for known symptoms. For example, there are many error conditions that could lead to the stream showing black, so detectors were specifically invented to check for black. The trouble is, black sections are often intentional, particularly when transitioning into breaks or before end credits, so you may get false alarms.</p>
<h2 id="what-do-we-want">What do we want?</h2>
<p>To draw an analogy with software unit testing, this legacy monitoring would be like all your tests passing simply because the code runs without crashing! What we really want is a way to check that the stream output matches our intent. A good unit-test asserts that, given a known input, an expected output is observed.</p>
<p>We want to be able to say, “At 8:30pm we should be showing season six, episode ten of “Better Call Saul”, the English, Spanish, French and German audio and captions should all be available. The channel logo should be showing in the top-right of the screen.”</p>
<p>So, give a copy of the schedule to a machine that can watch the stream and report discrepancies. Easy? No.</p>
<p>Humans are really good at answering that question about what should be happening at 8:30pm. They can glance at a small screen, read some notes about the episode and reason that it’s correct. They can check the correct channel logo is in the right place. They can tell the difference between these languages even if they are not fluent. Machines have a hard time with this.</p>
<p>“AI and Machine learning!”, I hear you cry. Yes, these types of problems are becoming more solvable using these techniques but, it’s expensive. Expensive to train and expensive run. It would end up costing more to monitor the channel than to produce it!</p>
<h2 id="what-did-we-make">What did we make?</h2>
<p>For evertz.io we’ve built a cost-efficient way to test, through assertions, that the scheduler’s intent is produced in the output stream. We use statistical methods over perceptual representations of the source content and output stream to produce similarity scores. Thresholds over these scores allow us to pass or fail an assertion.</p>
<p><img src="/assets/img/blog/posts/diagram.png" alt="A diagram showing the flow of video to transmission and a separate flow of data for assertions"/></p>
<p>That question about what should be observed at 8:30pm is now answerable. And for 8:31 and 8:32 and 8:33… multiplied by as many channels as you like – the machines do not get overwhelmed. More so, it can do a better job than the human. An assertion will fail if you’re more than a few seconds out from where you should be in a show; an indicator of a problem that may lead to the cliffhanger getting cut off by an ad break! Like human operators, it’s not fluent in those languages, but it can statistically match what should be audible with what actually is. Therefore, it can check that Portuguese, Brazilian-Portuguese, French and Canadian-French are all present in the correct order through pattern matching. Not only that, but the machine will tell you that the correct seconds of the audio are playing at the correct time in the show, not simply, “Hmm, that sounds like French. Pass!”</p>
<p>Our patented technology can do all this and more using a few lambda functions and a single ARM CPU core per channel running our Rust assertion engine. Hard to beat that cost optimization!</p>
<h2 id="how-do-we-make-sense-of-it">How do we make sense of it?</h2>
<p>With assertions for every few seconds, when there is a failure there is a lot of data to comb through. And even once you’ve identified which spatiotemporal element is causing the failure, how do you go about identifying the cause? A modern playout chain is made up of a lot of moving parts, and a failure in any one of them could cause similar looking problems in the output.</p>
<p>By using <a href="https://www.honeycomb.io/">Honeycomb</a> to collect distributed tracing data across all elements of the playout chain, we can easily filter and zoom in on assertion failures. We can then use assertion trace attributes in queries across the services in the playout chain and pinpoint where the issue originated.</p>
<div style="display: flex;"><div style="flex: 50%; padding: 5px;"><img src="/assets/img/blog/posts/query.png" alt="A Honeycomb query visualization"/></div><div style="flex: 50%; padding: 5px;"><img src="/assets/img/blog/posts/trace.png" alt="A Honeycomb trace waterfall view"/></div></div>
<p>To help understand the failures, we attach a link to the <a href="https://www.honeycomb.io/">Honeycomb</a> spans to our own tooling to visualize an individual assertion report:</p>
<p><img src="/assets/img/blog/posts/report.png" alt="A detailed assertion report with data visualizations"/></p>
<p>Not only can we get the detailed report but we can look back at the timeline of events that led up to it. By slicing and dicing the data we can look for trends, groups, previous outcomes given similar inputs, you name it! There’s a wealth of data to dig into.</p>
<hr/>
<p>With video observability we can ensure our production system is running smoothly. Our distributed tracing and high-cardinality data in <a href="https://www.honeycomb.io/">Honeycomb</a> allows us to connect assertion outcomes to scheduled intent. We can safely deploy and release new versions of code to production multiple times a day; first to a <a href="https://en.wikipedia.org/wiki/Eating_your_own_dog_food">dogfood</a> tenant and then a gradual roll-out to our customers. We can analyze not only the performance of the new code but the correctness of the output stream. Through integrations and webhooks we can even automatically trigger a rollback to the previous release to resolve issues much faster than any human-in-the-loop monitoring and incident management process.</p></section><section class="mb-5"><div><hr><p class="footer-section"><a href="/">Evertz.io</a> combines decades of knowledge and highly tuned video, audio and related data processing systems with cloud-native techniques, processes and tools used by <a href="https://thenewstack.io/charity-majors-recipe-for-high-performing-teams/">high-performance teams</a>.</p><p class="footer-section">If this sounds interesting to you, check out our <a href="/careers">careers page</a> for open positions.</p></div></section><section class="mb-5"><div class="card bg-light"><div class="card-body"><!-- Comment with nested comments--><div class="d-flex"><!-- Parent comment--><div class="flex-shrink-0"><img class="rounded-circle eio-no-select" src="/assets/img/blog/bio-img/jeremy-blythe.webp" aria-hidden="true" alt="Author profile photo"></div><div class="ms-3"><div class="fw-bold">Jeremy Blythe</div><small>VP Engineering, Media Distribution</small><div class="blockquote-footer">Jeremy Blythe is VP Engineering for evertz.io. He provides leadership for multiple globally distributed engineering teams and makes Emmy award-winning products for the Media and Entertainment industry.</div></div></div></div></div></section></div></div></div></div><script type="module">(function(){for(var e=document.getElementsByTagName(`a`),t=0;t<e.length;t++)/^(https?:)?\/\//.test(e[t].getAttribute(`href`)??``)&&(e[t].target=`_blank`)})();</script></main><footer class="row footer" data-astro-cid-jo6i4kqk><div class="container-xl" data-astro-cid-jo6i4kqk><div class="row" data-astro-cid-jo6i4kqk><div class="col-md-3 col-sm-12" data-astro-cid-jo6i4kqk><ul data-astro-cid-jo6i4kqk><li class="foot_head" data-astro-cid-jo6i4kqk>Services</li><li data-astro-cid-jo6i4kqk><a href="/services" data-astro-cid-jo6i4kqk>All services</a></li><li data-astro-cid-jo6i4kqk><a href="/stream" data-astro-cid-jo6i4kqk>Stream</a></li><li data-astro-cid-jo6i4kqk><a href="/transform" data-astro-cid-jo6i4kqk>Transform</a></li><li data-astro-cid-jo6i4kqk><a href="/xchange" data-astro-cid-jo6i4kqk>XChange</a></li></ul></div><div class="col-md-3 col-sm-12" data-astro-cid-jo6i4kqk><ul data-astro-cid-jo6i4kqk><li class="foot_head" data-astro-cid-jo6i4kqk>Company</li><li data-astro-cid-jo6i4kqk><a href="/careers" data-astro-cid-jo6i4kqk>Careers</a></li><li data-astro-cid-jo6i4kqk><a href="/blog" data-astro-cid-jo6i4kqk>Blog</a></li><li data-astro-cid-jo6i4kqk><a href="/about" data-astro-cid-jo6i4kqk>About</a></li></ul></div><div class="col-md-3 col-sm-12" data-astro-cid-jo6i4kqk><ul data-astro-cid-jo6i4kqk><li class="foot_head" data-astro-cid-jo6i4kqk>Support</li><li data-astro-cid-jo6i4kqk><a href="https://support.evertz.io" data-astro-cid-jo6i4kqk>Support</a></li><li data-astro-cid-jo6i4kqk><a href="mailto:%73%61%6C%65%73%40%65%76%65%72%74%7A%2E%63%6F%6D" data-astro-cid-jo6i4kqk>Contact Us</a></li><li data-astro-cid-jo6i4kqk><a href="mailto:cybersecurity@evertz.com" data-astro-cid-jo6i4kqk>Cyber Security Inquiries</a></li></ul></div><div class="col-md-3 col-sm-12 d-flex align-items-center justify-content-center justify-content-md-start" data-astro-cid-jo6i4kqk><a href="/blog/2026-02-17-evertz-io-achieves-soc2-type-ii-compliance" title="View our SOC 2 Type II announcement" class="soc2-badge" data-astro-cid-jo6i4kqk><img src="/assets/img/badges/aicpa-soc2-certified-logo.svg" alt="SOC 2 Type II" width="100" height="100" data-astro-cid-jo6i4kqk></a></div></div></div></footer><footer class="row footer_btm" data-astro-cid-jo6i4kqk><div class="container-xl" data-astro-cid-jo6i4kqk><div class="row" data-astro-cid-jo6i4kqk><div class="col-xl-3 col-lg-3 col-12" data-astro-cid-jo6i4kqk><div class="social-media" data-astro-cid-jo6i4kqk><ul data-astro-cid-jo6i4kqk><li data-astro-cid-jo6i4kqk><a href="https://youtube.com/@evertzio5548" aria-label="Youtube" target="_blank" rel="nofollow noreferrer" data-astro-cid-jo6i4kqk><i class="fa-brands fa-youtube" aria-hidden="true" title="Youtube" data-astro-cid-jo6i4kqk></i></a></li><li data-astro-cid-jo6i4kqk><a href="https://twitter.com/evertzio" aria-label="Twitter" target="_blank" rel="nofollow noreferrer" data-astro-cid-jo6i4kqk><i class="fa-brands fa-twitter" aria-hidden="true" title="Twitter" data-astro-cid-jo6i4kqk></i></a></li><li data-astro-cid-jo6i4kqk><a href="https://instagram.com/evertz.io" aria-label="Instagram" target="_blank" rel="nofollow noreferrer" data-astro-cid-jo6i4kqk><i class="fa-brands fa-instagram" aria-hidden="true" title="Instagram" data-astro-cid-jo6i4kqk></i></a></li><li data-astro-cid-jo6i4kqk><a href="https://linkedin.com/company/evertz-io" aria-label="LinkedIn" target="_blank" rel="nofollow noreferrer" data-astro-cid-jo6i4kqk><i class="fa-brands fa-linkedin" aria-hidden="true" title="LinkedIn" data-astro-cid-jo6i4kqk></i></a></li><li data-astro-cid-jo6i4kqk><a href="/feed.xml" aria-label="RSS Feed" target="_blank" rel="nofollow noreferrer" data-astro-cid-jo6i4kqk><i class="fa fa-rss" aria-hidden="true" title="RSS Feed" data-astro-cid-jo6i4kqk></i></a></li></ul></div></div><div class="col-xl-9 col-lg-9 col-12 text-right" data-astro-cid-jo6i4kqk><div class="social-media privacy-links" data-astro-cid-jo6i4kqk><ul data-astro-cid-jo6i4kqk><li data-astro-cid-jo6i4kqk><a data-fancybox data-type="iframe" data-src="https://app.termly.io/document/privacy-policy/09028dbb-fbf3-4a41-bcee-702eaa8d2352" href="https://app.termly.io/document/privacy-policy/09028dbb-fbf3-4a41-bcee-702eaa8d2352" rel="nofollow noreferrer" data-astro-cid-jo6i4kqk>Privacy</a></li><li data-astro-cid-jo6i4kqk><a data-fancybox data-type="iframe" data-src="https://app.termly.io/document/terms-of-use-for-saas/efa6b8a5-c318-42c6-8ed9-3d0f8f673a28" href="https://app.termly.io/document/terms-of-use-for-saas/efa6b8a5-c318-42c6-8ed9-3d0f8f673a28" rel="nofollow noreferrer" data-astro-cid-jo6i4kqk>Terms</a></li><li data-astro-cid-jo6i4kqk><a href="https://app.termly.io/notify/09028dbb-fbf3-4a41-bcee-702eaa8d2352" target="_blank" rel="nofollow noreferrer" data-astro-cid-jo6i4kqk>Do Not Sell My Info</a></li><li class="mubu" data-astro-cid-jo6i4kqk><button class="termly-cookie-preference-button" type="button" style="background: transparent; border-radius: 0; border: none; display: inline-block;color: #555555; font-weight: 500; box-sizing: border-box; cursor: pointer; padding: 0; outline: none;" onclick="displayPreferenceModal()" data-astro-cid-jo6i4kqk>Manage Cookie Preferences</button></li></ul></div></div></div></div></footer></div><script src="/assets/js/vendor/jquery-3.6.0.min.js"></script><script src="/assets/js/vendor/bootstrap-4.5.0.min.js"></script><script src="/assets/js/vendor/jquery.fancybox-3.5.7.min.js"></script><script src="/assets/js/vendor/wow-1.3.0.min.js"></script><script src="/assets/js/vendor/slick-1.8.0.js"></script><script type="module">new WOW().init(),$(function(){$(`.parallax`).each(function(e){var t=$(this).data(`image-src`),n=$(this).data(`height`);$(this).css(`background-image`,`url(`+t+`)`),$(this).css(`height`,n)})});var e=0;$(`.moma`).each(function(){($(this).height()??0>e)&&(e=$(this).height()??0)}),$(`.moma`).height(e);</script></body></html>