Spaces:

srijithrajamohan
/

ai-research-redis

Running

App Files Files Community

srijithrajamohan commited on Mar 18

Commit

dac83bc

verified ·

1 Parent(s): be4e535

Publish site

Browse files

Files changed (6) hide show

index.html +21 -1
listings.json +2 -0
posts/hybrid-search-retrieval-bm25-modernbert.html +671 -0
posts/index.html +21 -1
search.json +51 -2
sitemap.xml +4 -0

index.html CHANGED Viewed

@@ -200,7 +200,27 @@ window.Quarto = {
 <div class="quarto-listing quarto-listing-container-default" id="listing-listing">
 <div class="list quarto-listing-default">
-<div class="quarto-post image-right" data-index="0" data-listing-date-sort="1773806400000" data-listing-file-modified-sort="1773875458545" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="1" data-listing-word-count-sort="14">
 <div class="body">
 <h3 class="no-anchor listing-title">
 <a href="./posts/hello.html" class="no-external">Hello</a>

 <div class="quarto-listing quarto-listing-container-default" id="listing-listing">
 <div class="list quarto-listing-default">
+<div class="quarto-post image-right" data-index="0" data-categories="c2VhcmNoJTJDcmV0cmlldmFsJTJDYm0yNSUyQ2VtYmVkZGluZ3M=" data-listing-date-sort="1773806400000" data-listing-file-modified-sort="1773878066120" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="4" data-listing-word-count-sort="632">
+<div class="body">
+<h3 class="no-anchor listing-title">
+<a href="./posts/hybrid-search-retrieval-bm25-modernbert.html" class="no-external">Hybrid Search Retrieval: Combining Semantic Search and BM25 Algorithm with Examples</a>
+</h3>
+<div class="delink listing-description"><a href="./posts/hybrid-search-retrieval-bm25-modernbert.html" class="no-external">
+<p>How to combine ModernBERT embedding similarity with BM25, plus failure modes (negation and identifiers).</p>
+</a></div>
+</div>
+<div class="metadata">
+<a href="./posts/hybrid-search-retrieval-bm25-modernbert.html" class="no-external">
+<div class="listing-date">
+Mar 18, 2026
+</div>
+<div class="listing-author">
+Redis AI Research
+</div>
+</a>
+</div>
+</div>
+<div class="quarto-post image-right" data-index="1" data-listing-date-sort="1773806400000" data-listing-file-modified-sort="1773875458545" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="1" data-listing-word-count-sort="14">
 <div class="body">
 <h3 class="no-anchor listing-title">
 <a href="./posts/hello.html" class="no-external">Hello</a>

listings.json CHANGED Viewed

@@ -2,12 +2,14 @@
   {
     "listing": "/index.html",
     "items": [
       "/posts/hello.html"
     ]
   },
   {
     "listing": "/posts/index.html",
     "items": [
       "/posts/hello.html"
     ]
   }

   {
     "listing": "/index.html",
     "items": [
+      "/posts/hybrid-search-retrieval-bm25-modernbert.html",
       "/posts/hello.html"
     ]
   },
   {
     "listing": "/posts/index.html",
     "items": [
+      "/posts/hybrid-search-retrieval-bm25-modernbert.html",
       "/posts/hello.html"
     ]
   }

posts/hybrid-search-retrieval-bm25-modernbert.html ADDED Viewed

	@@ -0,0 +1,671 @@

+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
+<meta charset="utf-8">
+<meta name="generator" content="quarto-1.8.27">
+<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
+<meta name="author" content="Redis AI Research">
+<meta name="dcterms.date" content="2026-03-18">
+<meta name="description" content="How to combine ModernBERT embedding similarity with BM25, plus failure modes (negation and identifiers).">
+<title>Hybrid Search Retrieval: Combining Semantic Search and BM25 Algorithm with Examples – AI Research @Redis</title>
+<style>
+code{white-space: pre-wrap;}
+span.smallcaps{font-variant: small-caps;}
+div.columns{display: flex; gap: min(4vw, 1.5em);}
+div.column{flex: auto; overflow-x: auto;}
+div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
+ul.task-list{list-style: none;}
+ul.task-list li input[type="checkbox"] {
+  width: 0.8em;
+  margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */
+  vertical-align: middle;
+}
+</style>
+<script src="../site_libs/quarto-nav/quarto-nav.js"></script>
+<script src="../site_libs/quarto-nav/headroom.min.js"></script>
+<script src="../site_libs/clipboard/clipboard.min.js"></script>
+<script src="../site_libs/quarto-search/autocomplete.umd.js"></script>
+<script src="../site_libs/quarto-search/fuse.min.js"></script>
+<script src="../site_libs/quarto-search/quarto-search.js"></script>
+<meta name="quarto:offset" content="../">
+<script src="../site_libs/quarto-html/quarto.js" type="module"></script>
+<script src="../site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
+<script src="../site_libs/quarto-html/axe/axe-check.js" type="module"></script>
+<script src="../site_libs/quarto-html/popper.min.js"></script>
+<script src="../site_libs/quarto-html/tippy.umd.min.js"></script>
+<script src="../site_libs/quarto-html/anchor.min.js"></script>
+<link href="../site_libs/quarto-html/tippy.css" rel="stylesheet">
+<link href="../site_libs/quarto-html/quarto-syntax-highlighting-ed96de9b727972fe78a7b5d16c58bf87.css" rel="stylesheet" id="quarto-text-highlighting-styles">
+<script src="../site_libs/bootstrap/bootstrap.min.js"></script>
+<link href="../site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
+<link href="../site_libs/bootstrap/bootstrap-a8976c3e89df70b272bdfba3d2fda974.min.css" rel="stylesheet" append-hash="true" id="quarto-bootstrap" data-mode="light">
+<script id="quarto-search-options" type="application/json">{
+  "location": "navbar",
+  "copy-button": false,
+  "collapse-after": 3,
+  "panel-placement": "end",
+  "type": "overlay",
+  "limit": 50,
+  "keyboard-shortcut": [
+    "f",
+    "/",
+    "s"
+  ],
+  "show-item-context": false,
+  "language": {
+    "search-no-results-text": "No results",
+    "search-matching-documents-text": "matching documents",
+    "search-copy-link-title": "Copy link to search",
+    "search-hide-matches-text": "Hide additional matches",
+    "search-more-match-text": "more match in this document",
+    "search-more-matches-text": "more matches in this document",
+    "search-clear-button-title": "Clear",
+    "search-text-placeholder": "",
+    "search-detached-cancel-button-title": "Cancel",
+    "search-submit-button-title": "Submit",
+    "search-label": "Search"
+  }
+}</script>
+</head>
+<body class="nav-fixed quarto-light">
+<div id="quarto-search-results"></div>
+  <header id="quarto-header" class="headroom fixed-top">
+    <nav class="navbar navbar-expand-lg " data-bs-theme="dark">
+      <div class="navbar-container container-fluid">
+      <div class="navbar-brand-container mx-auto">
+    <a class="navbar-brand" href="../index.html">
+    <span class="navbar-title">AI Research <span class="citation" data-cites="Redis">@Redis</span></span>
+    </a>
+  </div>
+            <div id="quarto-search" class="" title="Search"></div>
+          <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse" role="menu" aria-expanded="false" aria-label="Toggle navigation" onclick="if (window.quartoToggleHeadroom) { window.quartoToggleHeadroom(); }">
+  <span class="navbar-toggler-icon"></span>
+</button>
+          <div class="collapse navbar-collapse" id="navbarCollapse">
+            <ul class="navbar-nav navbar-nav-scroll me-auto">
+  <li class="nav-item">
+    <a class="nav-link" href="../index.html">
+<span class="menu-text">Home</span></a>
+  </li>
+  <li class="nav-item">
+    <a class="nav-link" href="../posts/index.html">
+<span class="menu-text">Posts</span></a>
+  </li>
+</ul>
+          </div> <!-- /navcollapse -->
+            <div class="quarto-navbar-tools">
+</div>
+      </div> <!-- /container-fluid -->
+    </nav>
+</header>
+<!-- content -->
+<div id="quarto-content" class="quarto-container page-columns page-rows-contents page-layout-article page-navbar">
+<!-- sidebar -->
+<!-- margin-sidebar -->
+    <div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
+        <nav id="TOC" role="doc-toc" class="toc-active">
+    <h2 id="toc-title">On this page</h2>
+  <ul>
+  <li><a href="#example-1" id="toc-example-1" class="nav-link active" data-scroll-target="#example-1">Example 1</a></li>
+  <li><a href="#example-2" id="toc-example-2" class="nav-link" data-scroll-target="#example-2">Example 2</a></li>
+  <li><a href="#failure-mode-1---negationsdirectionality" id="toc-failure-mode-1---negationsdirectionality" class="nav-link" data-scroll-target="#failure-mode-1---negationsdirectionality">Failure mode 1 - Negations/directionality</a></li>
+  <li><a href="#failure-mode-2---attributesidentifiers" id="toc-failure-mode-2---attributesidentifiers" class="nav-link" data-scroll-target="#failure-mode-2---attributesidentifiers">Failure mode 2 - Attributes/Identifiers</a></li>
+  <li><a href="#how-does-the-hybrid-score-and-rrf-fare" id="toc-how-does-the-hybrid-score-and-rrf-fare" class="nav-link" data-scroll-target="#how-does-the-hybrid-score-and-rrf-fare">How does the hybrid score and RRF fare?</a></li>
+  <li><a href="#but-what-did-we-want-to-achieve-with-bm25" id="toc-but-what-did-we-want-to-achieve-with-bm25" class="nav-link" data-scroll-target="#but-what-did-we-want-to-achieve-with-bm25">But what did we want to achieve with BM25?</a></li>
+  </ul>
+</nav>
+    </div>
+<!-- main -->
+<main class="content" id="quarto-document-content">
+<header id="title-block-header" class="quarto-title-block default">
+<div class="quarto-title">
+<h1 class="title">Hybrid Search Retrieval: Combining Semantic Search and BM25 Algorithm with Examples</h1>
+  <div class="quarto-categories">
+    <div class="quarto-category">search</div>
+    <div class="quarto-category">retrieval</div>
+    <div class="quarto-category">bm25</div>
+    <div class="quarto-category">embeddings</div>
+  </div>
+  </div>
+<div>
+  <div class="description">
+    How to combine ModernBERT embedding similarity with BM25, plus failure modes (negation and identifiers).
+  </div>
+</div>
+<div class="quarto-title-meta">
+    <div>
+    <div class="quarto-title-meta-heading">Author</div>
+    <div class="quarto-title-meta-contents">
+             <p>Redis AI Research </p>
+          </div>
+  </div>
+    <div>
+    <div class="quarto-title-meta-heading">Published</div>
+    <div class="quarto-title-meta-contents">
+      <p class="date">March 18, 2026</p>
+    </div>
+  </div>
+  </div>
+</header>
+<p>Source: https://redislabs.atlassian.net/wiki/spaces/RED/pages/5798592518/Hybrid+Search+Retrieval+Combining+Semantic+Search+and+BM25+Algorithm+with+Examples</p>
+<p>The examples below illustrate a hybrid search retrieval of a query from a corpus of documents by combining semantic search using ModernBERT embeddings and the BM25 algorithm.</p>
+<p>The scores from each are normalized between 0 and 1 in order to create a weighted average hybrid score. The raw embedding score is also shown in the columns for diagnostics. The ranks in each measure are then combined using RRF to create another score that can be used for retrieval.</p>
+<p>The most correct match for each query is highlighted in each example.</p>
+<section id="example-1" class="level2">
+<h2 class="anchored" data-anchor-id="example-1">Example 1</h2>
+<p>(Screenshot reference: <code>Screenshot 2026-01-08 at 9.07.36 PM.png</code>)</p>
+</section>
+<section id="example-2" class="level2">
+<h2 class="anchored" data-anchor-id="example-2">Example 2</h2>
+<p>(Screenshot reference: <code>Screenshot 2026-01-08 at 9.11.31 PM.png</code>)</p>
+</section>
+<section id="failure-mode-1---negationsdirectionality" class="level2">
+<h2 class="anchored" data-anchor-id="failure-mode-1---negationsdirectionality">Failure mode 1 - Negations/directionality</h2>
+<p>BM25 is not keyword search; it up-weights rare terms and not just keywords.</p>
+<p>In example 1, the key term of interest or the attribute/identifier is <code>#124</code>. However, the other terms in the query <code>semantic</code>, <code>search</code>, <code>for</code>, <code>doc</code> are all relatively rare and contribute to higher BM25 scores as evidenced in the first two records in the table index 8 and 11. These have negated intents however and therefore incorrect matches. BM25 has no ability to identify negations.</p>
+<p>The similarity scores fortunately are not the highest but will most likely exceed most commonly used thresholds. Semantic search can fail to identify negated intent.</p>
+</section>
+<section id="failure-mode-2---attributesidentifiers" class="level2">
+<h2 class="anchored" data-anchor-id="failure-mode-2---attributesidentifiers">Failure mode 2 - Attributes/Identifiers</h2>
+<p>“The next three docs given by ids 5, 7 and 6 have the same phrasing but with the wrong identifier of #123, #1243 and #1233 respectively which lowers their BM25 score a bit but certainly higher than all the other documents. BM25 can detect a different attribute/identifier but the actual magnitude of this will depend on what the rest of the sentence looks like.”</p>
+<ul>
+<li>If there is strong lexical overlap but only the identifier is different, there will be a small drop in the BM25 score.</li>
+<li>If the rest of the sentence is NULL then the BM25 score is 0.</li>
+<li>If there is very little overlap in the rest of the query and the doc.</li>
+</ul>
+<p>BM25 can identify attribute/identifier changes but is hard to put a threshold on this score.</p>
+<p>W.r.t embeddings two things are notable:</p>
+<ul>
+<li>There are two forces at play here: the global semantic intent match and the identifier match.</li>
+<li>In 8 and 11, there is negated intent whereas there is an identifier match.</li>
+<li>In 5, 6 and 7 the general intent is correct but the identifier is incorrect.</li>
+<li>The resulting score is a complex interaction of the two and the answer to “Can we identify negations or attribute changes with a threshold?” is heavily data-dependent.</li>
+</ul>
+<p>Note that the embedding scores for 5, 7 and 6 are some of the highest because attribute/identifier differences are something that embeddings struggle to detect.</p>
+</section>
+<section id="how-does-the-hybrid-score-and-rrf-fare" class="level2">
+<h2 class="anchored" data-anchor-id="how-does-the-hybrid-score-and-rrf-fare">How does the hybrid score and RRF fare?</h2>
+<p>Averaging out with a weighted hybrid score or RRF has no impact here. In example 1, record id 9 which is the closest match still does not rank highly in either of these scenarios.</p>
+<p>To summarize the above for both approaches for the failure modes, it seems that embeddings seem to have the slight edge here.</p>
+<table class="caption-top table">
+<thead>
+<tr class="header">
+<th>Method</th>
+<th>Negation</th>
+<th>Directionality</th>
+<th>Attribute/Identifier</th>
+</tr>
+</thead>
+<tbody>
+<tr class="odd">
+<td>BM25</td>
+<td>N</td>
+<td>N</td>
+<td>N (hard to put a threshold)</td>
+</tr>
+<tr class="even">
+<td>Embedding</td>
+<td>Sometimes</td>
+<td>N</td>
+<td>Sometimes</td>
+</tr>
+</tbody>
+</table>
+<p>We can’t eliminate all the issues, but we can hope to mitigate based on our understanding of the data.</p>
+</section>
+<section id="but-what-did-we-want-to-achieve-with-bm25" class="level2">
+<h2 class="anchored" data-anchor-id="but-what-did-we-want-to-achieve-with-bm25">But what did we want to achieve with BM25?</h2>
+<p>We are really looking to filter out matches that do not have the right attributes/identifiers we seek. BM25 looks like it could provide that, but does more than attribute matching.</p>
+<ul>
+<li>Extract identifiers from query.</li>
+<li>Perform embedding based retrieval of query + docs to retrieve set M.</li>
+<li>Filter out docs where identifiers do not match in set M.</li>
+<li>Unfortunately, this does nothing to address the negation/directionality limitation.</li>
+</ul>
+</section>
+</main> <!-- /main -->
+<script id="quarto-html-after-body" type="application/javascript">
+  window.document.addEventListener("DOMContentLoaded", function (event) {
+    const icon = "";
+    const anchorJS = new window.AnchorJS();
+    anchorJS.options = {
+      placement: 'right',
+      icon: icon
+    };
+    anchorJS.add('.anchored');
+    const isCodeAnnotation = (el) => {
+      for (const clz of el.classList) {
+        if (clz.startsWith('code-annotation-')) {
+          return true;
+        }
+      }
+      return false;
+    }
+    const onCopySuccess = function(e) {
+      // button target
+      const button = e.trigger;
+      // don't keep focus
+      button.blur();
+      // flash "checked"
+      button.classList.add('code-copy-button-checked');
+      var currentTitle = button.getAttribute("title");
+      button.setAttribute("title", "Copied!");
+      let tooltip;
+      if (window.bootstrap) {
+        button.setAttribute("data-bs-toggle", "tooltip");
+        button.setAttribute("data-bs-placement", "left");
+        button.setAttribute("data-bs-title", "Copied!");
+        tooltip = new bootstrap.Tooltip(button,
+          { trigger: "manual",
+            customClass: "code-copy-button-tooltip",
+            offset: [0, -8]});
+        tooltip.show();
+      }
+      setTimeout(function() {
+        if (tooltip) {
+          tooltip.hide();
+          button.removeAttribute("data-bs-title");
+          button.removeAttribute("data-bs-toggle");
+          button.removeAttribute("data-bs-placement");
+        }
+        button.setAttribute("title", currentTitle);
+        button.classList.remove('code-copy-button-checked');
+      }, 1000);
+      // clear code selection
+      e.clearSelection();
+    }
+    const getTextToCopy = function(trigger) {
+      const outerScaffold = trigger.parentElement.cloneNode(true);
+      const codeEl = outerScaffold.querySelector('code');
+      for (const childEl of codeEl.children) {
+        if (isCodeAnnotation(childEl)) {
+          childEl.remove();
+        }
+      }
+      return codeEl.innerText;
+    }
+    const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
+      text: getTextToCopy
+    });
+    clipboard.on('success', onCopySuccess);
+    if (window.document.getElementById('quarto-embedded-source-code-modal')) {
+      const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
+        text: getTextToCopy,
+        container: window.document.getElementById('quarto-embedded-source-code-modal')
+      });
+      clipboardModal.on('success', onCopySuccess);
+    }
+      var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
+      var mailtoRegex = new RegExp(/^mailto:/);
+        var filterRegex = new RegExp("https:\/\/srijithrajamohan-ai-research-redis\.static\.hf\.space");
+      var isInternal = (href) => {
+          return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
+      }
+      // Inspect non-navigation links and adorn them if external
+     var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
+      for (var i=0; i<links.length; i++) {
+        const link = links[i];
+        if (!isInternal(link.href)) {
+          // undo the damage that might have been done by quarto-nav.js in the case of
+          // links that we want to consider external
+          if (link.dataset.originalHref !== undefined) {
+            link.href = link.dataset.originalHref;
+          }
+        }
+      }
+    function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
+      const config = {
+        allowHTML: true,
+        maxWidth: 500,
+        delay: 100,
+        arrow: false,
+        appendTo: function(el) {
+            return el.parentElement;
+        },
+        interactive: true,
+        interactiveBorder: 10,
+        theme: 'quarto',
+        placement: 'bottom-start',
+      };
+      if (contentFn) {
+        config.content = contentFn;
+      }
+      if (onTriggerFn) {
+        config.onTrigger = onTriggerFn;
+      }
+      if (onUntriggerFn) {
+        config.onUntrigger = onUntriggerFn;
+      }
+      window.tippy(el, config);
+    }
+    const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
+    for (var i=0; i<noterefs.length; i++) {
+      const ref = noterefs[i];
+      tippyHover(ref, function() {
+        // use id or data attribute instead here
+        let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
+        try { href = new URL(href).hash; } catch {}
+        const id = href.replace(/^#\/?/, "");
+        const note = window.document.getElementById(id);
+        if (note) {
+          return note.innerHTML;
+        } else {
+          return "";
+        }
+      });
+    }
+    const xrefs = window.document.querySelectorAll('a.quarto-xref');
+    const processXRef = (id, note) => {
+      // Strip column container classes
+      const stripColumnClz = (el) => {
+        el.classList.remove("page-full", "page-columns");
+        if (el.children) {
+          for (const child of el.children) {
+            stripColumnClz(child);
+          }
+        }
+      }
+      stripColumnClz(note)
+      if (id === null || id.startsWith('sec-')) {
+        // Special case sections, only their first couple elements
+        const container = document.createElement("div");
+        if (note.children && note.children.length > 2) {
+          container.appendChild(note.children[0].cloneNode(true));
+          for (let i = 1; i < note.children.length; i++) {
+            const child = note.children[i];
+            if (child.tagName === "P" && child.innerText === "") {
+              continue;
+            } else {
+              container.appendChild(child.cloneNode(true));
+              break;
+            }
+          }
+          if (window.Quarto?.typesetMath) {
+            window.Quarto.typesetMath(container);
+          }
+          return container.innerHTML
+        } else {
+          if (window.Quarto?.typesetMath) {
+            window.Quarto.typesetMath(note);
+          }
+          return note.innerHTML;
+        }
+      } else {
+        // Remove any anchor links if they are present
+        const anchorLink = note.querySelector('a.anchorjs-link');
+        if (anchorLink) {
+          anchorLink.remove();
+        }
+        if (window.Quarto?.typesetMath) {
+          window.Quarto.typesetMath(note);
+        }
+        if (note.classList.contains("callout")) {
+          return note.outerHTML;
+        } else {
+          return note.innerHTML;
+        }
+      }
+    }
+    for (var i=0; i<xrefs.length; i++) {
+      const xref = xrefs[i];
+      tippyHover(xref, undefined, function(instance) {
+        instance.disable();
+        let url = xref.getAttribute('href');
+        let hash = undefined;
+        if (url.startsWith('#')) {
+          hash = url;
+        } else {
+          try { hash = new URL(url).hash; } catch {}
+        }
+        if (hash) {
+          const id = hash.replace(/^#\/?/, "");
+          const note = window.document.getElementById(id);
+          if (note !== null) {
+            try {
+              const html = processXRef(id, note.cloneNode(true));
+              instance.setContent(html);
+            } finally {
+              instance.enable();
+              instance.show();
+            }
+          } else {
+            // See if we can fetch this
+            fetch(url.split('#')[0])
+            .then(res => res.text())
+            .then(html => {
+              const parser = new DOMParser();
+              const htmlDoc = parser.parseFromString(html, "text/html");
+              const note = htmlDoc.getElementById(id);
+              if (note !== null) {
+                const html = processXRef(id, note);
+                instance.setContent(html);
+              }
+            }).finally(() => {
+              instance.enable();
+              instance.show();
+            });
+          }
+        } else {
+          // See if we can fetch a full url (with no hash to target)
+          // This is a special case and we should probably do some content thinning / targeting
+          fetch(url)
+          .then(res => res.text())
+          .then(html => {
+            const parser = new DOMParser();
+            const htmlDoc = parser.parseFromString(html, "text/html");
+            const note = htmlDoc.querySelector('main.content');
+            if (note !== null) {
+              // This should only happen for chapter cross references
+              // (since there is no id in the URL)
+              // remove the first header
+              if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
+                note.children[0].remove();
+              }
+              const html = processXRef(null, note);
+              instance.setContent(html);
+            }
+          }).finally(() => {
+            instance.enable();
+            instance.show();
+          });
+        }
+      }, function(instance) {
+      });
+    }
+        let selectedAnnoteEl;
+        const selectorForAnnotation = ( cell, annotation) => {
+          let cellAttr = 'data-code-cell="' + cell + '"';
+          let lineAttr = 'data-code-annotation="' +  annotation + '"';
+          const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
+          return selector;
+        }
+        const selectCodeLines = (annoteEl) => {
+          const doc = window.document;
+          const targetCell = annoteEl.getAttribute("data-target-cell");
+          const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
+          const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
+          const lines = annoteSpan.getAttribute("data-code-lines").split(",");
+          const lineIds = lines.map((line) => {
+            return targetCell + "-" + line;
+          })
+          let top = null;
+          let height = null;
+          let parent = null;
+          if (lineIds.length > 0) {
+              //compute the position of the single el (top and bottom and make a div)
+              const el = window.document.getElementById(lineIds[0]);
+              top = el.offsetTop;
+              height = el.offsetHeight;
+              parent = el.parentElement.parentElement;
+            if (lineIds.length > 1) {
+              const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
+              const bottom = lastEl.offsetTop + lastEl.offsetHeight;
+              height = bottom - top;
+            }
+            if (top !== null && height !== null && parent !== null) {
+              // cook up a div (if necessary) and position it
+              let div = window.document.getElementById("code-annotation-line-highlight");
+              if (div === null) {
+                div = window.document.createElement("div");
+                div.setAttribute("id", "code-annotation-line-highlight");
+                div.style.position = 'absolute';
+                parent.appendChild(div);
+              }
+              div.style.top = top - 2 + "px";
+              div.style.height = height + 4 + "px";
+              div.style.left = 0;
+              let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
+              if (gutterDiv === null) {
+                gutterDiv = window.document.createElement("div");
+                gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
+                gutterDiv.style.position = 'absolute';
+                const codeCell = window.document.getElementById(targetCell);
+                const gutter = codeCell.querySelector('.code-annotation-gutter');
+                gutter.appendChild(gutterDiv);
+              }
+              gutterDiv.style.top = top - 2 + "px";
+              gutterDiv.style.height = height + 4 + "px";
+            }
+            selectedAnnoteEl = annoteEl;
+          }
+        };
+        const unselectCodeLines = () => {
+          const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
+          elementsIds.forEach((elId) => {
+            const div = window.document.getElementById(elId);
+            if (div) {
+              div.remove();
+            }
+          });
+          selectedAnnoteEl = undefined;
+        };
+          // Handle positioning of the toggle
+      window.addEventListener(
+        "resize",
+        throttle(() => {
+          elRect = undefined;
+          if (selectedAnnoteEl) {
+            selectCodeLines(selectedAnnoteEl);
+          }
+        }, 10)
+      );
+      function throttle(fn, ms) {
+      let throttle = false;
+      let timer;
+        return (...args) => {
+          if(!throttle) { // first call gets through
+              fn.apply(this, args);
+              throttle = true;
+          } else { // all the others get throttled
+              if(timer) clearTimeout(timer); // cancel #2
+              timer = setTimeout(() => {
+                fn.apply(this, args);
+                timer = throttle = false;
+              }, ms);
+          }
+        };
+      }
+        // Attach click handler to the DT
+        const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
+        for (const annoteDlNode of annoteDls) {
+          annoteDlNode.addEventListener('click', (event) => {
+            const clickedEl = event.target;
+            if (clickedEl !== selectedAnnoteEl) {
+              unselectCodeLines();
+              const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
+              if (activeEl) {
+                activeEl.classList.remove('code-annotation-active');
+              }
+              selectCodeLines(clickedEl);
+              clickedEl.classList.add('code-annotation-active');
+            } else {
+              // Unselect the line
+              unselectCodeLines();
+              clickedEl.classList.remove('code-annotation-active');
+            }
+          });
+        }
+    const findCites = (el) => {
+      const parentEl = el.parentElement;
+      if (parentEl) {
+        const cites = parentEl.dataset.cites;
+        if (cites) {
+          return {
+            el,
+            cites: cites.split(' ')
+          };
+        } else {
+          return findCites(el.parentElement)
+        }
+      } else {
+        return undefined;
+      }
+    };
+    var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
+    for (var i=0; i<bibliorefs.length; i++) {
+      const ref = bibliorefs[i];
+      const citeInfo = findCites(ref);
+      if (citeInfo) {
+        tippyHover(citeInfo.el, function() {
+          var popup = window.document.createElement('div');
+          citeInfo.cites.forEach(function(cite) {
+            var citeDiv = window.document.createElement('div');
+            citeDiv.classList.add('hanging-indent');
+            citeDiv.classList.add('csl-entry');
+            var biblioDiv = window.document.getElementById('ref-' + cite);
+            if (biblioDiv) {
+              citeDiv.innerHTML = biblioDiv.innerHTML;
+            }
+            popup.appendChild(citeDiv);
+          });
+          return popup.innerHTML;
+        });
+      }
+    }
+  });
+  </script>
+</div> <!-- /content -->
+<footer class="footer">
+  <div class="nav-footer">
+    <div class="nav-footer-left">
+<p>Discover this project on Hugging Face Spaces: https://huggingface.co/spaces/srijithrajamohan/ai-research-redis</p>
+</div>
+    <div class="nav-footer-center">
+      &nbsp;
+    </div>
+    <div class="nav-footer-right">
+      &nbsp;
+    </div>
+  </div>
+</footer>
+</body></html>

posts/index.html CHANGED Viewed

@@ -199,7 +199,27 @@ window.Quarto = {
 <div class="quarto-listing quarto-listing-container-default" id="listing-listing">
 <div class="list quarto-listing-default">
-<div class="quarto-post image-right" data-index="0" data-listing-date-sort="1773806400000" data-listing-file-modified-sort="1773875458545" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="1" data-listing-word-count-sort="14">
 <div class="body">
 <h3 class="no-anchor listing-title">
 <a href="../posts/hello.html" class="no-external">Hello</a>

 <div class="quarto-listing quarto-listing-container-default" id="listing-listing">
 <div class="list quarto-listing-default">
+<div class="quarto-post image-right" data-index="0" data-categories="c2VhcmNoJTJDcmV0cmlldmFsJTJDYm0yNSUyQ2VtYmVkZGluZ3M=" data-listing-date-sort="1773806400000" data-listing-file-modified-sort="1773878066120" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="4" data-listing-word-count-sort="632">
+<div class="body">
+<h3 class="no-anchor listing-title">
+<a href="../posts/hybrid-search-retrieval-bm25-modernbert.html" class="no-external">Hybrid Search Retrieval: Combining Semantic Search and BM25 Algorithm with Examples</a>
+</h3>
+<div class="delink listing-description"><a href="../posts/hybrid-search-retrieval-bm25-modernbert.html" class="no-external">
+<p>How to combine ModernBERT embedding similarity with BM25, plus failure modes (negation and identifiers).</p>
+</a></div>
+</div>
+<div class="metadata">
+<a href="../posts/hybrid-search-retrieval-bm25-modernbert.html" class="no-external">
+<div class="listing-date">
+Mar 18, 2026
+</div>
+<div class="listing-author">
+Redis AI Research
+</div>
+</a>
+</div>
+</div>
+<div class="quarto-post image-right" data-index="1" data-listing-date-sort="1773806400000" data-listing-file-modified-sort="1773875458545" data-listing-date-modified-sort="NaN" data-listing-reading-time-sort="1" data-listing-word-count-sort="14">
 <div class="body">
 <h3 class="no-anchor listing-title">
 <a href="../posts/hello.html" class="no-external">Hello</a>

search.json CHANGED Viewed

@@ -4,14 +4,63 @@
     "href": "index.html",
     "title": "AI Research @Redis",
     "section": "",
-    "text": "Homepage of the AI Research team at Redis.\n\n\n\n\n\n\n\n\n\nHello\n\n\nFirst Quarto post (template).\n\n\n\n\n\nMar 18, 2026\n\n\nRedis AI Research\n\n\n\n\n\nNo matching items"
   },
   {
     "objectID": "posts/index.html",
     "href": "posts/index.html",
     "title": "Posts",
     "section": "",
-    "text": "Hello\n\n\nFirst Quarto post (template).\n\n\n\n\n\nMar 18, 2026\n\n\nRedis AI Research\n\n\n\n\n\nNo matching items"
   },
   {
     "objectID": "posts/hello.html",

     "href": "index.html",
     "title": "AI Research @Redis",
     "section": "",
+    "text": "Homepage of the AI Research team at Redis.\n\n\n\n\n\n\n\n\n\nHybrid Search Retrieval: Combining Semantic Search and BM25 Algorithm with Examples\n\n\nHow to combine ModernBERT embedding similarity with BM25, plus failure modes (negation and identifiers).\n\n\n\n\n\nMar 18, 2026\n\n\nRedis AI Research\n\n\n\n\n\n\n\nHello\n\n\nFirst Quarto post (template).\n\n\n\n\n\nMar 18, 2026\n\n\nRedis AI Research\n\n\n\n\n\nNo matching items"
+  },
+  {
+    "objectID": "posts/hybrid-search-retrieval-bm25-modernbert.html",
+    "href": "posts/hybrid-search-retrieval-bm25-modernbert.html",
+    "title": "Hybrid Search Retrieval: Combining Semantic Search and BM25 Algorithm with Examples",
+    "section": "",
+    "text": "Source: https://redislabs.atlassian.net/wiki/spaces/RED/pages/5798592518/Hybrid+Search+Retrieval+Combining+Semantic+Search+and+BM25+Algorithm+with+Examples\nThe examples below illustrate a hybrid search retrieval of a query from a corpus of documents by combining semantic search using ModernBERT embeddings and the BM25 algorithm.\nThe scores from each are normalized between 0 and 1 in order to create a weighted average hybrid score. The raw embedding score is also shown in the columns for diagnostics. The ranks in each measure are then combined using RRF to create another score that can be used for retrieval.\nThe most correct match for each query is highlighted in each example."
+  },
+  {
+    "objectID": "posts/hybrid-search-retrieval-bm25-modernbert.html#example-1",
+    "href": "posts/hybrid-search-retrieval-bm25-modernbert.html#example-1",
+    "title": "Hybrid Search Retrieval: Combining Semantic Search and BM25 Algorithm with Examples",
+    "section": "Example 1",
+    "text": "Example 1\n(Screenshot reference: Screenshot 2026-01-08 at 9.07.36 PM.png)"
+  },
+  {
+    "objectID": "posts/hybrid-search-retrieval-bm25-modernbert.html#example-2",
+    "href": "posts/hybrid-search-retrieval-bm25-modernbert.html#example-2",
+    "title": "Hybrid Search Retrieval: Combining Semantic Search and BM25 Algorithm with Examples",
+    "section": "Example 2",
+    "text": "Example 2\n(Screenshot reference: Screenshot 2026-01-08 at 9.11.31 PM.png)"
+  },
+  {
+    "objectID": "posts/hybrid-search-retrieval-bm25-modernbert.html#failure-mode-1---negationsdirectionality",
+    "href": "posts/hybrid-search-retrieval-bm25-modernbert.html#failure-mode-1---negationsdirectionality",
+    "title": "Hybrid Search Retrieval: Combining Semantic Search and BM25 Algorithm with Examples",
+    "section": "Failure mode 1 - Negations/directionality",
+    "text": "Failure mode 1 - Negations/directionality\nBM25 is not keyword search; it up-weights rare terms and not just keywords.\nIn example 1, the key term of interest or the attribute/identifier is #124. However, the other terms in the query semantic, search, for, doc are all relatively rare and contribute to higher BM25 scores as evidenced in the first two records in the table index 8 and 11. These have negated intents however and therefore incorrect matches. BM25 has no ability to identify negations.\nThe similarity scores fortunately are not the highest but will most likely exceed most commonly used thresholds. Semantic search can fail to identify negated intent."
+  },
+  {
+    "objectID": "posts/hybrid-search-retrieval-bm25-modernbert.html#failure-mode-2---attributesidentifiers",
+    "href": "posts/hybrid-search-retrieval-bm25-modernbert.html#failure-mode-2---attributesidentifiers",
+    "title": "Hybrid Search Retrieval: Combining Semantic Search and BM25 Algorithm with Examples",
+    "section": "Failure mode 2 - Attributes/Identifiers",
+    "text": "Failure mode 2 - Attributes/Identifiers\n“The next three docs given by ids 5, 7 and 6 have the same phrasing but with the wrong identifier of #123, #1243 and #1233 respectively which lowers their BM25 score a bit but certainly higher than all the other documents. BM25 can detect a different attribute/identifier but the actual magnitude of this will depend on what the rest of the sentence looks like.”\n\nIf there is strong lexical overlap but only the identifier is different, there will be a small drop in the BM25 score.\nIf the rest of the sentence is NULL then the BM25 score is 0.\nIf there is very little overlap in the rest of the query and the doc.\n\nBM25 can identify attribute/identifier changes but is hard to put a threshold on this score.\nW.r.t embeddings two things are notable:\n\nThere are two forces at play here: the global semantic intent match and the identifier match.\nIn 8 and 11, there is negated intent whereas there is an identifier match.\nIn 5, 6 and 7 the general intent is correct but the identifier is incorrect.\nThe resulting score is a complex interaction of the two and the answer to “Can we identify negations or attribute changes with a threshold?” is heavily data-dependent.\n\nNote that the embedding scores for 5, 7 and 6 are some of the highest because attribute/identifier differences are something that embeddings struggle to detect."
+  },
+  {
+    "objectID": "posts/hybrid-search-retrieval-bm25-modernbert.html#how-does-the-hybrid-score-and-rrf-fare",
+    "href": "posts/hybrid-search-retrieval-bm25-modernbert.html#how-does-the-hybrid-score-and-rrf-fare",
+    "title": "Hybrid Search Retrieval: Combining Semantic Search and BM25 Algorithm with Examples",
+    "section": "How does the hybrid score and RRF fare?",
+    "text": "How does the hybrid score and RRF fare?\nAveraging out with a weighted hybrid score or RRF has no impact here. In example 1, record id 9 which is the closest match still does not rank highly in either of these scenarios.\nTo summarize the above for both approaches for the failure modes, it seems that embeddings seem to have the slight edge here.\n\n\n\nMethod\nNegation\nDirectionality\nAttribute/Identifier\n\n\n\n\nBM25\nN\nN\nN (hard to put a threshold)\n\n\nEmbedding\nSometimes\nN\nSometimes\n\n\n\nWe can’t eliminate all the issues, but we can hope to mitigate based on our understanding of the data."
+  },
+  {
+    "objectID": "posts/hybrid-search-retrieval-bm25-modernbert.html#but-what-did-we-want-to-achieve-with-bm25",
+    "href": "posts/hybrid-search-retrieval-bm25-modernbert.html#but-what-did-we-want-to-achieve-with-bm25",
+    "title": "Hybrid Search Retrieval: Combining Semantic Search and BM25 Algorithm with Examples",
+    "section": "But what did we want to achieve with BM25?",
+    "text": "But what did we want to achieve with BM25?\nWe are really looking to filter out matches that do not have the right attributes/identifiers we seek. BM25 looks like it could provide that, but does more than attribute matching.\n\nExtract identifiers from query.\nPerform embedding based retrieval of query + docs to retrieve set M.\nFilter out docs where identifiers do not match in set M.\nUnfortunately, this does nothing to address the negation/directionality limitation."
   },
   {
     "objectID": "posts/index.html",
     "href": "posts/index.html",
     "title": "Posts",
     "section": "",
+    "text": "Hybrid Search Retrieval: Combining Semantic Search and BM25 Algorithm with Examples\n\n\nHow to combine ModernBERT embedding similarity with BM25, plus failure modes (negation and identifiers).\n\n\n\n\n\nMar 18, 2026\n\n\nRedis AI Research\n\n\n\n\n\n\n\nHello\n\n\nFirst Quarto post (template).\n\n\n\n\n\nMar 18, 2026\n\n\nRedis AI Research\n\n\n\n\n\nNo matching items"
   },
   {
     "objectID": "posts/hello.html",

sitemap.xml CHANGED Viewed

@@ -4,6 +4,10 @@
     <loc>https://srijithrajamohan-ai-research-redis.static.hf.space/index.html</loc>
     <lastmod>2026-03-18T23:10:24.034Z</lastmod>
   </url>
   <url>
     <loc>https://srijithrajamohan-ai-research-redis.static.hf.space/posts/index.html</loc>
     <lastmod>2026-03-18T23:10:26.244Z</lastmod>

     <loc>https://srijithrajamohan-ai-research-redis.static.hf.space/index.html</loc>
     <lastmod>2026-03-18T23:10:24.034Z</lastmod>
   </url>
+  <url>
+    <loc>https://srijithrajamohan-ai-research-redis.static.hf.space/posts/hybrid-search-retrieval-bm25-modernbert.html</loc>
+    <lastmod>2026-03-18T23:54:26.120Z</lastmod>
+  </url>
   <url>
     <loc>https://srijithrajamohan-ai-research-redis.static.hf.space/posts/index.html</loc>
     <lastmod>2026-03-18T23:10:26.244Z</lastmod>