feat(git,opt-in): experimental git grep backend

Issue
=====

The performance in huge repositories like
https://github.com/llvm/llvm-project is extremely slow. It takes ripgrep
about a minute to search through the entire codebase before any
completion suggestions are shown.

Solution
========

Provide a much faster alternative backend that uses `git grep` to search
the codebase. This is still beta quality, but I think it can be improved
incrementally to be pretty good.

This backend has the following benefits and drawbacks:

Benefits:
- much faster (5 seconds instead of 1 minute in llvm-project)

Drawbacks:
- does not work in non-git repositories
- not very configurable yet, but can be improved
- no way to ignore files based on their size like with ripgrep

Please provide feedback on this! I would like to add good support and
tests for
- git submodules
- git worktrees
- any additional features that are cool

Related to #110

README.md

@@ -141,6 +141,10 @@ return {
                 -- completion results. Example: "<leader>tg"
                 on_off = "<leader>tg",
               },
+
+              -- The backend to use for searching. Defaults to "ripgrep".
+              -- "gitgrep" is available as a preview right now.
+              backend = "ripgrep",
             },
 
             -- Show debug information in `:messages` that can help in

integration-tests/MyTestDirectory.ts

@@ -82,6 +82,10 @@ export const MyTestDirectorySchema = z.object({
           name: z.literal("use_case_sensitive_search.lua"),
           type: z.literal("file"),
         }),
+        "use_gitgrep_backend.lua": z.object({
+          name: z.literal("use_gitgrep_backend.lua"),
+          type: z.literal("file"),
+        }),
         "use_manual_mode.lua": z.object({
           name: z.literal("use_manual_mode.lua"),
           type: z.literal("file"),
@@ -176,6 +180,7 @@ export const testDirectoryFiles = z.enum([
   "config-modifications/set_ignore_paths.lua",
   "config-modifications/use_additional_paths.lua",
   "config-modifications/use_case_sensitive_search.lua",
+  "config-modifications/use_gitgrep_backend.lua",
   "config-modifications/use_manual_mode.lua",
   "config-modifications/use_not_found_project_root.lua",
   "config-modifications",

integration-tests/cypress/e2e/blink-ripgrep/basic_spec.cy.ts → integration-tests/cypress/e2e/blink-ripgrep/backend_gitgrep_spec.cy.ts

@@ -1,11 +1,27 @@
 import { flavors } from "@catppuccin/palette"
 import { rgbify } from "@tui-sandbox/library/dist/src/client/color-utilities"
+import type { NeovimContext } from "cypress/support/tui-sandbox"
+import z from "zod"
 import { createGitReposToLimitSearchScope } from "./createGitReposToLimitSearchScope"
 
-describe("the basics", () => {
+type NeovimArguments = Parameters<typeof cy.startNeovim>[0]
+
+function startNeovimWithGitBackend(
+  options: Partial<NeovimArguments>,
+): Cypress.Chainable<NeovimContext> {
+  if (!options) options = {}
+  options.startupScriptModifications = options.startupScriptModifications ?? []
+  if (!options.startupScriptModifications.includes("use_gitgrep_backend.lua")) {
+    options.startupScriptModifications.push("use_gitgrep_backend.lua")
+  }
+  assert(options.startupScriptModifications.includes("use_gitgrep_backend.lua"))
+  return cy.startNeovim(options)
+}
+
+describe("the GitGrepBackend", () => {
   it("shows words in other files as suggestions", () => {
     cy.visit("/")
-    cy.startNeovim().then((nvim) => {
+    startNeovimWithGitBackend({}).then((nvim) => {
       // wait until text on the start screen is visible
       cy.contains("If you see this text, Neovim is ready!")
       createGitReposToLimitSearchScope()
@@ -38,7 +54,7 @@ describe("the basics", () => {
 
   it("allows invoking manually as a blink-cmp keymap", () => {
     cy.visit("/")
-    cy.startNeovim({
+    startNeovimWithGitBackend({
       startupScriptModifications: [
         "use_manual_mode.lua",
         // make sure this is tested somewhere. it doesn't really belong to this
@@ -66,7 +82,7 @@ describe("the basics", () => {
 
   it("can use an underscore (_) ca be used to trigger blink completions", () => {
     cy.visit("/")
-    cy.startNeovim({}).then(() => {
+    startNeovimWithGitBackend({}).then(() => {
       // wait until text on the start screen is visible
       cy.contains("If you see this text, Neovim is ready!")
       createGitReposToLimitSearchScope()
@@ -85,57 +101,13 @@ describe("the basics", () => {
     })
   })
 
-  it("does not search in ignore_paths", () => {
-    // By default, the paths ignored via git and ripgrep are also automatically
-    // ignored by blink-ripgrep.nvim, without any extra features (this is a
-    // ripgrep feature). However, the user may want to ignore some paths from
-    // blink-ripgrep.nvim specifically. Here we test that feature.
-    cy.visit("/")
-    cy.startNeovim({
-      filename: "limited/subproject/file1.lua",
-      startupScriptModifications: ["set_ignore_paths.lua"],
-    }).then((nvim) => {
-      // wait until text on the start screen is visible
-      cy.contains("This is text from file1.lua")
-      createGitReposToLimitSearchScope()
-      const ignorePath = nvim.dir.rootPathAbsolute + "/limited"
-      nvim.runLuaCode({
-        luaCode: `_G.set_ignore_paths({ "${ignorePath}" })`,
-      })
-
-      // clear the current line and enter insert mode
-      cy.typeIntoTerminal("cc")
-
-      // this will match text from ../../../test-environment/other-file.lua
-      //
-      // If the plugin works, this text should show up as a suggestion.
-      cy.typeIntoTerminal("hip")
-
-      nvim
-        .runLuaCode({
-          luaCode: `return _G.blink_ripgrep_invocations`,
-        })
-        .should((result) => {
-          // ripgrep should only have been invoked once
-          expect(result.value).to.be.an("array")
-          expect(result.value).to.have.length(1)
-
-          const invocations = (result.value as string[][])[0]
-          const invocation = invocations[0]
-          expect(invocation).to.eql("ignored")
-        })
-
-      cy.contains("Hippopotamus" + "234 (rg)").should("not.exist")
-    })
-  })
-
   it("shows 5 lines around the match by default", () => {
     // The match context means the lines around the matched line.
     // We want to show context so that the user can see/remember where the match
     // was found. Although we don't explicitly show all the matches in the
     // project, this can still be very useful.
     cy.visit("/")
-    cy.startNeovim().then(() => {
+    startNeovimWithGitBackend({}).then(() => {
       cy.contains("If you see this text, Neovim is ready!")
       createGitReposToLimitSearchScope()
 
@@ -161,26 +133,6 @@ describe("the basics", () => {
     })
   })
 
-  it("can use additional_paths to include additional files in the search", () => {
-    // By default, ripgrep allows using gitignore files to exclude files from
-    // the search. It works exactly like git does, and allows an intuitive way
-    // to exclude files.
-    cy.visit("/")
-    cy.startNeovim({
-      filename: "limited/dir with spaces/file with spaces.txt",
-      startupScriptModifications: ["use_additional_paths.lua"],
-    }).then(() => {
-      // wait until text on the start screen is visible
-      cy.contains("this is file with spaces.txt")
-      createGitReposToLimitSearchScope()
-      cy.typeIntoTerminal("cc")
-
-      // search for something that will be found in the additional words.txt file
-      cy.typeIntoTerminal("abas")
-      cy.contains("abased")
-    })
-  })
-
   function assertMatchVisible(
     match: string,
     color?: typeof flavors.macchiato.colors.green.rgb,
@@ -191,4 +143,128 @@ describe("the basics", () => {
       rgbify(color ?? flavors.macchiato.colors.green.rgb),
     )
   }
+
+  afterEach(() => {
+    verifyGitGrepBackendWasUsedInTest()
+  })
+})
+
+describe("in debug mode", () => {
+  it("can execute the debug command in a shell", () => {
+    cy.visit("/")
+    startNeovimWithGitBackend({}).then((nvim) => {
+      // wait until text on the start screen is visible
+      cy.contains("If you see this text, Neovim is ready!")
+      createGitReposToLimitSearchScope()
+
+      // clear the current line and enter insert mode
+      cy.typeIntoTerminal("cc")
+
+      cy.typeIntoTerminal("spa")
+      cy.contains("spaceroni-macaroni")
+
+      nvim.runExCommand({ command: "messages" }).then((result) => {
+        // make sure the logged command can be run in a shell
+        expect(result.value)
+        cy.log(result.value ?? "")
+
+        cy.typeIntoTerminal("{esc}:term{enter}", { delay: 3 })
+
+        // get the current buffer name
+        nvim.runExCommand({ command: "echo expand('%')" }).then((bufname) => {
+          cy.log(bufname.value ?? "")
+          expect(bufname.value).to.contain("term://")
+        })
+
+        // start insert mode
+        cy.typeIntoTerminal("a")
+
+        // Quickly send the text over instead of typing it out. Cypress is a
+        // bit slow when writing a lot of text.
+        nvim.runLuaCode({
+          luaCode: `vim.api.nvim_feedkeys([[${result.value}]], "n", true)`,
+        })
+        cy.typeIntoTerminal("{enter}")
+
+        // The results will be 5-10 lines of jsonl.
+        // Somewhere in the results, we should see the match, if the search was
+        // successful.
+        cy.contains(`spaceroni-macaroni`)
+      })
+    })
+  })
+
+  it("highlights the search word when a new search is started", () => {
+    cy.visit("/")
+    startNeovimWithGitBackend({}).then((nvim) => {
+      // wait until text on the start screen is visible
+      cy.contains("If you see this text, Neovim is ready!")
+      createGitReposToLimitSearchScope()
+
+      // clear the current line and enter insert mode
+      cy.typeIntoTerminal("cc")
+
+      // debug mode should be on by default for all tests. Otherwise it doesn't
+      // make sense to test this, as nothing will be displayed.
+      nvim.runLuaCode({
+        luaCode: `assert(require("blink-ripgrep").config.debug)`,
+      })
+
+      // this will match text from ../../../test-environment/other-file.lua
+      //
+      // If the plugin works, this text should show up as a suggestion.
+      cy.typeIntoTerminal("hip")
+      // the search should have been started for the prefix "hip"
+      cy.contains("hip").should(
+        "have.css",
+        "backgroundColor",
+        rgbify(flavors.macchiato.colors.flamingo.rgb),
+      )
+      //
+      // blink is now in the Fuzzy(3) stage, and additional keypresses must not
+      // start a new ripgrep search. They must be used for filtering the
+      // results instead.
+      // https://cmp.saghen.dev/development/architecture.html#architecture
+      cy.contains("Hippopotamus" + "234 (rg)") // wait for blink to show up
+      cy.typeIntoTerminal("234")
+
+      // wait for the highlight to disappear to test that too
+      cy.contains("hip").should(
+        "have.css",
+        "backgroundColor",
+        rgbify(flavors.macchiato.colors.base.rgb),
+      )
+
+      nvim
+        .runLuaCode({
+          luaCode: `return _G.blink_ripgrep_invocations`,
+        })
+        .should((result) => {
+          // ripgrep should only have been invoked once
+          expect(result.value).to.be.an("array")
+          expect(result.value).to.have.length(1)
+        })
+    })
+  })
+
+  // TODO add a test for "can clean up (kill) a previous search". This is too
+  // fast and currently needs to be timing based (👎🏻).
+
+  afterEach(() => {
+    verifyGitGrepBackendWasUsedInTest()
+  })
 })
+
+function verifyGitGrepBackendWasUsedInTest() {
+  cy.nvim_runLuaCode({
+    luaCode: `return require("blink-ripgrep").config`,
+  }).then((result) => {
+    assert(result.value)
+    const config = z
+      .object({ future_features: z.object({ backend: z.string() }) })
+      .safeParse(result.value)
+    // eslint-disable-next-line @typescript-eslint/no-unused-expressions
+    expect(config.error).to.be.undefined
+    expect(config.data?.future_features.backend).to.equal("gitgrep")
+  })
+}