Implement Migemo target matching

1 files changed, 318 insertions, 1 deletions

diff --git a/tests/run.lua b/tests/run.lua
index 9034b97..6ca2327 100644
--- a/tests/run.lua
+++ b/tests/run.lua
@@ -10,6 +10,7 @@ local domain = require("clever_f.domain")
 local capabilities = require("clever_f.capabilities")
 local case_policy = require("clever_f.case_policy")
 local policy = require("clever_f.policy")
+local migemo_catalog = require("clever_f.migemo_catalog")
 local sequence_state = require("clever_f.sequence_state")
 local state_transitions = require("clever_f.state_transitions")
 local text_topology = require("clever_f.text_topology")
@@ -1819,6 +1820,322 @@ test("Target plan inputs enforce matching contracts", function()
   end, "preserve")
 end)
 
+local MIGEMO_HIRAGANA_A = "\227\129\130"
+local MIGEMO_KANJI = "\230\188\162"
+local MIGEMO_SHELTER = "\229\142\166"
+local MIGEMO_GATE = "\233\150\128"
+
+local function read_binary(path)
+  local handle, open_error = io.open(path, "rb")
+  if handle == nil then
+    error("could not open " .. path .. ": " .. tostring(open_error), 2)
+  end
+  local contents = handle:read("*a")
+  handle:close()
+  return contents
+end
+
+local function migemo_components(configuration)
+  local _, transitions = fresh_sequence_state()
+  local host = MemoryHost.new({ configuration = configuration })
+  local service = policy.new(host)
+  local catalog = migemo_catalog.new({
+    policy = service,
+    transitions = transitions,
+  })
+  local factory = target_plan.new({
+    policy = service,
+    migemo_catalog = catalog,
+  })
+  return host, service, catalog, factory, transitions
+end
+
+test("Bundled Migemo assets equal the normative source data", function()
+  local assets = {
+    {
+      encoding = "utf-8",
+      file = "utf8.vim",
+      hash = "fccd16452c105fcb7303d855b96d841f8174d37affa0c43d8b49a54e68289f3e",
+    },
+    {
+      encoding = "cp932",
+      file = "cp932.vim",
+      hash = "3823fda790f43876be78d53e116187f2e0eedc14d203756ea66a3814ca9707f1",
+    },
+    {
+      encoding = "euc-jp",
+      file = "eucjp.vim",
+      hash = "d6d7a2a795d7e3e14cdefd0cf2597bc48bd3d89dbdc41019f460027377855079",
+    },
+  }
+
+  for _, asset in ipairs(assets) do
+    local bundled_path = migemo_catalog.bundled_asset_path(asset.encoding)
+    local bundled = read_binary(bundled_path)
+    same(asset.hash, vim.fn.sha256(bundled), asset.file .. " checksum")
+  end
+end)
+
+test("Migemo dictionaries validate ordered keys and cache each encoding", function()
+  local state, transitions = fresh_sequence_state()
+  local catalog = migemo_catalog.new({ transitions = transitions })
+  local expected_keys = migemo_catalog.expected_keys()
+  same(52, #expected_keys)
+  same(
+    "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",
+    table.concat(expected_keys)
+  )
+
+  local encodings = {
+    { requested = "UTF8", canonical = "utf-8" },
+    { requested = "CP-932", canonical = "cp932" },
+    { requested = "EUCJP", canonical = "euc-jp" },
+  }
+  for _, item in ipairs(encodings) do
+    local first = catalog:get(item.requested)
+    local second = catalog:get(item.canonical)
+    same(first, second)
+    same(first, state:get_migemo(item.canonical))
+    same(item.canonical, first.encoding)
+    same(52, first.entry_count)
+    list_same(expected_keys, first:keys())
+    same(1, catalog:load_count(item.canonical))
+
+    for _, key in ipairs(expected_keys) do
+      same("function", type(first[key]))
+      truthy(first:has(key))
+      truthy(first:matches(key, key, domain.CaseMode.SENSITIVE))
+    end
+    fails(function()
+      first.encoding = "changed"
+    end, "immutable")
+  end
+  same(3, map_size(state.migemo_cache))
+end)
+
+test("Migemo catalog rejects wrong entry counts and key order", function()
+  local _, transitions = fresh_sequence_state()
+  local keys = migemo_catalog.expected_keys()
+  local data = {}
+  for _, key in ipairs(keys) do
+    data[key] = key
+  end
+  local compiler = function()
+    return function()
+      return false
+    end
+  end
+
+  local wrong_order = migemo_catalog.expected_keys()
+  wrong_order[1], wrong_order[2] = wrong_order[2], wrong_order[1]
+  local order_catalog = migemo_catalog.new({
+    transitions = transitions,
+    asset_loader = function()
+      return data, wrong_order, "ordered-test"
+    end,
+    pattern_compiler = compiler,
+  })
+  fails(function()
+    order_catalog:get("utf-8")
+  end, "ordered a through z, then A through Z")
+
+  data[keys[#keys]] = nil
+  local count_catalog = migemo_catalog.new({
+    transitions = transitions,
+    asset_loader = function()
+      return data, keys, "count-test"
+    end,
+    pattern_compiler = compiler,
+  })
+  fails(function()
+    count_catalog:get("utf-8")
+  end, "exactly 52 keys")
+end)
+
+test("Native Migemo predicates agree across a fixed encoding corpus", function()
+  local _, transitions = fresh_sequence_state()
+  local catalog = migemo_catalog.new({ transitions = transitions })
+  local corpus = {
+    { "a", "a", domain.CaseMode.SENSITIVE, true },
+    { "a", "A", domain.CaseMode.SENSITIVE, false },
+    { "a", "A", domain.CaseMode.INSENSITIVE, true },
+    { "a", MIGEMO_HIRAGANA_A, domain.CaseMode.SENSITIVE, true },
+    { "k", MIGEMO_KANJI, domain.CaseMode.SENSITIVE, true },
+    {
+      "a",
+      MIGEMO_SHELTER .. "\n" .. MIGEMO_GATE,
+      domain.CaseMode.SENSITIVE,
+      true,
+    },
+    { "a", "x" .. MIGEMO_HIRAGANA_A, domain.CaseMode.SENSITIVE, false },
+  }
+
+  local saved_ignorecase = vim.o.ignorecase
+  local saved_smartcase = vim.o.smartcase
+  vim.o.ignorecase = true
+  vim.o.smartcase = true
+  for _, encoding in ipairs(migemo_catalog.supported_encodings()) do
+    local dictionary = catalog:get(encoding)
+    for index, item in ipairs(corpus) do
+      same(
+        item[4],
+        dictionary:matches(item[1], item[2], item[3]),
+        encoding .. " corpus item " .. tostring(index)
+      )
+    end
+  end
+  vim.o.ignorecase = saved_ignorecase
+  vim.o.smartcase = saved_smartcase
+end)
+
+test("Migemo activation uses live policy, scope, and branch precedence", function()
+  local host, _, _, factory = migemo_components({
+    use_migemo = true,
+    search_current_line_only = false,
+    chars_match_any_signs = "a;",
+  })
+  local ascii_view = text_topology.new({ "xax" }, "utf-8")
+  local context = {
+    text_view = ascii_view,
+    origin = domain.Position.new(1, 1),
+  }
+
+  local migemo = factory:build(target("a"), nil, context)
+  same(domain.TargetPlanKind.MIGEMO, migemo.kind)
+
+  host:set_configuration("use_migemo", false)
+  local symbol = factory:build(target("a"), nil, context)
+  same(domain.TargetPlanKind.SYMBOL, symbol.kind)
+
+  host:set_configuration("use_migemo", true)
+  same(domain.TargetPlanKind.SYMBOL, factory:build(target(";"), nil, context).kind)
+  same(
+    domain.TargetPlanKind.LITERAL,
+    factory:build(target(MIGEMO_HIRAGANA_A, 0x3042), nil, context).kind
+  )
+  same(domain.TargetPlanKind.LITERAL, factory:build(target("ab", 97), nil, context).kind)
+
+  host:set_configuration("chars_match_any_signs", "")
+  host:set_configuration("search_current_line_only", true)
+  local ascii_line = factory:build(target("a"), nil, context)
+  same(domain.TargetPlanKind.LITERAL, ascii_line.kind)
+
+  local multibyte_view = text_topology.new({ "x" .. MIGEMO_HIRAGANA_A }, "utf-8")
+  local multibyte_line = factory:build(target("a"), nil, {
+    text_view = multibyte_view,
+    origin = domain.Position.new(1, 1),
+  })
+  same(domain.TargetPlanKind.MIGEMO, multibyte_line.kind)
+end)
+
+test("Migemo plans enforce case and one-character candidate constraints", function()
+  local host, _, catalog, factory = migemo_components({
+    use_migemo = true,
+    ignore_case = false,
+    smart_case = false,
+  })
+  local view = text_topology.new({
+    "aA" .. MIGEMO_HIRAGANA_A .. "Es",
+  }, "utf-8")
+  local context = {
+    text_view = view,
+    search_scope = domain.SearchScope.BUFFER,
+  }
+  local sensitive = factory:build(target("a"), nil, context)
+  same(domain.CaseMode.SENSITIVE, sensitive.case_mode)
+  truthy(sensitive:matches_at(view, view:position_for_character_index(1, 1)))
+  falsy(sensitive:matches_at(view, view:position_for_character_index(1, 2)))
+  truthy(sensitive:matches_at(view, view:position_for_character_index(1, 3)))
+
+  local other_letter = view:position_for_character_index(1, 4)
+  truthy(catalog:get("utf-8"):matches(
+    "a",
+    view:text_suffix(other_letter),
+    domain.CaseMode.SENSITIVE
+  ))
+  falsy(sensitive:matches_at(view, other_letter))
+  falsy(sensitive:matches("x", view:position_for_character_index(1, 1), view))
+
+  host:set_configuration("ignore_case", true)
+  local insensitive = factory:build(target("a"), nil, context)
+  same(domain.CaseMode.INSENSITIVE, insensitive.case_mode)
+  truthy(insensitive:matches_at(view, view:position_for_character_index(1, 2)))
+end)
+
+test("Migemo assertions inspect later lines while limiting candidate starts", function()
+  local _, _, _, factory = migemo_components({
+    use_migemo = true,
+    search_current_line_only = true,
+  })
+  local view = text_topology.new({
+    "x" .. MIGEMO_SHELTER,
+    "",
+    MIGEMO_GATE,
+    MIGEMO_HIRAGANA_A,
+  }, "utf-8")
+  local candidate = view:position_for_character_index(1, 2)
+  local plan = factory:build(target("a"), nil, {
+    text_view = view,
+    origin = domain.Position.new(1, 1),
+  })
+
+  same(domain.TargetPlanKind.MIGEMO, plan.kind)
+  same(
+    MIGEMO_SHELTER .. "\n\n" .. MIGEMO_GATE
+      .. "\n" .. MIGEMO_HIRAGANA_A,
+    view:text_suffix(candidate)
+  )
+  truthy(plan:matches_at(view, candidate))
+  local matched_start = plan:matches_at(view, candidate) and candidate or nil
+  same(candidate, matched_start)
+  falsy(plan:matches_at(view, domain.Position.new(4, 1)))
+end)
+
+test("Unsupported Migemo encoding disables live policy and raises exactly", function()
+  local host, service, _, factory = migemo_components({
+    use_migemo = true,
+    search_current_line_only = false,
+  })
+  local view = text_topology.new({ "a" }, "latin1")
+  local ok, diagnostic = pcall(function()
+    factory:build(target("a"), nil, {
+      text_view = view,
+      search_scope = domain.SearchScope.BUFFER,
+    })
+  end)
+
+  falsy(ok)
+  same(
+    "clever-f: Encoding 'latin1' is not supported. Migemo is disabled",
+    diagnostic
+  )
+  falsy(service:get_boolean("use_migemo"))
+  same(false, host:read_configuration("use_migemo"))
+  same(0, map_size(sequence_state.get().migemo_cache))
+
+  local literal = factory:build(target("a"), nil, {
+    text_view = view,
+    search_scope = domain.SearchScope.BUFFER,
+  })
+  same(domain.TargetPlanKind.LITERAL, literal.kind)
+end)
+
+test("Public Reset discards loaded Migemo dictionary objects", function()
+  local state, transitions = fresh_sequence_state()
+  local catalog = migemo_catalog.new({ transitions = transitions })
+  local first = catalog:get("utf-8")
+  same(first, state:get_migemo("utf-8"))
+
+  transitions:PublicReset()
+  same(nil, state:get_migemo("utf-8"))
+  same(nil, catalog:cached("utf-8"))
+
+  local second = catalog:get("utf-8")
+  falsy(first == second)
+  same(second, state:get_migemo("utf-8"))
+  same(2, catalog:load_count("utf-8"))
+end)
+
 for _, item in ipairs(tests) do
   local ok, failure = xpcall(item.body, debug.traceback)
   if not ok then
@@ -1828,4 +2145,4 @@ for _, item in ipairs(tests) do
   passed = passed + 1
 end
 
-io.stdout:write(string.format("Phase 6: %d tests passed\n", passed))
+io.stdout:write(string.format("Phase 7: %d tests passed\n", passed))