"""
SQLite FTS5 unified search index for Accordance Port.

Stores at ~/.accordance/search_index.db (falls back to project data/ if
home is not writable).  The index file may reach 400--700 MB after a
full build -- be aware but do not block.

Schema
------
search_content  -- source metadata + decoded text (rowid, source_type,
                   source_name, source_id, language, citation,
                   body_text, body_norm, book)
search_idx      -- FTS5 virtual table referencing search_content by rowid
index_meta      -- tracks per-item file mtimes for incremental updates

Concurrency
-----------
Uses WAL journal mode + check_same_thread=False.  Concurrent reads during
a rebuild are safe (SQLite snapshot isolation), but may return partial
results for the source being rebuilt.  Acceptable for a single-user local
tool.

Granularity
-----------
Commentary sections are indexed at the existing section level (typically
pericope or verse-range).  Per-verse sub-indexing is a future enhancement
that would require natural-language boundary detection within prose.
"""

import json
import logging
import os
import re
import sqlite3
import sys
import time
import unicodedata

# itertools.product formerly used by transliteration code (moved to accordance_parser/transliteration.py)
from pathlib import Path

# Prefer pysqlite3-binary (bundled SQLite with high MAX_MMAP_SIZE) for full mmap support
# on macOS where system SQLite caps mmap at ~2GB. Falls back silently to stdlib sqlite3.
try:
    import pysqlite3 as sqlite3  # type: ignore
except ImportError:
    pass  # use stdlib sqlite3 (mmap capped on macOS)

from accordance_parser.citation_format import lexicon_citation, commentary_citation
from accordance_parser.encoding import decode
from accordance_parser.canon import resolve_book_id
from accordance_parser.corpus_registry import (
    detect_corpus,
    extract_work_ref,
    format_ref as format_corpus_ref,
    CORPUS_REGISTRY,
)
from accordance_parser.transliteration import (
    Transliterator,
    reverse_transliterate_greek,
    reverse_transliterate_hebrew,
    reverse_transliterate_greek_variants,
)

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Path helpers
# ---------------------------------------------------------------------------

_HOME_DB = Path.home() / ".accordance" / "search_index.db"
_PROJ_DB = Path(__file__).resolve().parent.parent / "data" / "search_index.db"
_JSON_DIR = Path(__file__).resolve().parent.parent / "data"

# When set, skip expensive index operations (FTS5 prefix rebuild, build_all).
# Used by CI/test suites to keep pytest collection under 60 s.
_SKIP_INDEX_BUILD = bool(os.environ.get("ACCORDANCE_SKIP_INDEX_BUILD"))


def _default_db_path() -> str:
    """Return the best available path for the search index database."""
    if _HOME_DB.parent.exists() or _try_makedirs(_HOME_DB.parent):
        return str(_HOME_DB)
    _try_makedirs(_PROJ_DB.parent)
    return str(_PROJ_DB)


def _try_makedirs(p: Path) -> bool:
    try:
        p.mkdir(parents=True, exist_ok=True)
        return True
    except (OSError, PermissionError):
        return False


# ---------------------------------------------------------------------------
# Normalization utilities
# ---------------------------------------------------------------------------

_STRONGS_RE = None  # compiled lazily

# C0 control characters: NULL through BS (0x00-0x08), VT (0x0B), FF (0x0C),
# SO through US (0x0E-0x1F).  Excludes HT (0x09), LF (0x0A), CR (0x0D).
_C0_RE = re.compile(r"[\x00-\x08\x0B\x0C\x0E-\x1F]")


def _get_strongs_re():
    global _STRONGS_RE
    if _STRONGS_RE is None:
        import re

        _STRONGS_RE = re.compile(r"<S>\d+</S>")
    return _STRONGS_RE


def strip_strongs_tags(text: str) -> str:
    """Remove <S>NNNN</S> patterns from text."""
    return _get_strongs_re().sub("", text).strip()


def strip_c0_controls(text: str) -> str:
    """Strip C0 control characters (0x00-0x08, 0x0B-0x0C, 0x0E-0x1F) from text.

    These are non-printable bytes that leak into the index from module name
    citations with trailing binary garbage in source metadata (titles, etc.).
    Excludes HT (0x09), LF (0x0A), CR (0x0D) which are legitimate whitespace.
    """
    return _C0_RE.sub("", text)


def strip_diacritics(text: str) -> str:
    """Strip Greek/Hebrew combining marks via NFD decomposition.

    Also normalizes final sigma (ς) → regular sigma (σ) for consistent
    searching across Greek text sources.

    Examples:
        'λόγος' → 'λογοσ'
        'בְּרֵאשִׁית' → 'בראשית'
    """
    nfd = unicodedata.normalize("NFD", text)
    return "".join("σ" if c == "ς" else c for c in nfd if not unicodedata.combining(c))


# Unicode ranges for language detection
_GREEK_RANGE = (0x0370, 0x03FF)
_GREEK_EXT = (0x1F00, 0x1FFF)  # Greek extended (polytonic)
_HEBREW_RANGE = (0x0590, 0x05FF)

# CJK character ranges for tokenization fix.
# unicode61 treats consecutive "token" characters as a single token,
# which breaks CJK search (e.g. "起初神创造天地" is one token).
# Fix: insert spaces between CJK characters so each becomes its own FTS5 token.
# Ranges: CJK Unified Ideographs + Ext A + Compatibility + Supplement (SIP).
_CJK_RANGES: list[tuple[int, int]] = [
    (0x4E00, 0x9FFF),  # CJK Unified Ideographs
    (0x3400, 0x4DBF),  # CJK Unified Ideographs Extension A
    (0xF900, 0xFAFF),  # CJK Compatibility Ideographs
    (0x20000, 0x2A6DF),  # CJK Unified Ideographs Extension B
    (0x2A700, 0x2B73F),  # CJK Unified Ideographs Extension C
    (0x2B740, 0x2B81F),  # CJK Unified Ideographs Extension D
    (0x2B820, 0x2CEAF),  # CJK Unified Ideographs Extension E
    (0x2CEB0, 0x2EBEF),  # CJK Unified Ideographs Extension F
]

# Precompiled regex: matches any CJK character
_CJK_CHAR_RE: re.Pattern | None = None


def _get_cjk_char_re() -> re.Pattern:
    """Return a compiled regex matching any single CJK character."""
    global _CJK_CHAR_RE
    if _CJK_CHAR_RE is None:
        # Build a regex that matches one CJK char from any of the ranges
        parts = []
        for lo, hi in _CJK_RANGES:
            if lo <= 0xFFFF and hi <= 0xFFFF:
                parts.append(f"\\u{lo:04X}-\\u{hi:04X}")
            else:
                # Supplementary plane: use Python's \U escape
                # (this covers Ext B-F in the SIP)
                parts.append(f"\\U{lo:08X}-\\U{hi:08X}")
        _CJK_CHAR_RE = re.compile("[" + "".join(parts) + "]")
    return _CJK_CHAR_RE


def _is_cjk_char(ch: str) -> bool:
    """Return True if *ch* is a single CJK character."""
    return _get_cjk_char_re().match(ch) is not None


def separate_cjk_chars(text: str) -> str:
    """Insert a space between consecutive CJK characters for FTS5 tokenization.

    unicode61 treats consecutive letters as one token.  In Chinese text
    without whitespace between words, this means entire phrases become
    single tokens and individual-character search fails.  This function
    inserts a space between consecutive CJK characters so each becomes
    a separate FTS5 token while preserving existing whitespace/punctuation.

    Example:
        '起初神创造天地' → '起 初 神 创 造 天 地'
        '神说：要有光'   → '神 说 ： 要 有 光'
    """
    if not text:
        return text
    cjk_re = _get_cjk_char_re()
    result: list[str] = []
    prev_is_cjk = False
    for ch in text:
        is_cjk = cjk_re.match(ch) is not None
        if is_cjk and prev_is_cjk:
            result.append(" ")
        result.append(ch)
        prev_is_cjk = is_cjk
    return "".join(result)


def has_cjk_chars(text: str) -> bool:
    """Return True if *text* contains any CJK characters."""
    return _get_cjk_char_re().search(text) is not None


def _count_chars_in_ranges(text: str, ranges: list[tuple[int, int]]) -> int:
    """Count characters in text that fall within the given Unicode ranges."""
    return sum(1 for c in text if any(lo <= ord(c) <= hi for lo, hi in ranges))


def detect_language(text: str) -> str:
    """Heuristic language detection for a text string.

    Returns 'el' if >50% of chars are Greek, 'he' if >50% are Hebrew,
    'zh' if >30% are CJK, otherwise 'en'.
    """
    if not text:
        return "en"
    text_chars = [c for c in text if not c.isspace()]
    if not text_chars:
        return "en"
    total = len(text_chars)
    greek_count = _count_chars_in_ranges(text, [_GREEK_RANGE, _GREEK_EXT])
    hebrew_count = _count_chars_in_ranges(text, [_HEBREW_RANGE])
    cjk_count = _count_chars_in_ranges(text, _CJK_RANGES)

    if greek_count > total * 0.5:
        return "el"
    if hebrew_count > total * 0.5:
        return "he"
    if cjk_count > total * 0.3:
        return "zh"
    return "en"


# ---------------------------------------------------------------------------
# Section heading extraction (for commentary citations)
# ---------------------------------------------------------------------------

# Regex patterns for extracting section titles from commentary first lines.
# Matches patterns like:
#   "Genesis 1:1, 2. THE CREATION OF HEAVEN AND EARTH."
#   "Matthew 3:5-12. The Preaching of John the Baptist"
#   "1Corinthians 15:1. The Resurrection of Christ"
_SECTION_HEADING_RE = re.compile(
    r"^[A-Z][a-zA-Z]*(?:\s+[A-Z][a-zA-Z]*)*\s+\d+:\d+(?:[-\d,\s]+)?[.;]\s+(.+)$"
)

# Secondary pattern for Hermeneia-style headings (just a dash-separated title)
_SECTION_HEADING_ALT_RE = re.compile(
    r"^[A-Z][a-zA-Z]*(?:\s+[A-Z][a-zA-Z]*)*\s+\d+:\d+-\d+\s+(.+)$"
)

# Format C: All-caps abbreviation (Chrysostom)
#   "ROM. 1:1, 2. — \"Paul, a servant of Jesus Christ...\""
_SECTION_HEADING_CHRYSOSTOM_RE = re.compile(r"^[A-Z]{2,}\.\s+\d+:\d+.*?[-–—]+\s*(.+)$")

# Format B: Bare ref + title (IVP-NB Commentary)
#   "1:1-2:3    Prologue: God creates the world"
_SECTION_HEADING_BARE_REF_RE = re.compile(r"^\d+:\d+(?:[-\d,:]+)?\s{2,}(.+)$")

# Format A: Dot-notation (Hermeneia)
#   "1.1 the Baptist's Call to Israel to Repent (3:112)"
_SECTION_HEADING_DOT_RE = re.compile(r"^\d+\.\d+(?:[-.]\d+)?\s+(.+)$")


def _extract_section_heading(text: str) -> str | None:
    """Extract section heading from the first line of commentary text.

    Returns the heading string if the first line matches a known
    section-header pattern, or None otherwise.
    """
    if not text:
        return None
    first_line = text.split("\n", 1)[0].strip()
    if not first_line:
        return None

    # Fix 5: Skip binary garbage preamble (control chars, non-printable bytes)
    garbage_chars = sum(1 for c in first_line if ord(c) < 0x20 and c not in "\t\r\n")
    if len(first_line) > 0 and garbage_chars > len(first_line) * 0.5:
        rest = text.split("\n", 1)[1] if "\n" in text else ""
        if rest.strip():
            first_line = rest.split("\n")[0].strip()
    if not first_line:
        return None

    # Try primary pattern: "Book ch:v[s]. TITLE"
    m = _SECTION_HEADING_RE.match(first_line)
    if m:
        heading = m.group(1).strip().rstrip(".")
        if heading and len(heading) >= 2:
            return heading

    # Try alternate pattern: "Book ch:v-v TITLE"
    m = _SECTION_HEADING_ALT_RE.match(first_line)
    if m:
        heading = m.group(1).strip().rstrip(".")
        if heading and len(heading) >= 2:
            return heading

    # Format C: All-caps abbreviation (Chrysostom)
    m = _SECTION_HEADING_CHRYSOSTOM_RE.match(first_line)
    if m:
        heading = m.group(1).strip()
        if heading and len(heading) >= 2:
            return heading[:100]

    # Format B: Bare ref + title (IVP-NB Commentary)
    m = _SECTION_HEADING_BARE_REF_RE.match(first_line)
    if m:
        heading = m.group(1).strip()
        if heading and len(heading) >= 2:
            return heading

    # Format A: Dot-notation (Hermeneia)
    m = _SECTION_HEADING_DOT_RE.match(first_line)
    if m:
        heading = m.group(1).strip()
        if heading and len(heading) >= 2:
            return heading

    return None


# ---------------------------------------------------------------------------
# English stemming (lightweight suffix-stripping, no NLTK dependency)
# ---------------------------------------------------------------------------

# Simple English suffix-stripping stemmer for search normalization.
# Strips common suffixes so that word variants map to the same root
# (e.g. "baptism" ↔ "baptize" ↔ "baptizing" all → "bapt").
# Applied to body_norm for English-language documents during indexing.
# ---------------------------------------------------------------------------
# Hermeneia book name → (corpus_id, work_abbrev) mapping for non-Bible
# commentary modules (AF, Philo, Q).  The Hermeneia parser produces
# spaceless book names from HERMENEIA_TITLE_MAP; this maps them to
# corpus work abbreviations for proper citation formatting.
# ---------------------------------------------------------------------------

_HERMENEIA_BOOK_TO_CORPUS_WORK: dict[str, tuple[str, str]] = {
    # Apostolic Fathers
    "IgnatiusEphesians": ("apostolic_fathers", "IgnEph"),
    "IgnatiusMagnesians": ("apostolic_fathers", "IgnMagn"),
    "IgnatiusTrallians": ("apostolic_fathers", "IgnTr"),
    "IgnatiusRomans": ("apostolic_fathers", "IgnRom"),
    "IgnatiusPhiladelphians": ("apostolic_fathers", "IgnPhld"),
    "IgnatiusSmyrnaeans": ("apostolic_fathers", "IgnSm"),
    "IgnatiusPolycarp": ("apostolic_fathers", "IgnPol"),
    "ShepherdofHermas": ("apostolic_fathers", "Herm"),
    "Didache": ("apostolic_fathers", "Did"),
    "1Clement": ("apostolic_fathers", "1Clem"),
    "2Clement": ("apostolic_fathers", "2Clem"),
    "EpistleofBarnabas": ("apostolic_fathers", "Barn"),
    "MartyrdomofPolycarp": ("apostolic_fathers", "MartPol"),
    "EpistletoDiognetus": ("apostolic_fathers", "Diogn"),
    "IgnatiusofAntioch": ("apostolic_fathers", "Ign"),
    # Q source
    "QSource": ("q_source", "Q"),
}

# ---------------------------------------------------------------------------
# English stemming (lightweight suffix-stripping, no NLTK dependency)
# ---------------------------------------------------------------------------

_ENGLISH_SUFFIXES: list[str] = [
    "ization",
    "isation",  # organize → organiz (longer forms first)
    "fulness",
    "lessness",
    "ability",
    "ibility",
    "alize",
    "alise",
    "ment",
    "tion",
    "sion",
    "ness",
    "ful",
    "less",
    "able",
    "ible",
    "ive",
    "ing",
    "ed",
    "es",
    "ly",
    "er",
    "est",
    "s",  # shortest, last
]

# Minimum stem length to avoid over-truncation
_MIN_STEM_LEN = 3


def _stem_english(text: str) -> str:
    """Apply simple suffix-stripping stemming to English words in *text*.

    Only stems words with >=4 characters so short words like "God", "sin",
    "law" are preserved.  Returns the text with each eligible word stemmed.
    """
    words = text.split()
    result: list[str] = []
    for word in words:
        lower = word.lower()
        # Only stem words of length >= 4 to preserve short terms
        if len(lower) < 4:
            result.append(lower)
            continue
        stemmed = lower
        for suffix in _ENGLISH_SUFFIXES:
            new_len = len(lower) - len(suffix)
            if lower.endswith(suffix) and new_len >= _MIN_STEM_LEN:
                stemmed = lower[:new_len]
                break
        result.append(stemmed)
    return " ".join(result)


# ---------------------------------------------------------------------------
# Transliteration support (Greek / Hebrew ↔ ASCII Latin)
# ---------------------------------------------------------------------------
# All transliteration logic is now in accordance_parser.transliteration.
# Re-export the original names for backward compatibility.
# ---------------------------------------------------------------------------

# Backward-compatible aliases for internal callers that import from search_index
_is_likely_transliteration = Transliterator.classify
_reverse_transliterate_greek_variants = reverse_transliterate_greek_variants
_reverse_transliterate_greek = reverse_transliterate_greek
_reverse_transliterate_hebrew = reverse_transliterate_hebrew


# ---------------------------------------------------------------------------
# SearchIndex
# ---------------------------------------------------------------------------

# JSON → module-abbreviation mapping for Bible text files
_JSON_TO_ABBREV = {
    "kjv": "KJVS",
    "esv": "ESVS",
    "nabre": "NABRE",
    "nasb": "NAS95S",
    "lxx": "LXX1",
    "wlc": "MT-ETCBC-A",
    "morphhb": "MORPHHB",
    "opengnt": "OPENGNT",
    "tisch": "TISCH",
    "tr": "TR",
}

# Generic-format modules that are reference indices, not prose.
# Excluded from FTS5 indexing (still loaded for reference lookups).
_GENERIC_NOISE_ABBREVS = {
    "Guide to NT Vocab",
    "Guide to LXX Vocab",
    "Guide to BH Vocab",
    "People & Places of Bible",
}

# Modules whose binary parser returns unreliable verse text (raw binary
# garbage in the decoded output).  Skip these during Bible indexing — their
# verse text is routed to bolls.life API for runtime lookups instead.
#
# Includes:
#   - Binary parser broken, bolls.life API used at runtime (NAS95S, NET,
#     HMT-DEMO, MT-ETCBC-A, NASB)
#   - No bolls.life equivalent available (CNT)
#   - Parallel-text layout variants whose base module is already indexed
#     (CEB2, NETS2, NRSVue2, LXX2)
#   - Apparatus/corpus format modules that are NOT Bible texts
#     (AF-E/L/T, AFL-E/T, APOL-E/T, ORIGEN-E/T)
_BYPASS_INDEX = frozenset(
    {
        # Binary parser broken — routed to bolls.life API at runtime.
        "NET",
        "HMT-DEMO",
        "MT-ETCBC-A",
        # No bolls.life equivalent
        "CNT",
        # Modules removed from _BOLLS_FALLBACK 2026-06-10 — user preference
        "ASV",
        "ELBER05",
        "WEB",
        "LBLA",
        "LSV",
        "MESSAGE",
        "NAS95S",
        "RVR09S",
        "DOUAY",
        "CEB",
        "NASB",
        "NETS",
        # No bolls.life equivalent — binary parser unreliable
        "NBLH",
        "BBE",
        "VDC2014",
        # Not needed for indexing — user preference
        "CUVS-SIMP",
        "CUVS-TRAD",
        # Parallel-text layout variants (base module already indexed)
        "CEB2",
        "NETS2",
        "NRSVue2",
        "LXX2",
        # Apparatus/corpus format — not Bible texts
        "AF-E",
        "AF-L",
        "AF-T",
        "AFL-E",
        "AFL-T",
        "APOL-E",
        "APOL-T",
        "ORIGEN-E",
        "ORIGEN-T",
    }
)

# Mapping from TextModule abbreviation -> bolls.life translation code.
# Used as fallback when the binary parser returns None for all verses.
# Codes verified against https://bolls.life/get-verse/<CODE>/1/1/1/
_BOLLS_FALLBACK: dict[str, str] = {
    # Original-language and scholarly texts — keep in index
    "NABRE": "NABRE",  # NABRE Catholic — binary parser drifts, use API
    "KJVA": "KJV",
    "NRSVUES": "NRSVCE",  # NRSV Catholic Edition (closest; abbrev is "NRSVueS")
    "LXX1": "LXX",  # Septuagint
    "LXX_INFLECTED": "LXX",  # Septuagint (morphologically tagged)
    "GNT-TRS": "TR",  # Textus Receptus (NT only)
    "GNT28-T": "TISCH",  # Tischendorf (closest critical text)
    "NA28-T": "TISCH",  # Nestle-Aland 28 (closest: Tischendorf)
}

# Canonical Bible book names for detecting commentary vs generic module format.
# A book name that starts with one of these is a commentary; otherwise generic.
_BIBLE_BOOK_NAMES = (
    "Genesis",
    "Exodus",
    "Leviticus",
    "Numbers",
    "Deuteronomy",
    "Joshua",
    "Judges",
    "Ruth",
    "1Samuel",
    "2Samuel",
    "1Kings",
    "2Kings",
    "1Chronicles",
    "2Chronicles",
    "Ezra",
    "Nehemiah",
    "Esther",
    "Job",
    "Psalms",
    "Proverbs",
    "Ecclesiastes",
    "SongofSolomon",
    "Isaiah",
    "Jeremiah",
    "Lamentations",
    "Ezekiel",
    "Daniel",
    "Hosea",
    "Joel",
    "Amos",
    "Obadiah",
    "Jonah",
    "Micah",
    "Nahum",
    "Habakkuk",
    "Zephaniah",
    "Haggai",
    "Zechariah",
    "Malachi",
    # Deuterocanonical / Apocrypha
    "Tobit",
    "Judith",
    "1Maccabees",
    "2Maccabees",
    "Wisdom",
    "Sirach",
    "Baruch",
    # Greek Additions to Daniel / Esther
    "Susanna",
    "Bel",
    "PrAzar",
    "AddEsth",
    # New Testament
    "Matthew",
    "Mark",
    "Luke",
    "John",
    "Acts",
    "Romans",
    "1Corinthians",
    "2Corinthians",
    "Galatians",
    "Ephesians",
    "Philippians",
    "Colossians",
    "1Thessalonians",
    "2Thessalonians",
    "1Timothy",
    "2Timothy",
    "Titus",
    "Philemon",
    "Hebrews",
    "James",
    "1Peter",
    "2Peter",
    "1John",
    "2John",
    "3John",
    "Jude",
    "Revelation",
)


class SearchIndex:
    """SQLite FTS5 unified search index over Bibles, lexicons, and commentaries."""

    # Batch-insert size (rows per transaction)
    _BATCH_SIZE = 500

    # Source-type ranking: lower multiplier = higher priority in results.
    # Bible verses rank highest, then lexicons, commentaries, apparatus, acc2.
    # Applied multiplicatively with _SOURCE_QUALITY_BOOST in _apply_quality_boost().
    _SOURCE_TYPE_BOOST: dict[str, float] = {
        "bible": 0.8,
        "lexicon": 0.9,
        "commentary": 1.0,
        "apparatus": 1.15,
        "apparatus_verse": 1.2,
        "acc2": 1.3,
    }

    # Citation-aware ranking: lower multiplier = better source (promote scholarly)
    # Applied as effective_rank = raw_rank * boost. Default 1.0 for unlisted.
    _SOURCE_QUALITY_BOOST = {
        # Top-tier scholarly lexicons
        "BDAG": 0.8,
        "HALOT": 0.8,
        "Liddell & Scott": 0.85,
        "BDB Complete": 0.85,
        "TDNT Complete": 0.8,
        "TDOT": 0.8,
        "Louw & Nida": 0.85,
        # Mid-tier
        "Thayer": 1.0,
        "Mounce Greek Dictionary": 1.0,
        "MGS": 1.0,
        "Greek Strong's": 1.1,
        "Hebrew Strong's": 1.1,
        "NAS Greek": 1.0,
        "NAS Hebrew": 1.0,
        # Lower-tier (study bibles, topical indices)
        "EDG": 1.0,
        "KM Hebrew Dictionary": 1.0,
        "Jenni-Westermann": 1.0,
        "Trench-Synonyms": 1.0,
        "Spicq": 1.0,
        # Generic tools
        "Easton": 1.3,
        "ISBE": 1.2,
        "Nave's": 1.4,
        "Names": 1.5,
        "Place Names": 1.5,
        "Sites": 1.5,
        "Bible Words": 1.3,
        "NAS Topical Index": 1.4,
        "NI Bible Dictionary": 1.0,
    }

    def __init__(self, db_path: str | None = None):
        self.db_path = db_path or _default_db_path()
        self._conn: sqlite3.Connection | None = None
        self._ensure_schema()

    # ------------------------------------------------------------------
    # Connection + schema
    # ------------------------------------------------------------------

    def _db_file_size(self) -> int:
        """Return the current DB file size in bytes, or 0 if it doesn't exist."""
        try:
            return os.path.getsize(self.db_path)
        except OSError:
            return 0

    def _ensure_conn(self) -> sqlite3.Connection:
        if self._conn is not None:
            return self._conn
        self._conn = sqlite3.connect(self.db_path, check_same_thread=False)
        self._conn.execute("PRAGMA journal_mode=WAL")
        self._conn.execute("PRAGMA synchronous=NORMAL")

        # --- Performance pragmas scaled to DB size ---
        db_size = self._db_file_size()

        # mmap_size: direct memory-mapped I/O, avoids syscalls per read.
        # Use max(2GB, db_size + 1GB) to keep the entire DB mapped plus some
        # headroom for WAL growth.  Note: SQLite silently caps mmap at its
        # compile-time MAX_MMAP_SIZE (commonly 2GB).  Even the floor of 2GB
        # is a massive improvement over the default of 0 (no mmap).
        mmap_bytes = max(2 * 1024**3, db_size + 1024**3)
        self._conn.execute(f"PRAGMA mmap_size={mmap_bytes}")

        # cache_size: negative means KB.  ~40% of DB size, floor 200 MB, ceiling 4 GB.
        cache_kb = int(db_size * 0.4 / 1024)
        cache_kb = max(200000, min(cache_kb, 4000000))
        self._conn.execute(f"PRAGMA cache_size=-{cache_kb}")

        # Auto-checkpoint every 1000 pages to prevent WAL bloat.
        self._conn.execute("PRAGMA wal_autocheckpoint=1000")

        return self._conn

    def _bulk_mode(self):
        """Enable fast, unsafe pragmas for bulk inserts. Call _safe_mode after."""
        conn = self._ensure_conn()
        conn.execute("PRAGMA synchronous=OFF")
        conn.execute("PRAGMA journal_mode=OFF")
        return conn

    def _safe_mode(self):
        """Restore safe pragmas after bulk inserts."""
        conn = self._ensure_conn()
        conn.execute("PRAGMA synchronous=NORMAL")
        conn.execute("PRAGMA journal_mode=WAL")
        return conn

    def checkpoint(self):
        """Truncate the WAL file by forcing a full checkpoint.

        Call after bulk rebuilds to keep the WAL from growing large.
        Safe to call at any time; ignored if no connection exists.
        """
        if self._conn is not None:
            self._conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")

    def _ensure_schema(self):
        conn = self._ensure_conn()
        conn.executescript("""
            CREATE TABLE IF NOT EXISTS search_content (
                rowid INTEGER PRIMARY KEY,
                source_type TEXT NOT NULL,
                source_name TEXT NOT NULL,
                source_id TEXT NOT NULL,
                language TEXT NOT NULL,
                body_text TEXT NOT NULL,
                body_norm TEXT NOT NULL
            );

            CREATE VIRTUAL TABLE IF NOT EXISTS search_idx USING fts5(
                body_text,
                body_norm,
                content='search_content',
                content_rowid='rowid',
                tokenize='unicode61 remove_diacritics 0',
                prefix='2 3'
            );

            CREATE TABLE IF NOT EXISTS index_meta (
                source_type TEXT NOT NULL,
                source_name TEXT NOT NULL,
                source_id TEXT NOT NULL,
                file_mtime REAL,
                indexed_at REAL,
                PRIMARY KEY (source_type, source_name, source_id)
            );
        """)

        # Migrate: add citation column if not already present
        cur = conn.execute("PRAGMA table_info(search_content)")
        cols = [row[1] for row in cur.fetchall()]
        if "citation" not in cols:
            conn.execute(
                "ALTER TABLE search_content ADD COLUMN citation TEXT NOT NULL DEFAULT ''"
            )
            conn.commit()

        # Migrate: add book column if not already present
        cur = conn.execute("PRAGMA table_info(search_content)")
        cols = [row[1] for row in cur.fetchall()]
        if "book" not in cols:
            conn.execute(
                "ALTER TABLE search_content ADD COLUMN book TEXT NOT NULL DEFAULT ''"
            )
            conn.commit()

        # Migrate: add corpus column if not already present
        if "corpus" not in cols:
            conn.execute(
                "ALTER TABLE search_content ADD COLUMN corpus TEXT NOT NULL DEFAULT ''"
            )
            conn.commit()

        # Migrate: add work column if not already present
        cur = conn.execute("PRAGMA table_info(search_content)")
        if "work" not in [row[1] for row in cur.fetchall()]:
            conn.execute(
                "ALTER TABLE search_content ADD COLUMN work TEXT NOT NULL DEFAULT ''"
            )
            conn.commit()

        # Migrate: check whether the FTS5 table has prefix indexes.
        # If not, prefix queries (trailing-* searches) will silently fail.
        # Auto-heal: DROP the FTS5 virtual table, recreate with prefix='2 3',
        # then rebuild from search_content (no data loss — FTS5 just indexes it).
        # Skip this expensive migration when ACCORDANCE_SKIP_INDEX_BUILD is set
        # (test suites / CI — the DB is disposable).
        if not _SKIP_INDEX_BUILD:
            fts_sql = conn.execute(
                "SELECT sql FROM sqlite_master WHERE type='table' AND name='search_idx'"
            ).fetchone()
            if fts_sql and fts_sql[0] and "prefix=" not in fts_sql[0]:
                logger.warning(
                    "FTS5 table 'search_idx' does not have prefix indexes. "
                    "Dropping and recreating with prefix='2 3' ..."
                )
                import time as _time

                t_start = _time.time()
                try:
                    conn.execute("DROP TABLE IF EXISTS search_idx")
                    conn.execute("""
                        CREATE VIRTUAL TABLE search_idx USING fts5(
                            body_text,
                            body_norm,
                            content='search_content',
                            content_rowid='rowid',
                            tokenize='unicode61 remove_diacritics 0',
                            prefix='2 3'
                        )
                    """)
                    conn.execute("INSERT INTO search_idx(search_idx) VALUES('rebuild')")
                    conn.commit()
                    conn.execute("INSERT INTO search_idx(search_idx) VALUES('optimize')")
                    conn.commit()
                    t_elapsed = _time.time() - t_start
                    logger.info(
                        "FTS5 prefix index rebuild complete in %.1f seconds", t_elapsed
                    )
                    print(f"FTS5 prefix index rebuilt in {t_elapsed:.1f}s")
                except Exception as exc:
                    logger.warning(
                        "FTS5 prefix index rebuild failed: %s. "
                        "Prefix queries will not work until a force rebuild.",
                        exc,
                    )

    def rebuild_prefix_indexes(self) -> bool:
        """Force-rebuild FTS5 prefix indexes.

        If the FTS5 table already has prefix='2 3' in its schema, only a
        rebuild is needed (populates the internal prefix index structures).
        If the table lacks prefix indexes, it is dropped and recreated with
        prefix='2 3', then rebuilt from search_content.

        The entire operation is wrapped in a transaction so that partial
        failures (timeout, crash) roll back and the old index is preserved.

        Returns True on success, False if the rebuild failed.
        """
        conn = self._ensure_conn()
        fts_sql = conn.execute(
            "SELECT sql FROM sqlite_master WHERE type='table' AND name='search_idx'"
        ).fetchone()
        if not fts_sql or not fts_sql[0]:
            logger.warning(
                "FTS5 table 'search_idx' does not exist; nothing to rebuild."
            )
            return False

        has_prefix = "prefix=" in fts_sql[0]
        import time as _time

        t_start = _time.time()

        try:
            conn.execute("BEGIN")
            if has_prefix:
                logger.info(
                    "FTS5 table already has prefix indexes; rebuilding to "
                    "populate prefix index structures ..."
                )
                conn.execute("INSERT INTO search_idx(search_idx) VALUES('rebuild')")
            else:
                logger.warning(
                    "FTS5 table 'search_idx' lacks prefix indexes. "
                    "Dropping and recreating with prefix='2 3' ..."
                )
                conn.execute("DROP TABLE IF EXISTS search_idx")
                conn.execute("""
                    CREATE VIRTUAL TABLE search_idx USING fts5(
                        body_text,
                        body_norm,
                        content='search_content',
                        content_rowid='rowid',
                        tokenize='unicode61 remove_diacritics 0',
                        prefix='2 3'
                    )
                """)
                conn.execute("INSERT INTO search_idx(search_idx) VALUES('rebuild')")
            conn.execute("INSERT INTO search_idx(search_idx) VALUES('optimize')")
            conn.commit()
            t_elapsed = _time.time() - t_start
            logger.info("FTS5 prefix index rebuild complete in %.1f seconds", t_elapsed)
            print(f"FTS5 prefix indexes rebuilt in {t_elapsed:.1f}s")
            return True
        except Exception as exc:
            conn.rollback()
            logger.warning("FTS5 prefix index rebuild failed (rolled back): %s.", exc)
            return False

    def ensure_index(self) -> int:
        """Ensure the search index exists and return the current row count.

        If the index is empty, the schema is already initialized by
        _ensure_schema() at construction time.  Returns the current number of
        indexed rows so callers can decide whether to populate with build_all().
        """
        conn = self._ensure_conn()
        row = conn.execute("SELECT COUNT(*) FROM search_content").fetchone()
        return row[0] if row else 0

    def close(self):
        if self._conn:
            self._conn.close()
            self._conn = None

    # ------------------------------------------------------------------
    # Incremental helpers
    # ------------------------------------------------------------------

    def _is_indexed(self, source_type: str, source_name: str, source_id: str) -> bool:
        conn = self._ensure_conn()
        row = conn.execute(
            "SELECT 1 FROM index_meta WHERE source_type=? AND source_name=? AND source_id=?",
            (source_type, source_name, source_id),
        ).fetchone()
        return row is not None

    def _needs_reindex(
        self, source_type: str, source_name: str, file_mtime: float
    ) -> bool:
        """Return True if this source needs (re)indexing based on file mtime."""
        conn = self._ensure_conn()
        row = conn.execute(
            "SELECT MAX(file_mtime) FROM index_meta WHERE source_type=? AND source_name=?",
            (source_type, source_name),
        ).fetchone()
        if row is None or row[0] is None:
            return True
        return file_mtime > row[0]

    def _mark_indexed(
        self, source_type: str, source_name: str, source_id: str, file_mtime: float
    ):
        conn = self._ensure_conn()
        conn.execute(
            "INSERT OR REPLACE INTO index_meta VALUES (?, ?, ?, ?, ?)",
            (source_type, source_name, source_id, file_mtime, time.time()),
        )

    def _clear_source(self, source_type: str, source_name: str):
        """Remove all entries for a given source before reindexing."""
        conn = self._ensure_conn()
        conn.execute(
            "DELETE FROM search_content WHERE source_type=? AND source_name=?",
            (source_type, source_name),
        )
        # Also clear index_meta for this source
        conn.execute(
            "DELETE FROM index_meta WHERE source_type=? AND source_name=?",
            (source_type, source_name),
        )
        conn.commit()

    # ------------------------------------------------------------------
    # Full build
    # ------------------------------------------------------------------

    def build_all(
        self,
        text_modules=None,
        lexicon_modules=None,
        commentary_modules=None,
        acc2_modules=None,
        force=False,
    ):
        """Build or refresh the full search index.

        Parameters are module dicts keyed by abbreviation.
        Keyword args:
            text_modules: dict of TextModule instances.
            lexicon_modules: dict of LexiconModule instances.
            commentary_modules: dict of CommentaryModule instances.
            acc2_modules: dict of Acc2Module instances.
            force: if True, clear all data before rebuilding.
        """
        t0 = time.time()
        self._bulk_mode()  # Fast pragmas for bulk inserts
        try:
            if force:
                conn = self._ensure_conn()
                conn.execute("DELETE FROM search_content")
                conn.execute("DELETE FROM index_meta")
                conn.execute("DELETE FROM search_idx")
                conn.commit()
                print("Cleared existing index (force rebuild)")

            total = 0

            if text_modules:
                total += self._index_bibles(text_modules)
            else:
                print("  (no text_modules passed — skipping Bible indexing)")

            # Original-language JSON texts (TR, TISCH, LXX, WLC) with explicit
            # el/he tagging.  Must run after _index_bibles (which may also index
            # lxx/wlc with auto-detected language -- _needs_reindex will skip
            # if already up-to-date from the newer explicit index pass here).
            total += self._index_original_language_texts()
            # Apparatus texts from data/*_apparatus_text.json
            total += self._index_apparatus_texts()
            # Apparatus verse-keyed entries from data/*_verse_index.json
            total += self._index_apparatus_verses()
            if lexicon_modules:
                print(f"  Lexicon modules: {len(lexicon_modules)} loaded, indexing...")
                total += self._index_lexicons(lexicon_modules)
            else:
                print("  (no lexicon_modules passed — skipping lexicon indexing)")
            if commentary_modules:
                total += self._index_commentaries(commentary_modules)
            if acc2_modules:
                total += self._index_acc2(acc2_modules)

            # ── Bolls cache (public-domain translations) ──
            try:
                total += self._index_bolls_cache()
            except Exception as exc:
                print(f"  WARNING: _index_bolls_cache failed: {exc}", file=sys.stderr)

            # ── Open Scriptures Strong's lexicons ──
            try:
                total += self._index_openscriptures()
            except Exception as exc:
                print(
                    f"  WARNING: _index_openscriptures failed: {exc}", file=sys.stderr
                )

            elapsed = time.time() - t0
            if total > 0:
                # Rebuild FTS5 index to ensure content-table sync is current
                try:
                    conn = self._ensure_conn()
                    conn.execute("INSERT INTO search_idx(search_idx) VALUES('rebuild')")
                    conn.commit()
                    # Merge b-tree segments for query performance
                    conn.execute(
                        "INSERT INTO search_idx(search_idx) VALUES('optimize')"
                    )
                    conn.commit()
                except Exception as exc:
                    print(f"ERROR: FTS5 rebuild failed: {exc}", file=sys.stderr)
                    raise
                print(f"Indexed {total} items in {elapsed:.1f}s  →  {self.db_path}")
            return total
        finally:
            self._safe_mode()  # Restore safe pragmas even on crash
            self.checkpoint()  # Truncate any WAL growth from rebuild

    # ------------------------------------------------------------------
    # _index_bibles  --  installed .atext modules + JSON fallbacks
    # ------------------------------------------------------------------

    def _index_bibles(self, modules: dict) -> int:
        """Index Bible texts from installed .atext modules and JSON files.

        Installed .atext modules (TextModule instances) take priority over
        JSON fallbacks.  For each .atext module, canonical verses are read
        via get_verse() and indexed.  JSON files are only indexed for
        abbreviations that do not have a corresponding .atext module.
        """
        count = 0

        # ---- Index installed .atext modules first ----
        atext_covered: set[str] = set()
        for abbrev, mod in modules.items():
            # Fix: skip modules whose binary parser is known-broken
            if abbrev in _BYPASS_INDEX:
                print(
                    f"  Skipping Bible: {abbrev} (binary parser unreliable, "
                    f"routed to bolls.life API)"
                )
                # Clear any stale data from previous builds
                self._clear_source("bible", abbrev)
                atext_covered.add(abbrev.lower())
                continue

            try:
                path = mod.filepath
                mtime = path.stat().st_mtime if path.exists() else 0.0
            except Exception:
                mtime = 0.0

            if not self._needs_reindex("bible", abbrev, mtime):
                continue

            print(f"  Indexing Bible: {abbrev} (.atext) ...", end="", flush=True)
            self._clear_source("bible", abbrev)
            sub_count = self._index_atext_module(mod, abbrev, mtime)
            print(f" {sub_count} verses")
            count += sub_count
            atext_covered.add(abbrev.lower())

        # ---- JSON fallbacks for abbreviations without .atext modules ----
        if not _JSON_DIR.exists():
            return count

        for json_path in sorted(_JSON_DIR.glob("*.json")):
            if json_path.name == "search_index.json":
                continue
            stem = json_path.stem.lower()
            abbrev = _JSON_TO_ABBREV.get(stem, stem.upper())

            # Skip if already covered by an installed .atext module
            if abbrev.lower() in atext_covered:
                continue

            # Fix: skip modules whose binary parser is known-broken
            if abbrev in _BYPASS_INDEX:
                print(
                    f"  Skipping Bible: {abbrev} ({json_path.name}) — binary "
                    f"parser unreliable, routed to bolls.life API"
                )
                # Clear any stale data from previous builds
                self._clear_source("bible", abbrev)
                continue

            mtime = json_path.stat().st_mtime
            if not self._needs_reindex("bible", abbrev, mtime):
                continue

            print(
                f"  Indexing Bible: {abbrev} ({json_path.name}) ...", end="", flush=True
            )
            self._clear_source("bible", abbrev)
            sub_count = self._index_json_bible(json_path, abbrev, mtime)
            print(f" {sub_count} verses")
            count += sub_count

        return count

    # ------------------------------------------------------------------
    # _index_original_language_texts  --  TR / TISCH / LXX / WLC
    # ------------------------------------------------------------------

    # Mapping: JSON stem → (abbrev, language)
    _ORIGINAL_LANGUAGE_SOURCES: dict[str, tuple[str, str]] = {
        "tr": ("TR", "el"),
        "tisch": ("TISCH", "el"),
        "lxx": ("LXX1", "el"),
        "wlc": ("WLC", "he"),
    }

    def _index_original_language_texts(self) -> int:
        """Index TR, TISCH, LXX, and WLC JSON caches with explicit language tags.

        Unlike _index_bibles() which auto-detects language, this method
        hard-codes 'el' (Greek) for TR/TISCH/LXX and 'he' (Hebrew) for WLC.
        Designed as a fast, standalone prototype for ingesting original-language
        Bible texts into the FTS5 search index.

        Returns:
            Total number of verses indexed.
        """
        count = 0

        if not _JSON_DIR.exists():
            print("  (data/ directory not found — skipping original-language texts)")
            return 0

        for json_path in sorted(_JSON_DIR.glob("*.json")):
            stem = json_path.stem.lower()
            if stem not in self._ORIGINAL_LANGUAGE_SOURCES:
                continue

            abbrev, lang = self._ORIGINAL_LANGUAGE_SOURCES[stem]

            # Fix: skip modules whose binary parser is known-broken
            if abbrev in _BYPASS_INDEX:
                print(
                    f"  Skipping Original: {abbrev} ({json_path.name}) — "
                    f"binary parser unreliable, routed to bolls.life API"
                )
                # Clear any stale data from previous builds
                self._clear_source("bible", abbrev)
                continue
            mtime = json_path.stat().st_mtime

            if not self._needs_reindex("bible", abbrev, mtime):
                continue

            print(
                f"  Indexing Original: {abbrev} ({json_path.name}) [{lang}] ...",
                end="",
                flush=True,
            )
            self._clear_source("bible", abbrev)
            sub_count = self._index_json_bible(json_path, abbrev, mtime, language=lang)
            print(f" {sub_count} verses")
            count += sub_count

        return count

    # ------------------------------------------------------------------
    # _index_apparatus_texts  --  NA28 / LXX / LXX2 / MT-ETCBC-A apparatus
    # ------------------------------------------------------------------

    # Mapping: apparatus JSON name → language
    _APPARATUS_SOURCES: dict[str, str] = {
        "NA28": "el",
        "LXX": "el",
        "LXX2": "el",
        "MT-ETCBC-A": "he",
    }

    # Valid Latin-1 Supplement characters in apparatus text (surviving
    # decode_apparatus).  Everything else in U+0080-U+00FF is mojibake
    # from unhandled high bytes and must be stripped before FTS5 indexing.
    _APPARATUS_VALID_LATIN1 = frozenset(
        {
            0xA6,  # ¦  broken bar (apparatus separator)
            0xA7,  # §  section sign (legacy final-sigma marker)
            0xB8,  # ¸  cedilla (apparatus punctuation)
            0xB9,  # ¹  superscript 1 (used in f¹)
            0xC4,  # Ä  German A-umlaut (Prolegomena)
            0xD6,  # Ö  German O-umlaut (Prolegomena)
            0xD7,  # ×  multiplication sign (attestation separator)
            0xDC,  # Ü  German U-umlaut (Prolegomena)
            0xDF,  # ß  German sharp s (Prolegomena)
            0xE4,  # ä  German a-umlaut (Prolegomena)
            0xF6,  # ö  German o-umlaut (Prolegomena)
            0xFC,  # ü  German u-umlaut (Prolegomena)
        }
    )

    @staticmethod
    def _sanitize_apparatus_text(text: str) -> str:
        """Strip Latin-1 mojibake characters from apparatus text.

        Characters in U+0080-U+00FF that are NOT in the whitelist of
        known-valid apparatus punctuation are replaced with space.
        Characters outside this range pass through unchanged.
        """
        _valid = SearchIndex._APPARATUS_VALID_LATIN1
        result = []
        for ch in text:
            cp = ord(ch)
            if 0x80 <= cp <= 0xFF and cp not in _valid:
                result.append(" ")
            else:
                result.append(ch)
        return "".join(result)

    def _index_apparatus_texts(self) -> int:
        """Index apparatus text entries from data/*_apparatus_text.json files.

        Each JSON file contains a list of entries under the ``entries`` key
        with ``index`` (entry number) and ``raw_text`` (content) fields.
        Source-type is set to ``apparatus`` so searches can filter by it
        (e.g. ``source_type="apparatus"``).

        Language tagging:
          - NA28 / LXX / LXX2 → "el" (Greek apparatus)
          - MT-ETCBC-A        → "he" (Hebrew apparatus)

        Returns:
            Total number of apparatus entries indexed.
        """
        count = 0

        if not _JSON_DIR.exists():
            print("  (data/ directory not found — skipping apparatus texts)")
            return 0

        for json_path in sorted(_JSON_DIR.glob("*_apparatus_text.json")):
            try:
                with open(json_path, "r", encoding="utf-8") as f:
                    data = json.load(f)
            except Exception:
                continue

            name = data.get("name", "")
            if not name:
                continue

            lang = self._APPARATUS_SOURCES.get(name, "en")
            entries = data.get("entries", [])
            if not entries:
                continue

            mtime = json_path.stat().st_mtime

            if not self._needs_reindex("apparatus", name, mtime):
                continue

            print(
                f"  Indexing Apparatus: {name} ({json_path.name}) [{lang}] ...",
                end="",
                flush=False,
            )
            self._clear_source("apparatus", name)

            conn = self._ensure_conn()
            batch = []
            sub_count = 0

            for entry in []:
                idx = entry.get("index")
                # Prefer decoded_text (contains proper Unicode Greek/sigla);
                # fall back to raw_text for entries that weren't decoded.
                raw_text = entry.get("decoded_text", "") or entry.get("raw_text", "")
                if not raw_text or not raw_text.strip():
                    continue

                # Fix: skip §0 entries (binary headers with no real text)
                if idx is not None and int(idx) == 0:
                    continue

                # Fix: skip entries where text is mostly non-printable
                # (binary garbage that escaped the decoder)
                printable = sum(1 for c in raw_text if c.isprintable() or c in "\t\n\r")
                if len(raw_text) > 0 and printable < len(raw_text) * 0.5:
                    continue

                # Fix: skip apparatus §0-§2 entries that are binary
                # headers.  Some apparatus formats (MT-ETCBC-A) have
                # multiple header entries before real data.
                if idx is not None and int(idx) <= 2:
                    continue

                source_id = str(idx) if idx is not None else "?"
                clean = self._sanitize_apparatus_text(raw_text.strip())

                # Skip entries that are too short or appear to be binary noise
                if len(clean) < 4:
                    continue

                # Skip entries with too few "words" (consecutive letter-like runs).
                # Hebrew apparatus sigla (ℵ, 𝔐, etc.) are not alphabetic by
                # isalpha(), so we also check Hebrew, Greek, and Letterlike
                # Symbols ranges.  Threshold lowered to 1 for MT-ETCBC-A.
                min_words = 1 if name == "MT-ETCBC-A" else 3
                alpha_words = 0
                in_word = False
                for ch in clean:
                    cp = ord(ch)
                    is_word_char = (
                        ch.isalpha()
                        or 0x0590 <= cp <= 0x05FF  # Hebrew
                        or 0x0370 <= cp <= 0x03FF  # Greek
                        or 0x1F00 <= cp <= 0x1FFF  # Greek Extended
                        and 0x2100 <= cp <= 0x214F  # Letterlike Symbols (ℵ etc.)
                    )
                    if is_word_char:
                        if not in_word:
                            alpha_words += 1
                            in_word = True
                    else:
                        in_word = False
                if alpha_words < min_words:
                    continue

                # Citation: "NA28 Apparatus §42" or similar
                citation = f"{name} Apparatus \u00a7{source_id}"

                body_text = citation + " — " + clean
                if lang in ("el", "he"):
                    body_norm = (
                        strip_diacritics(citation).lower()
                        + " — "
                        + strip_diacritics(clean).lower()
                    )
                else:
                    body_norm = citation.lower() + " — " + clean.lower()

                batch.append(
                    (
                        "apparatus",
                        name,
                        source_id,
                        lang,
                        citation,
                        body_text,
                        body_norm,
                        "",
                    )
                )

                if len(batch) >= self._BATCH_SIZE:
                    self._insert_batch(conn, batch)
                    sub_count += len(batch)
                    batch = []
                    if sub_count % 5000 == 0:
                        print(".", end="", flush=True)

            if batch:
                self._insert_batch(conn, batch)
                sub_count += len(batch)

            self._mark_indexed("apparatus", name, "_all", mtime)
            print(f" {sub_count} entries")
            count += sub_count

        return count

    # ------------------------------------------------------------------
    # _index_apparatus_verses  --  verse-keyed apparatus entries
    # ------------------------------------------------------------------

    def _index_apparatus_verses(self) -> int:
        """Index apparatus entries keyed by Bible verse reference.

        Reads verse indices from data/*_verse_index.json and inserts
        FTS5 rows so that verse-based queries (e.g. "John 1:1") can
        surface apparatus results.

        Each row gets a citation like "Matthew 1:1 (NA28 Apparatus)"
        and a body_text that includes all linked entry citations so
        that FTS5 matches the verse reference in the citation prefix.

        Returns:
            Total number of verse-keyed rows inserted.
        """
        count = 0

        if not _JSON_DIR.exists():
            return 0

        for vi_path in sorted(_JSON_DIR.glob("*_verse_index.json")):
            name = vi_path.stem.replace("_verse_index", "")
            if not name:
                continue

            mtime = vi_path.stat().st_mtime

            # Check if already indexed (use source_type="apparatus_verse")
            if not self._needs_reindex("apparatus_verse", name, mtime):
                continue

            try:
                with open(vi_path, "r", encoding="utf-8") as f:
                    verse_index = json.load(f)
            except Exception:
                continue

            if not verse_index:
                continue

            lang = self._APPARATUS_SOURCES.get(name, "en")

            print(
                f"  Indexing Apparatus Verses: {name} ({vi_path.name}) …",
                end="",
                flush=True,
            )
            self._clear_source("apparatus_verse", name)

            conn = self._ensure_conn()
            batch = []
            sub_count = 0

            for verse_key, entry_indices in []:
                if not entry_indices:
                    continue

                # Build citation: "Matthew 1:1 (NA28 Apparatus)"
                citation = f"{verse_key} ({name} Apparatus)"

                # Build body text: citation + linked entry refs
                entry_refs = ", ".join(
                    f"{name} Apparatus §{ei}" for ei in entry_indices[:20]
                )
                if len(entry_indices) > 20:
                    entry_refs += f" … (+{len(entry_indices) - 20} more)"

                body_text = citation + " — " + entry_refs
                body_norm = citation.lower() + " — " + entry_refs.lower()

                # Extract book name from verse key for book filtering
                book = verse_key.rsplit(" ", 1)[0] if " " in verse_key else ""

                batch.append(
                    (
                        "apparatus_verse",
                        name,
                        verse_key,
                        lang,
                        citation,
                        body_text,
                        body_norm,
                        book,
                    )
                )

                if len(batch) >= self._BATCH_SIZE:
                    self._insert_batch(conn, batch)
                    sub_count += len(batch)
                    batch = []
                    if sub_count % 5000 == 0:
                        print(".", end="", flush=True)

            if batch:
                self._insert_batch(conn, batch)
                sub_count += len(batch)

            self._mark_indexed("apparatus_verse", name, "_all", mtime)
            print(f" {sub_count} verse entries")
            count += sub_count

        return count

    def _index_json_bible(
        self, json_path: Path, abbrev: str, mtime: float, language: str | None = None
    ) -> int:
        """Load a single Bible JSON file and index all verses.

        Args:
            json_path: Path to the JSON verse file.
            abbrev: Module abbreviation (e.g. 'KJVS', 'LXX1').
            mtime: File modification time for incremental tracking.
            language: Override language detection. 'el' for Greek, 'he' for
                      Hebrew, or None (default) for auto-detection.
        """
        from .bible_api import BOOK_ID_TO_NAME

        with open(json_path, "r", encoding="utf-8") as f:
            data = json.load(f)

        # Structural guard: skip non-verse-list JSON files (dicts, non-verse lists, etc.)
        if not isinstance(data, list):
            return 0
        if data and not all(k in data[0] for k in ("book", "chapter", "verse")):
            return 0

        conn = self._ensure_conn()
        batch = []
        count = 0

        for verse in data:
            book_id = verse.get("book")
            chapter = verse.get("chapter")
            verse_num = verse.get("verse")
            raw_text = verse.get("text", "")

            if book_id is None or chapter is None or verse_num is None:
                continue

            book_name = BOOK_ID_TO_NAME.get(book_id, f"Book {book_id}")
            ref = f"{book_name} {chapter}:{verse_num}"
            clean = strip_strongs_tags(raw_text)

            if not clean.strip():
                continue

            lang = language if language else detect_language(clean)
            if lang in ("el", "he"):
                norm = strip_diacritics(clean).lower()
            elif lang == "zh":
                norm = separate_cjk_chars(clean.lower())
            else:
                norm = clean.lower()

            # Citation: "Genesis 1:1 (ESV)"
            citation = f"{ref} ({abbrev})"

            # Prepend citation to body text and normalized form
            body_text = citation + " — " + clean
            if lang in ("el", "he"):
                body_norm = strip_diacritics(citation).lower() + " — " + norm
            elif lang == "zh":
                body_norm = separate_cjk_chars(citation.lower()) + " — " + norm
            else:
                body_norm = citation.lower() + " — " + norm

            batch.append(
                ("bible", abbrev, ref, lang, citation, body_text, body_norm, book_name)
            )

            if len(batch) >= self._BATCH_SIZE:
                self._insert_batch(conn, batch)
                count += len(batch)
                batch = []
                if count % 5000 == 0:
                    print(".", end="", flush=True)

        if batch:
            self._insert_batch(conn, batch)
            count += len(batch)

        self._mark_indexed("bible", abbrev, "_all", mtime)
        return count

    # ------------------------------------------------------------------
    # _index_atext_module  --  installed TextModule (.atext) verses
    # ------------------------------------------------------------------

    def _index_atext_module(self, mod, abbrev: str, mtime: float) -> int:
        """Index all canonical verses from an installed .atext TextModule.

        Iterates the module's canon book list and per-book verse counts,
        calling mod.get_verse() for each reference.  Verse text is stripped
        of Strong's tags, language-detected, and batch-inserted into the
        FTS5 search_content table.

        When the binary parser returns None for a verse, falls back to the
        bolls.life API (rate-limited at 50ms/call).  Module abbreviations
        are mapped via _BOLLS_FALLBACK and bible_api.BOLLS_CODES.

        Special case: KJVA (KJV with Apocrypha) — this is a companion module
        to KJVS containing only the Apocrypha.  Its binary parser cannot
        calibrate (no Genesis text), and the TextModule's default Protestant
        canon excludes all deuterocanonical books.  KJVA is indexed via a
        separate path (_index_kjva) that uses CANON_CATHOLIC and the
        bolls.life KJV API exclusively.

        Only marks the source as indexed when at least one verse was
        successfully stored (prevents silent zero-row failures).
        """
        # ── KJVA: route to Catholic canon + bolls.life KJV ──
        if abbrev == "KJVA":
            return self._index_kjva(mtime)

        # Resolve bolls.life fallback code (None if unavailable)
        bolls_code = self._resolve_bolls_fallback(abbrev)

        # ── _BOLLS_FALLBACK modules: binary parser produces unreliable text ──
        # Modules listed in _BOLLS_FALLBACK (ASV, CEB, LBLA, LSV, WEB, etc.)
        # often have only 2 JSON anchors (Gen 1:1 + John 1:1), causing
        # get_verse() to return non-None but systematically wrong text for
        # most books.  Route these to bolls.life chapter-at-a-time for
        # correct verse text during indexing.
        if abbrev.upper() in _BOLLS_FALLBACK and bolls_code is not None:
            return self._index_bolls_fallback_module(mod, abbrev, mtime, bolls_code)

        conn = self._ensure_conn()
        batch: list[tuple] = []
        count = 0
        bolls_verses_fetched = 0

        for book in mod._books:
            book_id = resolve_book_id(book)
            vdata = mod._verse_counts.get(book)
            if not vdata:
                continue

            for ch_idx, verse_count in enumerate(vdata, 1):
                for vs in range(1, verse_count + 1):
                    ref = f"{book} {ch_idx}:{vs}"
                    try:
                        raw_text = mod.get_verse(ref)
                    except Exception:
                        raw_text = None

                    # Fallback to bolls.life API if binary parser failed
                    if (
                        raw_text is None
                        and bolls_code is not None
                        and book_id is not None
                    ):
                        try:
                            from accordance_parser.bible_api import fetch_bolls_verse

                            vr = fetch_bolls_verse(bolls_code, book_id, ch_idx, vs)
                            if vr and vr.text:
                                raw_text = vr.text
                                bolls_verses_fetched += 1
                                # Rate-limit: 50ms between bolls.life calls
                                time.sleep(0.05)
                        except Exception:
                            pass

                    if raw_text is None:
                        continue

                    clean = strip_strongs_tags(raw_text)
                    # Strip backtick/hash characters that appear in some
                    # module decoders as placeholder markers
                    clean = clean.replace("`", "").replace("#", "")
                    if not clean.strip():
                        continue

                    lang = detect_language(clean)
                    if lang in ("el", "he"):
                        norm = strip_diacritics(clean).lower()
                    elif lang == "zh":
                        norm = separate_cjk_chars(clean.lower())
                    else:
                        norm = clean.lower()

                    citation = f"{ref} ({abbrev})"
                    body_text = citation + " — " + clean
                    if lang in ("el", "he"):
                        body_norm = strip_diacritics(citation).lower() + " — " + norm
                    elif lang == "zh":
                        body_norm = separate_cjk_chars(citation.lower()) + " — " + norm
                    else:
                        body_norm = citation.lower() + " — " + norm

                    batch.append(
                        (
                            "bible",
                            abbrev,
                            ref,
                            lang,
                            citation,
                            body_text,
                            body_norm,
                            book,
                        )
                    )

                    if len(batch) >= self._BATCH_SIZE:
                        self._insert_batch(conn, batch)
                        count += len(batch)
                        batch = []
                        if count % 5000 == 0:
                            print(".", end="", flush=True)

        if batch:
            self._insert_batch(conn, batch)
            count += len(batch)

        # Fix A: only mark indexed if we actually inserted verses.
        # Otherwise the module stays unmarked so it will be retried.
        if count > 0:
            if bolls_verses_fetched > 0:
                print(f" [{bolls_verses_fetched} via bolls.life]", end="", flush=True)
            self._mark_indexed("bible", abbrev, "_all", mtime)
        else:
            print(
                " WARNING: 0 verses indexed (binary parser + bolls.life both failed)",
                end="",
            )
            # Mark with current time so we don't retry on every build_all
            # (avoids infinite retries for modules not available via either path).
            # The zero-count guard in _needs_reindex will still force retry on
            # next file change.
            self._mark_indexed("bible", abbrev, "_all", time.time())

        return count

    # ------------------------------------------------------------------
    # _index_kjva  --  KJV with Apocrypha (Catholic canon + bolls.life)
    # ------------------------------------------------------------------

    def _index_kjva(self, mtime: float) -> int:
        """Index KJVA using CANON_CATHOLIC and the bolls.life KJV API.

        KJVA is a companion module to KJVS containing only the Apocrypha/
        Deuterocanonical books.  The binary parser cannot calibrate (the
        module has no Genesis text), and the TextModule's default Protestant
        canon (66 books) excludes all deuterocanonical books.

        This method bypasses the binary parser entirely and uses the
        bolls.life KJV API for the full Catholic canon (73 books).  It
        fetches entire chapters at a time (one HTTP request per chapter)
        for performance — ~1,300 chapters at 50ms rate-limit ≈ ~65 seconds
        of API throttling.
        """
        from accordance_parser.canon import CANON_CATHOLIC
        from accordance_parser.bible_api import fetch_bolls_chapter

        conn = self._ensure_conn()
        batch: list[tuple] = []
        count = 0
        bolls_code = "KJV"
        abbrev = "KJVA"

        # Track which books yielded verses (for progress reporting)
        books_with_verses: list[str] = []

        for book_obj in CANON_CATHOLIC.books:
            book_name = book_obj.name
            book_id = book_obj.bolls_id
            if book_id is None:
                continue  # AddEsth, AddPs, 3Macc, 4Macc — no bolls.life ID

            vdata = CANON_CATHOLIC.last_verse.get(book_obj.osis)
            if not vdata:
                continue

            book_verse_count = 0
            for ch_idx, verse_count in enumerate(vdata, 1):
                if verse_count == 0:
                    continue

                # Fetch entire chapter from bolls.life in one API call
                try:
                    chapter_verses = fetch_bolls_chapter(bolls_code, book_id, ch_idx)
                    time.sleep(0.05)  # rate-limit between chapters
                except Exception:
                    continue

                for vr in chapter_verses:
                    if not vr.text:
                        continue
                    vs = vr.verse
                    raw_text = vr.text

                    clean = strip_strongs_tags(raw_text)
                    clean = clean.replace("`", "").replace("#", "")
                    if not clean.strip():
                        continue

                    lang = detect_language(clean)
                    if lang in ("el", "he"):
                        norm = strip_diacritics(clean).lower()
                    elif lang == "zh":
                        norm = separate_cjk_chars(clean.lower())
                    else:
                        norm = clean.lower()

                    ref = f"{book_name} {ch_idx}:{vs}"
                    citation = f"{ref} ({abbrev})"
                    body_text = citation + " — " + clean
                    if lang in ("el", "he"):
                        body_norm = strip_diacritics(citation).lower() + " — " + norm
                    elif lang == "zh":
                        body_norm = separate_cjk_chars(citation.lower()) + " — " + norm
                    else:
                        body_norm = citation.lower() + " — " + norm

                    batch.append(
                        (
                            "bible",
                            abbrev,
                            ref,
                            lang,
                            citation,
                            body_text,
                            body_norm,
                            book_name,
                        )
                    )
                    book_verse_count += 1

                    if len(batch) >= self._BATCH_SIZE:
                        self._insert_batch(conn, batch)
                        count += len(batch)
                        batch = []
                        if count % 5000 == 0:
                            print(".", end="", flush=True)

            if book_verse_count > 0:
                books_with_verses.append(book_name)

        if batch:
            self._insert_batch(conn, batch)
            count += len(batch)

        if count > 0:
            # Report which books were indexed
            deutero_books = [
                b
                for b in books_with_verses
                if b
                not in {
                    "Genesis",
                    "Exodus",
                    "Leviticus",
                    "Numbers",
                    "Deuteronomy",
                    "Joshua",
                    "Judges",
                    "Ruth",
                    "1 Samuel",
                    "2 Samuel",
                    "1 Kings",
                    "2 Kings",
                    "1 Chronicles",
                    "2 Chronicles",
                    "Ezra",
                    "Nehemiah",
                    "Esther",
                    "Job",
                    "Psalms",
                    "Proverbs",
                    "Ecclesiastes",
                    "Song of Solomon",
                    "Isaiah",
                    "Jeremiah",
                    "Lamentations",
                    "Ezekiel",
                    "Daniel",
                    "Hosea",
                    "Joel",
                    "Amos",
                    "Obadiah",
                    "Jonah",
                    "Micah",
                    "Nahum",
                    "Habakkuk",
                    "Zephaniah",
                    "Haggai",
                    "Zechariah",
                    "Malachi",
                    "Matthew",
                    "Mark",
                    "Luke",
                    "John",
                    "Acts",
                    "Romans",
                    "1 Corinthians",
                    "2 Corinthians",
                    "Galatians",
                    "Ephesians",
                    "Philippians",
                    "Colossians",
                    "1 Thessalonians",
                    "2 Thessalonians",
                    "1 Timothy",
                    "2 Timothy",
                    "Titus",
                    "Philemon",
                    "Hebrews",
                    "James",
                    "1 Peter",
                    "2 Peter",
                    "1 John",
                    "2 John",
                    "3 John",
                    "Jude",
                    "Revelation",
                }
            ]
            print(
                f" [{len(books_with_verses)} books, Apocrypha: "
                f"{', '.join(deutero_books[:8])}"
                f"{'...' if len(deutero_books) > 8 else ''}]",
                end="",
                flush=True,
            )
            self._mark_indexed("bible", abbrev, "_all", mtime)
        else:
            print(
                " WARNING: 0 verses indexed (bolls.life KJV returned no text)", end=""
            )

        return count

    # ------------------------------------------------------------------
    # _index_bolls_fallback_module  --  bolls.life-only indexing for
    # modules with unreliable binary parsers
    # ------------------------------------------------------------------

    def _index_bolls_fallback_module(
        self, mod, abbrev: str, mtime: float, bolls_code: str
    ) -> int:
        """Index a module using bolls.life chapter-at-a-time (skip binary parser).

        Modules listed in _BOLLS_FALLBACK (ASV, CEB, LBLA, LSV, WEB, etc.)
        have binary parsers that return non-None but systematically wrong
        text due to insufficient anchor data (typically only 2 anchors:
        Gen 1:1 + John 1:1).  This method bypasses the binary parser
        entirely and fetches all verse text from bolls.life, one chapter
        per API call (~1,200 chapters at 50ms rate limit ≈ 60 seconds).

        Uses the module's own _books and _verse_counts (Protestant canon)
        to drive iteration, and resolve_book_id() to map book names to
        bolls.life numeric book IDs.
        """
        from accordance_parser.bible_api import fetch_bolls_chapter

        conn = self._ensure_conn()
        batch: list[tuple] = []
        count = 0

        for book_name in mod._books:
            book_id = resolve_book_id(book_name)
            vdata = mod._verse_counts.get(book_name)
            if not vdata or book_id is None:
                continue

            for ch_idx, verse_count in enumerate(vdata, 1):
                if verse_count == 0:
                    continue

                # Fetch entire chapter from bolls.life
                try:
                    chapter_verses = fetch_bolls_chapter(bolls_code, book_id, ch_idx)
                    time.sleep(0.05)  # rate-limit between chapters
                except Exception:
                    continue

                # Index each verse from the chapter response
                for vr in chapter_verses:
                    if not vr.text:
                        continue
                    vs = vr.verse
                    raw_text = vr.text

                    clean = strip_strongs_tags(raw_text)
                    clean = clean.replace("`", "").replace("#", "")
                    if not clean.strip():
                        continue

                    lang = detect_language(clean)
                    norm = (
                        strip_diacritics(clean).lower()
                        if lang in ("el", "he")
                        else clean.lower()
                    )

                    ref = f"{book_name} {ch_idx}:{vs}"
                    citation = f"{ref} ({abbrev})"
                    body_text = citation + " — " + clean
                    if lang in ("el", "he"):
                        body_norm = strip_diacritics(citation).lower() + " — " + norm
                    else:
                        body_norm = citation.lower() + " — " + norm

                    batch.append(
                        (
                            "bible",
                            abbrev,
                            ref,
                            lang,
                            citation,
                            body_text,
                            body_norm,
                            book_name,
                        )
                    )

                    if len(batch) >= self._BATCH_SIZE:
                        self._insert_batch(conn, batch)
                        count += len(batch)
                        batch = []
                        if count % 5000 == 0:
                            print(".", end="", flush=True)

        if batch:
            self._insert_batch(conn, batch)
            count += len(batch)

        if count > 0:
            self._mark_indexed("bible", abbrev, "_all", mtime)
        else:
            print(
                " WARNING: 0 verses indexed (bolls.life returned no text for "
                f"{abbrev})",
                end="",
            )
            self._mark_indexed("bible", abbrev, "_all", time.time())

        return count

    @staticmethod
    def _resolve_bolls_fallback(abbrev: str) -> str | None:
        """Resolve a TextModule abbreviation to a bolls.life translation code.

        Checks the explicit _BOLLS_FALLBACK map first, then falls back to
        bible_api.BOLLS_CODES for codes that already have canonical mappings
        (e.g. DOUAY -> DRB).  Returns None if no mapping exists.
        """
        # 1. Explicit _BOLLS_FALLBACK map (modules whose abbreviation differs
        #    from the bolls.life code, or that aren't in BOLLS_CODES yet)
        code = _BOLLS_FALLBACK.get(abbrev.upper())
        if code:
            return code

        # 2. bible_api.BOLLS_CODES canonical map (e.g. DOUAY -> DRB)
        try:
            from accordance_parser.bible_api import BOLLS_CODES

            code = BOLLS_CODES.get(abbrev.lower())
            if code:
                return code
        except ImportError:
            pass

        return None

    # ------------------------------------------------------------------
    # _index_lexicons
    # ------------------------------------------------------------------

    def _index_lexicons(self, modules: dict) -> int:
        """Index lexicon entries using the module's internal decoder."""
        count = 0
        for abbrev, mod in sorted(modules.items()):
            try:
                path = mod.filepath
                mtime = path.stat().st_mtime if path.exists() else 0
            except Exception:
                mtime = 0

            if not self._needs_reindex("lexicon", abbrev, mtime):
                continue

            print(f"  Indexing Lexicon: {abbrev} ...", end="", flush=True)
            self._clear_source("lexicon", abbrev)
            try:
                sub_count = self._index_single_lexicon(abbrev, mod, mtime)
                print(f" {sub_count} entries")
                count += sub_count
            except Exception as exc:
                print(f" FAILED: {exc}", file=sys.stderr)
        return count

    def _index_single_lexicon(self, abbrev: str, mod, mtime: float) -> int:
        """Index all entries from a single lexicon module."""
        conn = self._ensure_conn()
        batch = []
        count = 0

        try:
            entries = mod._entries  # dict[str, list[int]]
        except Exception:
            return 0

        title = getattr(mod, "title", "") or abbrev

        for headword, offsets in entries.items():
            if not offsets:
                continue
            try:
                text = mod._read_entry(offsets[0])
            except Exception:
                continue
            if not text or not text.strip():
                continue

            # Use headword as source_id; language from module or detect
            lang = "el"
            if hasattr(mod, "_is_hebrew") and mod._is_hebrew:
                lang = "he"
            elif hasattr(mod, "_is_english") and mod._is_english:
                lang = "en"
            elif hasattr(mod, "language"):
                if mod.language == "Hebrew":
                    lang = "he"
                elif mod.language == "English":
                    lang = "en"

            # Citation: '"headword" — BDAG (Title)'
            citation = lexicon_citation(headword, abbrev, title)

            # Prepend headword so the entry key is searchable in body_text
            clean = headword + " " + text.strip()
            import re

            clean = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F]", "", clean)
            if lang in ("el", "he"):
                norm = (
                    strip_diacritics(citation).lower()
                    + " — "
                    + strip_diacritics(headword).lower()
                    + " "
                    + strip_diacritics(clean).lower()
                )
            else:
                content = headword.lower() + " " + clean.lower()
                norm = citation.lower() + " — " + _stem_english(content)

            body_text = citation + " — " + clean

            batch.append(
                ("lexicon", abbrev, headword, lang, citation, body_text, norm, "")
            )

            if len(batch) >= self._BATCH_SIZE:
                self._insert_batch(conn, batch)
                count += len(batch)
                batch = []
                if count % 5000 == 0:
                    print(".", end="", flush=True)

        if batch:
            self._insert_batch(conn, batch)
            count += len(batch)

        self._mark_indexed("lexicon", abbrev, "_all", mtime)
        return count

    # ------------------------------------------------------------------
    # _index_commentaries
    # ------------------------------------------------------------------

    def _index_commentaries(self, modules: dict) -> int:
        """Index commentary sections using the module's section reader."""
        count = 0
        for abbrev, mod in sorted(modules.items()):
            try:
                path = mod.filepath
                mtime = path.stat().st_mtime if path.exists() else 0
            except Exception:
                mtime = 0

            if not self._needs_reindex("commentary", abbrev, mtime):
                continue

            print(f"  Indexing Commentary: {abbrev} ...", end="", flush=True)
            self._clear_source("commentary", abbrev)
            sub_count = self._index_single_commentary(abbrev, mod, mtime)
            print(f" {sub_count} sections")
            count += sub_count
        return count

    def _index_single_commentary(self, abbrev: str, mod, mtime: float) -> int:
        """Index all sections from a single commentary module."""
        conn = self._ensure_conn()
        batch = []
        count = 0
        title = getattr(mod, "title", "") or abbrev

        try:
            entries = mod._entries  # list of (book, chapter, vs_start, vs_end, offset)
        except Exception:
            return 0

        # Detect corpus for non-Bible reference systems (AF, Philo, Q, etc.)
        corpus_id = detect_corpus(abbrev)

        # Pre-compute section boundaries from the entries list to avoid
        # the expensive forward byte-scan in _read_section().
        # Hermeneia modules with 7,639 entries used to take 4-7 minutes
        # per module; this fast path reads raw bytes directly using
        # offsets already known from the entry index.
        _format = getattr(mod, "_format", None)

        # ── Catena Aurea: pre-parsed text from JSON, no mmap needed ──
        if _format == "catena_aurea":
            return self._index_catena_aurea(abbrev, mod, mtime, conn, entries, title)

        # ── Apparatus: pre-extracted JSON, no mmap needed ──
        # Apparatus modules (NA28, LXX, LXX2, MT-ETCBC-A) have their
        # text pre-extracted into JSON files; entries[4] is a JSON entry
        # index, not a byte offset, so mmap reads would produce garbage.
        if _format == "apparatus":
            return self._index_apparatus_as_commentary(
                abbrev, mod, mtime, conn, entries, title
            )

        # Detect CR-delimited generic modules (reading plans, certain tools).
        # These use single-\r entry separators instead of \r\r, so the byte
        # skip from entry offset to text start is +1, not +2.
        rr_count = getattr(mod, "_rr_count", None)
        rt_count = getattr(mod, "_rt_count", None)
        use_cr_delim = (
            rt_count is not None
            and rr_count is not None
            and rt_count > rr_count * 5
            and rr_count < 50
        )
        byte_skip = 1 if use_cr_delim else 2

        section_bounds = []
        # Per-book section counter for ch=0 non-corpus entries within a
        # corpus module (e.g. Hermeneia AF intro sections). Ensures each
        # section gets a unique source_id instead of all being "section 0".
        _ch0_counter: dict[str, int] = {}

        if _format == "hermeneia":
            if corpus_id:
                # ── Non-Bible corpus modules (AF, Philo, Q) ──
                # Index ALL entries individually.  In these modules each
                # entry is a discrete commentary section on a specific
                # work (e.g. IgnEph.1.1, Did.8.1), not a sub-section
                # of a larger pericope.  Ch=0 entries (intro/bibliography
                # headers) also get indexed individually here.
                for i, entry in enumerate(entries):
                    start = entry[4]
                    end = entries[i + 1][4] if i + 1 < len(entries) else mod.size
                    section_bounds.append((entry, start, end))
            else:
                # ── Standard Bible Hermeneia ──
                # chapter=0 entries are top-level section headers;
                # chapter>0 entries are verse sub-sections within them.
                # Index only ch=0 entries (their bounds extend to the next
                # ch=0 entry, capturing all sub-section content).  Skip
                # ch>0 sub-entries to avoid duplicate content in the index.
                top_indices = [i for i, e in enumerate(entries) if e[1] == 0]

                if not top_indices:
                    # Fallback: no section headers found, index all entries
                    for i, entry in enumerate(entries):
                        start = entry[4]
                        end = entries[i + 1][4] if i + 1 < len(entries) else mod.size
                        section_bounds.append((entry, start, end))
                else:
                    # Handle entries before the first top-level header (if any)
                    if top_indices[0] > 0:
                        entry = entries[0]
                        start = entry[4]
                        end = entries[top_indices[0]][4]
                        section_bounds.append((entry, start, end))

                    for ti, idx in enumerate(top_indices):
                        entry = entries[idx]
                        start = entry[4]
                        end = (
                            entries[top_indices[ti + 1]][4]
                            if ti + 1 < len(top_indices)
                            else mod.size
                        )
                        section_bounds.append((entry, start, end))
        else:
            # JFB / generic / ja_nt2: every entry is its own section
            for i, entry in enumerate(entries):
                start = entry[4]
                end = entries[i + 1][4] if i + 1 < len(entries) else mod.size
                section_bounds.append((entry, start, end))

        for entry, start, end in section_bounds:
            try:
                book, chapter, vs_start, vs_end, offset = entry
                # jps/treasury formats: offsets point to [Book, p. NNN] headers,
                # not \r\r delimiters.  Use the module's own section reader
                # which knows how to skip past the header to actual text.
                if _format in ("jps", "treasury"):
                    raw_text = mod._read_section(offset)
                    if raw_text:
                        text = raw_text
                    else:
                        raw = mod.mm[start + byte_skip : end]
                        text = decode(raw)
                else:
                    raw = mod.mm[start + byte_skip : end]
                    text = decode(raw)
                if hasattr(mod, "_clean_text"):
                    text = mod._clean_text(text)
                text = text.replace("\r", "\n")
                text = text.replace("\t", "    ")
                text = text.strip()
            except Exception:
                continue
            if not text or not text.strip():
                continue

            # Fix 1: Skip noise modules (word-frequency lists, name directories)
            if abbrev in _GENERIC_NOISE_ABBREVS:
                continue

            # Fix 2: Detect generic-format modules (book name IS the module title,
            # not a Bible book).  Use a non-verse reference format for these.
            is_generic = not any(book.startswith(b) for b in _BIBLE_BOOK_NAMES)
            if is_generic:
                if vs_start == 0:
                    # For non-corpus ch=0 entries within a corpus module,
                    # use a sequential counter to give each section a
                    # unique source_id instead of all being "section 0".
                    if (
                        corpus_id
                        and chapter == 0
                        and book not in _HERMENEIA_BOOK_TO_CORPUS_WORK
                    ):
                        counter = _ch0_counter.get(book, 0) + 1
                        _ch0_counter[book] = counter
                        ref = f"{title}, section {counter}"
                    else:
                        ref = f"{title}, section {chapter}"
                else:
                    ref = f"{title} §{chapter}"
            else:
                ref = (
                    f"{book} {chapter}:{vs_start}-{vs_end}"
                    if vs_start != vs_end
                    else f"{book} {chapter}:{vs_start}"
                )

            import re

            clean = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F]", "", text.strip())
            lang = detect_language(clean)

            # Attempt to extract a section heading from the first line
            section_title = _extract_section_heading(clean)

            # Fix 3: Synthetic headings for non-generic modules (verse-reference as heading)
            if not section_title and not is_generic:
                if vs_start == vs_end:
                    section_title = f"{book} {chapter}:{vs_start}"
                else:
                    section_title = f"{book} {chapter}:{vs_start}-{vs_end}"

            # Fix 4: First-line heading extraction for generic modules
            if not section_title and is_generic:
                for line in clean.split("\n"):
                    line = line.strip()
                    if not line:
                        continue
                    # Skip page markers, single-word entries, and all-caps short labels
                    if re.match(r"^\[p\.\s*\d+\]", line):
                        continue
                    if len(line.split()) >= 1:
                        continue
                    # Use first real line as heading, capped at 80 chars
                    section_title = line[:80]
                    break

            # ── Corpus-aware reference from Hermeneia book name ──
            # For non-Bible Hermeneia modules (AF, Philo, Q), the book name
            # from the parser is a direct key to the corpus work.  Use it to
            # build proper citations (e.g. "Ignatius, To the Ephesians 1.1")
            # instead of generic "Hermeneia AF §X" labels.
            ctx_corpus = ""
            ctx_work = ""
            _book_mapped = False
            if (
                _format == "hermeneia"
                and corpus_id
                and book in _HERMENEIA_BOOK_TO_CORPUS_WORK
            ):
                book_corpus_id, work_abbrev = _HERMENEIA_BOOK_TO_CORPUS_WORK[book]
                if book_corpus_id == corpus_id:
                    _book_mapped = True
                    ctx_corpus = corpus_id
                    ctx_work = work_abbrev
                    # Get human-readable work name
                    cw_name = ""
                    corpus_def = CORPUS_REGISTRY.get(corpus_id)
                    if corpus_def:
                        cw_obj = corpus_def.works.get(work_abbrev)
                        if cw_obj:
                            cw_name = cw_obj.name
                    if chapter > 0 and vs_start > 0:
                        # Commentary on a specific work reference
                        ref = format_corpus_ref(
                            corpus_id, work_abbrev, chapter, vs_start
                        )
                        if cw_name:
                            section_title = f"{cw_name} {chapter}.{vs_start}"
                        else:
                            section_title = ref
                    else:
                        # Top-level section (introduction, header, bibliography)
                        # — use the work name with a generic label
                        ref = f"{work_abbrev} intro"
                        if cw_name:
                            section_title = cw_name
                        else:
                            section_title = work_abbrev

            citation = commentary_citation(section_title, abbrev, title, ref)

            body_text = citation + " — " + clean
            if lang in ("el", "he"):
                body_norm = (
                    strip_diacritics(citation).lower()
                    + " — "
                    + strip_diacritics(clean).lower()
                )
            else:
                body_norm = citation.lower() + " — " + _stem_english(clean.lower())

            # ── Corpus-aware reference extraction for non-Bible modules ──
            # Only run when the book name wasn't already mapped via
            # _HERMENEIA_BOOK_TO_CORPUS_WORK above (to avoid overwriting
            # metadata that we know is correct from the parser).
            if is_generic and corpus_id and not _book_mapped:
                ctx_corpus = corpus_id
                work_ref = extract_work_ref(corpus_id, clean)
                if work_ref:
                    work_abbrev, w_ch, w_vs, w_sub = work_ref
                    ctx_work = work_abbrev
                    # Override the generic ref with a corpus-aware reference
                    ref = format_corpus_ref(corpus_id, work_abbrev, w_ch, w_vs, w_sub)
                    # Update section title so the citation reflects the work
                    cw_name = ""
                    corpus_def = CORPUS_REGISTRY.get(corpus_id)
                    if corpus_def:
                        cw_obj = corpus_def.works.get(work_abbrev)
                        if cw_obj:
                            cw_name = cw_obj.name
                    if cw_name:
                        section_title = f"{cw_name} {ref}"
                    else:
                        section_title = ref
                    # Rebuild citation with the corpus-aware title
                    citation = commentary_citation(section_title, abbrev, title, ref)

            batch.append(
                (
                    "commentary",
                    abbrev,
                    ref,
                    lang,
                    citation,
                    body_text,
                    body_norm,
                    book,
                    ctx_corpus,
                    ctx_work,
                )
            )

            if len(batch) >= self._BATCH_SIZE:
                self._insert_batch(conn, batch)
                count += len(batch)
                batch = []
                if count % 5000 == 0:
                    print(".", end="", flush=True)

        if batch:
            self._insert_batch(conn, batch)
            count += len(batch)

        self._mark_indexed("commentary", abbrev, "_all", mtime)
        return count

    def _index_catena_aurea(
        self, abbrev: str, mod, mtime: float, conn, entries: list, title: str
    ) -> int:
        """Index Catena Aurea commentary entries from pre-parsed JSON text."""
        batch = []
        count = 0

        for entry in entries:
            try:
                book, chapter, vs_start, vs_end, offset = entry
            except Exception:
                continue

            try:
                text = mod._get_index_text(entry)
            except Exception:
                continue

            if not text or not text.strip():
                continue

            ref = (
                f"{book} {chapter}:{vs_start}-{vs_end}"
                if vs_start != vs_end
                else f"{book} {chapter}:{vs_start}"
            )

            import re

            clean = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F]", "", text.strip())
            lang = detect_language(clean)

            if vs_start == vs_end:
                section_title = f"{book} {chapter}:{vs_start}"
            else:
                section_title = f"{book} {chapter}:{vs_start}-{vs_end}"

            citation = commentary_citation(section_title, abbrev, title, ref)

            body_text = citation + " — " + clean
            if lang in ("el", "he"):
                body_norm = (
                    strip_diacritics(citation).lower()
                    + " — "
                    + strip_diacritics(clean).lower()
                )
            else:
                body_norm = citation.lower() + " — " + _stem_english(clean.lower())

            batch.append(
                ("commentary", abbrev, ref, lang, citation, body_text, body_norm, book)
            )

            if len(batch) >= self._BATCH_SIZE:
                self._insert_batch(conn, batch)
                count += len(batch)
                batch = []
                if count % 5000 == 0:
                    print(".", end="", flush=True)

        if batch:
            self._insert_batch(conn, batch)
            count += len(batch)

        self._mark_indexed("commentary", abbrev, "_all", mtime)
        return count

    def _index_apparatus_as_commentary(
        self, abbrev: str, mod, mtime: float, conn, entries: list, title: str
    ) -> int:
        """Index apparatus entries under source_type="commentary".

        Apparatus modules (NA28, LXX, LXX2, MT-ETCBC-A) pre-load their
        text from ``*_apparatus_text.json`` files.  The entries list
        maps (book, chapter, verse, verse, json_entry_index) — the
        "offset" field is actually a JSON entry index, not a byte offset.
        Reads decoded_text from ``mod._apparatus_data`` and indexes it
        with proper book:chapter:verse citations.
        """

        batch = []
        count = 0

        app_data = getattr(mod, "_apparatus_data", None)
        verse_index = getattr(mod, "_apparatus_verse_index", None)
        if not app_data:
            return 0

        json_entries = app_data.get("entries", [])
        if not json_entries:
            return 0

        # Build reverse index: JSON entry index → [(verse_key, book)]
        # so we can produce proper verse citations for each apparatus entry.
        entry_to_verses: dict[int, list[tuple[str, str]]] = {}
        if verse_index:
            for verse_key, entry_indices in verse_index.items():
                if not entry_indices:
                    continue
                # Extract book name from verse key (e.g. "Matthew 1:1" → "Matthew")
                book = verse_key.rsplit(" ", 1)[0] if " " in verse_key else ""
                for ei in entry_indices:
                    if isinstance(ei, int) and ei >= 0:
                        entry_to_verses.setdefault(ei, []).append((verse_key, book))

        for entry in entries:
            try:
                book, chapter, vs_start, vs_end, json_idx = entry
            except Exception:
                continue

            if json_idx < 0 or json_idx >= len(json_entries):
                continue

            je = json_entries[json_idx]
            raw_text = je.get("decoded_text", "") or je.get("raw_text", "")
            if not raw_text or not raw_text.strip():
                continue

            # Skip header entries (§0-§2) that are binary metadata
            if json_idx <= 2:
                continue

            clean = self._sanitize_apparatus_text(raw_text.strip())

            # Skip entries that are too short or binary noise
            if len(clean) < 4:
                continue

            # Skip entries with too few "words" (consecutive letter-like runs).
            # Hebrew apparatus sigla (ℵ, 𝔐, etc.) are not alphabetic by
            # isalpha(), so we also check Hebrew, Greek, and Letterlike
            # Symbols ranges.  Threshold lowered to 1 for MT-ETCBC-A.
            min_words = 1 if abbrev == "MT-ETCBC-A" else 3
            alpha_words = 0
            in_word = False
            for ch in clean:
                cp = ord(ch)
                is_word_char = (
                    ch.isalpha()
                    or 0x0590 <= cp <= 0x05FF  # Hebrew
                    or 0x0370 <= cp <= 0x03FF  # Greek
                    or 0x1F00 <= cp <= 0x1FFF  # Greek Extended
                    or 0x2100 <= cp <= 0x214F  # Letterlike Symbols (ℵ etc.)
                )
                if is_word_char:
                    if not in_word:
                        alpha_words += 1
                        in_word = True
                else:
                    in_word = False
            if alpha_words < min_words:
                continue

            # Build verse citation from reverse index, falling back to
            # the entry's book/chapter/verse from _entries if available.
            verselist = entry_to_verses.get(json_idx, [])
            if verselist:
                ref = verselist[0][0]
                # Use the first verse key's book name
                book = verselist[0][1]
            elif book:
                ref = (
                    f"{book} {chapter}:{vs_start}-{vs_end}"
                    if vs_start != vs_end
                    else f"{book} {chapter}:{vs_start}"
                )
            else:
                ref = f"{title} §{json_idx}"

            lang = self._APPARATUS_SOURCES.get(abbrev, "en")

            if vs_start == vs_end:
                section_title = f"{book} {chapter}:{vs_start}"
            else:
                section_title = f"{book} {chapter}:{vs_start}-{vs_end}"

            citation = commentary_citation(section_title, abbrev, title, ref)

            body_text = citation + " — " + clean
            if lang in ("el", "he"):
                body_norm = (
                    strip_diacritics(citation).lower()
                    + " — "
                    + strip_diacritics(clean).lower()
                )
            else:
                body_norm = citation.lower() + " — " + clean.lower()

            batch.append(
                ("commentary", abbrev, ref, lang, citation, body_text, body_norm, book)
            )

            if len(batch) >= self._BATCH_SIZE:
                self._insert_batch(conn, batch)
                count += len(batch)
                batch = []
                if count % 5000 == 0:
                    print(".", end="", flush=True)

        if batch:
            self._insert_batch(conn, batch)
            count += len(batch)

        self._mark_indexed("commentary", abbrev, "_all", mtime)
        return count

    # ------------------------------------------------------------------
    # _index_acc2
    # ------------------------------------------------------------------

    def _index_acc2(self, modules: dict) -> int:
        """Index .acc2 (type 15) module paragraphs using the module's entry reader."""
        total = 0
        for abbr, mod in sorted(modules.items()):
            try:
                path = mod.filepath
                mtime = path.stat().st_mtime if path.exists() else 0
            except Exception:
                mtime = 0

            if not self._needs_reindex("acc2", abbr, mtime):
                continue

            print(f"  Indexing ACC2: {abbr} ...", end="", flush=True)
            self._clear_source("acc2", abbr)
            conn = self._ensure_conn()
            batch = []
            count = 0
            title = getattr(mod, "title", abbr)
            try:
                entries = mod.entries()
            except Exception:
                self._mark_indexed("acc2", abbr, "_all", mtime)
                continue
            for num, entry_title, text in entries:
                if not text or not text.strip():
                    continue
                source_id = f"p{num}"
                citation = f'"{entry_title}" in {title} (\u00b6{num})'
                clean = entry_title + " " + text.strip()
                lang = detect_language(clean)
                if lang in ("el", "he"):
                    norm = (
                        strip_diacritics(citation).lower()
                        + " — "
                        + strip_diacritics(clean).lower()
                    )
                else:
                    norm = citation.lower() + " — " + _stem_english(clean.lower())
                body_text = citation + " — " + clean
                batch.append(
                    ("acc2", abbr, source_id, lang, citation, body_text, norm, "")
                )
                if len(batch) >= self._BATCH_SIZE:
                    self._insert_batch(conn, batch)
                    count += len(batch)
                    batch = []
                    if count % 5000 == 0:
                        print(".", end="", flush=True)
            if batch:
                self._insert_batch(conn, batch)
                count += len(batch)
            self._mark_indexed("acc2", abbr, "_all", mtime)
            print(f" {count} paragraphs")
            total += count
        return total

    # ------------------------------------------------------------------
    # _index_bolls_cache  --  cached bolls.life translations
    # ------------------------------------------------------------------

    def _index_bolls_cache(self) -> int:
        """Index cached Bible translations from the bolls.life cache.

        Indexes EVERY translation in the local cache (not just public domain),
        since this is for personal/non-commercial study use.

        Returns:
            Total number of verses indexed.
        """
        from accordance_parser.bolls_cache import BollsCache
        from accordance_parser.bible_api import BOOK_ID_TO_NAME

        cache = BollsCache()
        cached = cache.list_cached_translations()
        if not cached:
            return 0

        BATCH = 1000
        total = 0

        for code in cached:
            data = cache._load(code)
            if not data:
                continue

            mtime = (cache.cache_dir / f"{code}.json").stat().st_mtime
            source_name = f"bolls_cache:{code}"

            # Skip if already indexed and file hasn't changed
            if not self._needs_reindex("bible", source_name, mtime):
                continue

            print(f"  Indexing Bolls Cache: {code} ...", end="", flush=True)
            self._clear_source("bible", source_name)

            conn = self._ensure_conn()
            batch: list[tuple] = []
            count = 0

            for verse in data:
                book_id = verse.get("book")
                chapter = verse.get("chapter")
                verse_num = verse.get("verse")
                verse_text = verse.get("text", "")

                if book_id is None or chapter is None or verse_num is None:
                    continue
                if not verse_text or not verse_text.strip():
                    continue

                book_name = BOOK_ID_TO_NAME.get(book_id, f"Book {book_id}")
                ref = f"{book_name} {chapter}:{verse_num}"
                clean = strip_strongs_tags(verse_text)

                if not clean.strip():
                    continue

                lang = detect_language(clean)
                if lang in ("el", "he"):
                    norm = strip_diacritics(clean).lower()
                elif lang == "zh":
                    norm = separate_cjk_chars(clean.lower())
                else:
                    norm = clean.lower()

                citation = f"{ref} ({source_name})"
                body_text = citation + " \u2014 " + clean
                if lang in ("el", "he"):
                    body_norm = strip_diacritics(citation).lower() + " \u2014 " + norm
                elif lang == "zh":
                    body_norm = separate_cjk_chars(citation.lower()) + " \u2014 " + norm
                else:
                    body_norm = citation.lower() + " \u2014 " + norm

                batch.append(
                    (
                        "bible",
                        source_name,
                        ref,
                        lang,
                        citation,
                        body_text,
                        body_norm,
                        book_name,
                    )
                )

                if len(batch) >= BATCH:
                    self._insert_batch(conn, batch)
                    count += len(batch)
                    batch = []
                    if count % 5000 == 0:
                        print(".", end="", flush=True)

            if batch:
                self._insert_batch(conn, batch)
                count += len(batch)

            self._mark_indexed("bible", source_name, "_all", mtime)
            print(f" {count} verses")
            total += count

        return total

    # ------------------------------------------------------------------
    # _index_openscriptures  --  Strong's lexicon from Open Scriptures
    # ------------------------------------------------------------------

    def _index_openscriptures(self) -> int:
        """Index Strong's Hebrew and Greek definitions from Open Scriptures.

        Reads the Hebrew Strong's XML (OSIS namespace) and Greek Strong's
        XML (DTD format) from the OpenScripturesClient data directory.
        Each entry is inserted as source_type="lexicon".

        Returns:
            Total number of lexicon entries indexed.
        """
        from accordance_parser.openscriptures import OpenScripturesClient

        _OSIS_NS = "http://www.bibletechnologies.net/2003/OSIS/namespace"

        client = OpenScripturesClient()

        # Check data availability without triggering download
        if not client.available():
            return 0

        TOTAL = 0

        # ── Hebrew Strong's ──
        he_path = client._strongs_he_path
        if he_path.exists():
            try:
                import xml.etree.ElementTree as ET

                tree = ET.parse(str(he_path))
                root = tree.getroot()
            except ET.ParseError:
                root = None

            if root is not None:
                mtime = he_path.stat().st_mtime
                source_name = "openscriptures_strongs"

                if self._needs_reindex("lexicon", source_name, mtime):
                    print(
                        "  Indexing OpenScriptures: Hebrew Strong's ...",
                        end="",
                        flush=True,
                    )
                    self._clear_source("lexicon", source_name)

                    conn = self._ensure_conn()
                    batch: list[tuple] = []
                    count = 0

                    for div_elem in root.iter(f"{{{_OSIS_NS}}}div"):
                        if div_elem.get("type") != "entry":
                            continue
                        n = div_elem.get("n", "")
                        if not n:
                            continue

                        strongs_num = f"H{n}"

                        # Extract lemma and definition
                        lemma = ""
                        xlit = ""
                        def_parts: list[str] = []
                        for child in div_elem:
                            tag_local = (
                                child.tag.split("}", 1)[-1]
                                if "}" in child.tag
                                else child.tag
                            )
                            if tag_local == "w":
                                lemma = child.get("lemma", "")
                                xlit = child.get("xlit", "")
                            elif tag_local in ("seg", "note"):
                                def_parts.append("".join(child.itertext()).strip())

                        definition = " ".join(def_parts).strip()
                        if not definition:
                            continue

                        body_text = (
                            f"{lemma} ({xlit}) \u2014 {definition}"
                            if lemma
                            else definition
                        )
                        body_norm = body_text.lower()

                        citation = strongs_num

                        batch.append(
                            (
                                "lexicon",
                                source_name,
                                strongs_num,
                                "he",
                                citation,
                                body_text,
                                body_norm,
                                "",
                            )
                        )

                        if len(batch) >= self._BATCH_SIZE:
                            self._insert_batch(conn, batch)
                            count += len(batch)
                            batch = []
                            if count % 5000 == 0:
                                print(".", end="", flush=True)

                    if batch:
                        self._insert_batch(conn, batch)
                        count += len(batch)

                    self._mark_indexed("lexicon", source_name, "_all", mtime)
                    print(f" {count} entries")
                    TOTAL += count

        # ── Greek Strong's ──
        gr_path = client._strongs_gr_path
        if gr_path.exists():
            try:
                import xml.etree.ElementTree as ET

                tree = ET.parse(str(gr_path))
                root = tree.getroot()
            except ET.ParseError:
                root = None

            if root is not None:
                mtime = gr_path.stat().st_mtime
                source_name = "openscriptures_strongs"

                if self._needs_reindex("lexicon", source_name, mtime):
                    print(
                        "  Indexing OpenScriptures: Greek Strong's ...",
                        end="",
                        flush=True,
                    )
                    self._clear_source("lexicon", source_name)

                    conn = self._ensure_conn()
                    batch: list[tuple] = []
                    count = 0

                    for entry_elem in root.iter("entry"):
                        strongs_padded = entry_elem.get("strongs", "")
                        if not strongs_padded:
                            continue
                        n = strongs_padded.lstrip("0") or "0"
                        strongs_num = f"G{n}"

                        # Extract lemma and definition
                        lemma = ""
                        xlit = ""
                        def_parts: list[str] = []
                        for child in entry_elem:
                            if child.tag == "greek":
                                lemma = child.get("unicode", "")
                                xlit = child.get("translit", "")
                            elif child.tag == "strongs_def":
                                def_parts.append("".join(child.itertext()).strip())
                            elif child.tag == "kjv_def":
                                def_parts.append(
                                    "[KJV: " + "".join(child.itertext()).strip() + "]"
                                )

                        definition = " ".join(def_parts).strip()
                        if not definition:
                            continue

                        body_text = (
                            f"{lemma} ({xlit}) \u2014 {definition}"
                            if lemma
                            else definition
                        )
                        body_norm = body_text.lower()

                        citation = strongs_num

                        batch.append(
                            (
                                "lexicon",
                                source_name,
                                strongs_num,
                                "el",
                                citation,
                                body_text,
                                body_norm,
                                "",
                            )
                        )

                        if len(batch) >= self._BATCH_SIZE:
                            self._insert_batch(conn, batch)
                            count += len(batch)
                            batch = []
                            if count % 5000 == 0:
                                print(".", end="", flush=True)

                    if batch:
                        self._insert_batch(conn, batch)
                        count += len(batch)

                    self._mark_indexed("lexicon", source_name, "_all", mtime)
                    print(f" {count} entries")
                    TOTAL += count

        return TOTAL

    # ------------------------------------------------------------------
    # Batch insert
    # ------------------------------------------------------------------

    def _insert_batch(self, conn: sqlite3.Connection, batch: list[tuple]) -> int:
        """Insert a batch of rows into search_content.

        Accepts 8-tuples (legacy) or 10-tuples (with corpus, work columns):
          8-tuple: (source_type, source_name, source_id, lang, citation,
                     text, norm, book)
          10-tuple: same + (corpus, work)

        Normalizes body_text and body_norm by stripping C0 control characters
        before insertion.  8-tuples are padded to 10 columns with '' values.
        Returns number of rows that failed to insert (0 on success).
        """
        if not batch:
            return 0

        col_sql = (
            "source_type, source_name, source_id, language, citation, "
            "body_text, body_norm, book, corpus, work"
        )
        val_placeholders = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"

        # Strip C0 control characters from body_text and body_norm.
        # Pad 8-tuples to 10 columns with '' for corpus and work.
        normalized_batch: list[tuple] = []
        for row in batch:
            lst = list(row)
            if lst[5]:  # body_text
                lst[5] = strip_c0_controls(lst[5])
            if lst[6]:  # body_norm
                lst[6] = strip_c0_controls(lst[6])
            # Pad to 10 elements if 8-tuple
            while len(lst) < 10:
                lst.append("")
            normalized_batch.append(tuple(lst[:10]))

        try:
            conn.executemany(
                f"INSERT INTO search_content ({col_sql}) VALUES {val_placeholders}",
                normalized_batch,
            )
            conn.commit()
            return 0
        except sqlite3.Error:
            # If batch insert fails, try row-by-row to skip bad rows
            conn.rollback()
            failed = 0
            for row in normalized_batch:
                try:
                    conn.execute(
                        f"INSERT INTO search_content ({col_sql}) VALUES {val_placeholders}",
                        row,
                    )
                except sqlite3.Error as e:
                    source_type, source_name, source_id = row[0], row[1], row[2]
                    logger.warning(
                        "Failed to insert row in batch: source_type=%s, source_name=%s, source_id=%s: %s",
                        source_type,
                        source_name,
                        source_id,
                        e,
                    )
                    failed += 1
            conn.commit()
            return failed

    # ------------------------------------------------------------------
    # Existing data cleanup
    # ------------------------------------------------------------------

    def clean_existing_c0(self) -> int:
        """Strip C0 control characters from existing body_norm rows.

        Called once after the code-level fix to `_insert_batch` to clean
        rows that were inserted before C0 stripping was added.  Operates
        in batches of 500 to keep the WAL from growing.

        Returns:
            Number of rows updated.
        """
        conn = self._ensure_conn()
        total = 0
        batch_size = self._BATCH_SIZE

        while True:
            # Find rows still containing C0 controls (0x00-0x08, 0x0B-0x0C, 0x0E-0x1F)
            rows = conn.execute(
                "SELECT rowid, body_norm FROM search_content "
                "WHERE body_norm GLOB '*[' || char(0) || '-' || char(8) || char(11) || char(12) || char(14) || '-' || char(31) || ']*' "
                "LIMIT ?",
                (batch_size,),
            ).fetchall()

            if not rows:
                break

            updates = []
            for rowid, body_norm in rows:
                cleaned = strip_c0_controls(body_norm)
                updates.append((cleaned, rowid))

            conn.executemany(
                "UPDATE search_content SET body_norm = ? WHERE rowid = ?",
                updates,
            )
            conn.commit()
            total += len(updates)
            print(f"  Cleaned {total} rows...", end="\r", flush=True)

        if total:
            print(f"  Cleaned {total} rows total.           ")
            # Rebuild FTS5 to reflect the cleaned body_norm values
            conn.execute("INSERT INTO search_idx(search_idx) VALUES('rebuild')")
            conn.commit()
            # Merge b-tree segments for query performance
            conn.execute("INSERT INTO search_idx(search_idx) VALUES('optimize')")
            conn.commit()
        return total

    # ------------------------------------------------------------------
    # Search
    # ------------------------------------------------------------------

    def search(
        self,
        query: str,
        source_type: str = "all",
        language: str = "any",
        max_results: int = 20,
        fuzzy: bool = True,
        book: str = "",
        corpus: str = "",
        snippet: bool = True,
    ) -> list[tuple]:
        """Search the FTS5 index.

        Args:
            query: The search query string.
            source_type: 'all', 'bible', 'lexicon', 'commentary', or
                        comma-separated (e.g. 'commentary,apparatus').
            language: 'any', 'en', 'el', or 'he'.
            max_results: Maximum number of results to return.
            fuzzy: If True, strip diacritics from query for Greek/Hebrew.
            book: Optional book name filter (e.g. 'Genesis'). Empty string
                  means no book filter.
            corpus: Optional corpus filter (e.g. 'apostolic_fathers').
                    Empty string means no corpus filter.
            snippet: If False, skip FTS5 snippet generation (faster for
                     callers that don't need highlighted context). The
                     snippet field will be an empty string.

        Returns:
            List of (source_type, source_name, source_id, snippet, citation, rank)
            tuples.
        """
        conn = self._ensure_conn()
        lang = language.lower()

        # --- Detect prefix query (trailing '*') ---
        # e.g. "bapti*" → prefix search for tokens starting with "bapti"
        is_prefix = query.rstrip().endswith("*")
        if is_prefix:
            # Strip the trailing '*' before cleaning
            query = query.rstrip()[:-1]

        # --- Detect phrase search (double-quoted) ---
        # e.g. '"kingdom of God"' → exact phrase match in FTS5
        is_phrase = query.strip().startswith('"') and query.strip().endswith('"')
        if is_phrase:
            # Strip the outer quotes for cleaning, but re-add them in the
            # FTS5 MATCH expression below.
            query = query.strip()[1:-1]

        # Replace colons to prevent FTS5 column:value syntax errors.
        # ("John 3:16" → "John 3 16"; see Bug 1)
        query = query.replace(":", " ")

        # Escape FTS5 special characters.  Preserve single-quotes via SQL escaping.
        # Double-quotes are stripped unless this is a phrase query.
        if is_phrase:
            clean = query.replace("'", "''")
        else:
            clean = query.replace('"', "").replace("'", "''")

        # Re-add phrase quotes for FTS5 MATCH syntax
        if is_phrase:
            clean_for_fts = f'"{clean}"'
        else:
            clean_for_fts = clean

        # Append '*' for prefix queries
        if is_prefix:
            clean_for_fts += "*"

        # --- Detect CJK query: insert spaces between CJK characters ---
        # CJK text is stored in body_norm with spaces between characters
        # (see separate_cjk_chars).  Queries must be split the same way.
        cjk_mode = lang == "zh" or (lang == "any" and has_cjk_chars(clean))
        if cjk_mode:
            clean = separate_cjk_chars(clean.lower())
            clean_for_fts = separate_cjk_chars(clean_for_fts.lower())
            # Re-apply phrase/prefix wrapping after CJK separation
            if is_phrase:
                clean_for_fts = f'"{clean_for_fts}"'
            if is_prefix:
                clean_for_fts += "*"

        # Build the FTS5 MATCH expression
        fuzzy_mode = lang in ("el", "he") and fuzzy
        if cjk_mode:
            # Search body_norm (CJK-separated text stored there)
            fts_query = f"{{body_norm}}: {clean_for_fts}"
        elif fuzzy_mode:
            norm = strip_diacritics(clean)
            # Re-apply phrase/prefix wrapping to the normalized form
            if is_phrase:
                norm_for_fts = f'"{norm}"'
            else:
                norm_for_fts = norm
            if is_prefix:
                norm_for_fts += "*"
            # Search body_norm only for fair ranking across all sources.
            # Some sources have accented Greek (TISCH, LXX), others have
            # unaccented (OpenGNT, TR). Searching only the normalized column
            # ensures BM25 ranks them equally.
            fts_query = f"{{body_norm}}: {norm_for_fts}"
        else:
            # English or non-fuzzy: search body_text only
            fts_query = f"{{body_text}}: {clean_for_fts}"

        # Build WHERE clause for content-table filters
        where_parts = []
        params = []

        if source_type != "all":
            types = [t.strip() for t in source_type.split(",") if t.strip()]
            if types:
                placeholders = ",".join("?" * len(types))
                where_parts.append(f"sc.source_type IN ({placeholders})")
                params.extend(types)

        if lang != "any":
            where_parts.append("sc.language = ?")
            params.append(lang)

        if book:
            where_parts.append("sc.book = ?")
            params.append(book)

        if corpus:
            where_parts.append("sc.corpus = ?")
            params.append(corpus)

        where_clause = ""
        if where_parts:
            where_clause = " AND " + " AND ".join(where_parts)

        snippet_col = 1 if fuzzy_mode else 0
        if snippet:
            snippet_expr = (
                f"snippet(search_idx, {snippet_col}, '<mark>', '</mark>', '…', 32)"
            )
        else:
            snippet_expr = "''"
        sql = f"""
            SELECT sc.source_type, sc.source_name, sc.source_id,
                   {snippet_expr} AS snippet,
                   sc.citation,
                   rank
            FROM search_idx
            JOIN search_content sc ON search_idx.rowid = sc.rowid
            WHERE search_idx MATCH ?{where_clause}
            ORDER BY rank
            LIMIT ?
        """
        params.insert(0, fts_query)
        params.append(max_results)

        try:
            rows = conn.execute(sql, params).fetchall()
        except sqlite3.OperationalError:
            # FTS5 syntax error — fall back to LIKE on body_text with all
            # original filters intact
            fallback_params = []
            fallback_parts = ["body_text LIKE ?"]
            fallback_params.append(f"%{clean}%")
            if source_type != "all":
                types = [t.strip() for t in source_type.split(",") if t.strip()]
                if types:
                    placeholders = ",".join("?" * len(types))
                    fallback_parts.append(f"source_type IN ({placeholders})")
                    fallback_params.extend(types)
            if language != "any":
                fallback_parts.append("language = ?")
                fallback_params.append(language)
            if book:
                fallback_parts.append("book = ?")
                fallback_params.append(book)
            if corpus:
                fallback_parts.append("corpus = ?")
                fallback_params.append(corpus)
            where_clause2 = " AND ".join(fallback_parts)
            sql2 = f"""
                SELECT source_type, source_name, source_id,
                       body_text,
                       citation,
                       CAST(rowid AS REAL) AS rank
                FROM search_content
                WHERE {where_clause2}
                ORDER BY rowid
                LIMIT ?
            """
            rows = conn.execute(sql2, fallback_params + [max_results]).fetchall()
            # Reformat to 6-tuple shape; no FTS5 snippet available
            rows = [(r[0], r[1], r[2], r[3][:300], r[4], r[5]) for r in rows]

        # Apply citation-aware ranking: demote generic tools, promote scholarly
        if rows:
            rows = self._apply_quality_boost(rows)

        return rows

    def _apply_quality_boost(self, rows: list[tuple]) -> list[tuple]:
        """Apply source-quality AND source-type multipliers to ranks.

        effective_rank = raw_rank * quality_boost * type_boost
        Lower effective_rank = better.

        Source-type boost (from _SOURCE_TYPE_BOOST) gives priority to
        bible (0.8) over lexicons (0.9), commentaries (1.0), and
        apparatus/acc2 (1.15-1.3).
        Source-quality boost (from _SOURCE_QUALITY_BOOST) promotes
        scholarly references within the same type (e.g. BDAG over Strong's).
        """
        boosted = []
        for r in rows:
            raw_rank = r[5]
            source_type = r[0]
            source_name = r[1]
            quality = self._SOURCE_QUALITY_BOOST.get(source_name, 1.0)
            type_b = self._SOURCE_TYPE_BOOST.get(source_type, 1.0)
            effective = raw_rank / (quality * type_b)
            boosted.append((r[0], r[1], r[2], r[3], r[4], effective))
        boosted.sort(key=lambda x: x[5])
        return boosted

    def search_lexicons(
        self,
        query: str,
        language: str = "any",
        max_results: int = 20,
        fuzzy: bool = True,
        snippet: bool = True,
    ) -> list[dict]:
        """Search lexicon entries only, returning citation-enriched results.

        Convenience wrapper around search() that returns structured dicts
        with meaningful keys instead of raw tuples.

        Args:
            query: Search query string.
            language: 'any', 'el' (Greek), or 'he' (Hebrew).
            max_results: Maximum results.
            fuzzy: Strip diacritics from query for Greek/Hebrew fuzzy
                   matching.
            snippet: If False, skip FTS5 snippet generation (faster).

        Returns:
            List of dicts with keys: source_name, headword, citation,
            snippet, rank.
        """
        raw = self.search(
            query, "lexicon", language, max_results, fuzzy, snippet=snippet
        )
        return [
            {
                "source_name": r[1],  # e.g. "BDAG"
                "headword": r[2],  # e.g. "λόγος"
                "citation": r[4],  # e.g. '"λόγος" in BDAG (A Greek-English Lexicon...)'
                "snippet": r[3],  # FTS5 highlighted snippet
                "rank": r[5],  # BM25 rank (lower = better)
            }
            for r in raw
        ]

    def get_full_entry(self, source_name: str, source_id: str) -> str | None:
        """Retrieve the full body_text for a source+source_id pair.

        Args:
            source_name: Module abbreviation (e.g. 'BDAG')
            source_id: Entry ID (e.g. 'λόγος' for lexicon, 'John 3:16' for bible)

        Returns:
            Full body_text string or None if not found.
        """
        conn = self._ensure_conn()
        row = conn.execute(
            "SELECT body_text FROM search_content WHERE source_name=? AND source_id=? LIMIT 1",
            (source_name, source_id),
        ).fetchone()
        return row[0] if row else None

    def search_transliterated(
        self,
        query: str,
        source_type: str = "all",
        language: str = "any",
        max_results: int = 20,
        book: str = "",
        corpus: str = "",
        snippet: bool = True,
    ) -> list[tuple]:
        """Search with transliteration support for Greek/Hebrew queries.

        If the query looks like an ASCII transliteration of Greek or Hebrew
        (e.g. 'logos', 'agape', 'pneuma'), also searches for the native-script
        equivalent and merges results, preferring native-script matches.

        Args:
            query: Search query string (possibly transliterated).
            source_type, language, max_results, book: Same as search().
            snippet: If False, skip FTS5 snippet generation (faster).

        Returns:
            List of (source_type, source_name, source_id, snippet, citation, rank)
            tuples, merged and deduplicated.
        """
        is_translit, lang_hint = Transliterator.classify(query)

        if not is_translit:
            # Not a transliteration — fall through to normal search
            return self.search(
                query,
                source_type,
                language,
                max_results,
                fuzzy=(language in ("el", "he")),
                book=book,
                corpus=corpus,
                snippet=snippet,
            )

        # Determine target language for reverse transliteration
        if language != "any" and language in ("el", "he"):
            target_lang = language
        elif lang_hint:
            target_lang = lang_hint
        else:
            target_lang = "el"  # default to Greek

        # Reverse-transliterate the query to native script (with variants)
        if target_lang == "he":
            native_queries = [Transliterator.to_native(query, "he")]
        else:
            native_queries = Transliterator.to_native_variants(query, "el")

        # Search with each native-script variant (fuzzy = True)
        seen: set[tuple[str, str]] = set()
        native_results: list[tuple] = []
        for nq in native_queries:
            for r in self.search(
                nq,
                source_type,
                target_lang,
                max_results,
                fuzzy=True,
                book=book,
                corpus=corpus,
                snippet=snippet,
            ):
                key = (r[1], r[2])
                if key not in seen:
                    native_results.append(r)
                    seen.add(key)

        # Also search with the original ASCII query as a fallback
        ascii_results = self.search(
            query,
            source_type,
            language if language != "any" else target_lang,
            max_results,
            fuzzy=False,
            book=book,
            corpus=corpus,
            snippet=snippet,
        )

        # Merge: prefer native results, append ASCII results not already seen
        merged = list(native_results)
        for r in ascii_results:
            key = (r[1], r[2])
            if key not in seen:
                merged.append(r)
                seen.add(key)

        # Re-sort by rank (lower is better)
        merged.sort(key=lambda x: x[5])

        return merged[:max_results]

    @staticmethod
    def _escape_fts5(query: str) -> str:
        """Escape special characters for FTS5 query strings."""
        # FTS5 special chars: * " and characters outside the BMP
        return query.replace('"', '""')

    def get_status(self) -> dict:
        """Return index health and statistics.

        Returns dict with keys:
            db_path: str
            total_rows: int
            last_build: float (timestamp of most recent index_meta.indexed_at)
            sources: dict of source_type -> count
            commentary_books: list of unique book names in the commentary index
            bible_books: list of unique book names in the bible index
        """
        conn = self._ensure_conn()
        status = {
            "db_path": self.db_path,
            "total_rows": 0,
            "last_build": None,
            "sources": {},
            "commentary_books": [],
            "bible_books": [],
        }

        # Check if table exists at all
        try:
            exists = conn.execute(
                "SELECT name FROM sqlite_master WHERE type='table' AND name='search_content'"
            ).fetchone()
        except sqlite3.Error:
            return status

        if not exists:
            return status

        # Total rows
        row = conn.execute("SELECT COUNT(*) FROM search_content").fetchone()
        if row:
            status["total_rows"] = row[0]

        # Last build timestamp
        row = conn.execute("SELECT MAX(indexed_at) FROM index_meta").fetchone()
        if row and row[0] is not None:
            status["last_build"] = row[0]

        # Source type counts
        rows = conn.execute(
            "SELECT source_type, COUNT(*) FROM search_content GROUP BY source_type"
        ).fetchall()
        status["sources"] = {r[0]: r[1] for r in rows}

        # Per-source-name breakdown (e.g. bolls_cache:KJV, openscriptures_strongs)
        rows = conn.execute(
            "SELECT source_name, COUNT(*) FROM search_content "
            "WHERE source_name LIKE 'bolls_cache:%' OR source_name = 'openscriptures_strongs' "
            "GROUP BY source_name ORDER BY source_name"
        ).fetchall()
        status["source_names"] = {r[0]: r[1] for r in rows}

        # Commentary books
        rows = conn.execute(
            "SELECT DISTINCT book FROM search_content WHERE source_type='commentary' AND book != '' ORDER BY book"
        ).fetchall()
        status["commentary_books"] = [r[0] for r in rows]

        # Bible books
        rows = conn.execute(
            "SELECT DISTINCT book FROM search_content WHERE source_type='bible' AND book != '' ORDER BY book"
        ).fetchall()
        status["bible_books"] = [r[0] for r in rows]

        return status


# ---------------------------------------------------------------------------
# Quick-test helper (runs when invoked directly)
# ---------------------------------------------------------------------------

if __name__ == "__main__":
    si = SearchIndex()
    print(f"DB path: {si.db_path}")
    print("Schema ready.")

    # Quick size check
    row_count = (
        si._ensure_conn().execute("SELECT COUNT(*) FROM search_content").fetchone()[0]
    )
    print(f"Indexed rows: {row_count}")

    if row_count > 0:
        # --- Raw tuple search with citation ---
        print("\n--- Test: 'λόγος' in lexicons (raw search) ---")
        results = si.search("λόγος", "lexicon", "el", 5, fuzzy=True)
        for r in results:
            # 6-tuple: (source_type, source_name, source_id, snippet, citation, rank)
            print(f"  [{r[0]}] {r[1]}:{r[2]}  rank={r[5]:.2f}")
            print(f"    cite: {r[4]}")
            print(f"    {r[3][:150]}")

        print("\n--- Test: 'baptism' in all (raw search) ---")
        results = si.search("baptism", "all", "en", 5, fuzzy=False)
        for r in results:
            print(f"  [{r[0]}] {r[1]}:{r[2]}  rank={r[5]:.2f}")
            print(f"    cite: {r[4]}")
            print(f"    {r[3][:150]}")

        # --- Structured lexicon search ---
        print("\n--- Test: search_lexicons('λόγος', language='el') ---")
        lex_results = si.search_lexicons("λόγος", language="el", max_results=5)
        for entry in lex_results:
            print(
                f"  {entry['source_name']}: {entry['headword']} "
                f" (rank={entry['rank']:.2f})"
            )
            print(f"    citation: {entry['citation']}")
            print(f"    snippet:  {entry['snippet'][:120]}")

        print("\n--- Test: search_lexicons('baptism', language='en') ---")
        lex_results = si.search_lexicons(
            "baptism", language="en", max_results=5, fuzzy=False
        )
        for entry in lex_results:
            print(
                f"  {entry['source_name']}: {entry['headword']} "
                f" (rank={entry['rank']:.2f})"
            )
            print(f"    citation: {entry['citation']}")
            print(f"    snippet:  {entry['snippet'][:120]}")

    si.close()


# ---------------------------------------------------------------------------
# Module-level convenience
# ---------------------------------------------------------------------------

_index_instance: SearchIndex | None = None


def get_index() -> SearchIndex:
    """Get or create the global SearchIndex singleton."""
    global _index_instance
    if _index_instance is None:
        _index_instance = SearchIndex()
    return _index_instance
