#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
sugyot_to_topics.py — a "מראי מקומות" PDF (דפי סוגיות and the like) -> a topic file
that Otzaria's "מקורות ונושאים" plugin (topic-sources) imports.

    PDF lines ──layout──▶ headings + source lines ──parse──▶ citations
              ──otzaria-mcp──▶ (book, line) + text ──▶ topic JSON

  * Headings become נושאים / תתי-נושאים. Their level comes from the font: the
    styles bigger (or bolder) than the body text, biggest first; inside one style,
    unnumbered headings ("שיטת רש"י") sit one level below the numbered ones
    ("א. ...") — or become חוצצים with --minor-as divider.
  * Running headers (page titles, page numbers) are dropped by style/position.
  * Every source line is parsed into citations (work, masechet, daf / siman / perek /
    halacha ..., ד"ה, עד, שם), with the sugya's context: the masechet, the dafim of
    the סוגיא's opening lines (for "רש"י ד"ה X" with no daf), the last SA siman,
    the last Rambam hilchot/perek, "שם", "דלעיל/הנ"ל".
  * Each citation is located in Otzaria through otzaria-mcp (the same server
    ency_pairs.py uses, found the same way in claude_desktop_config.json):
    Talmud commentaries by book + daf (TOC) + ד"ה match; Rambam / SA / responsa /
    everything else through find-ref (resolve_ref), checked against the named work
    and the cited number, with a TOC walk as fallback. The text of the located
    line(s) is in-lined into the source.
  * Whatever does not resolve is still copied, as a "מקור ידני" carrying the
    citation as written, and listed in <out>.review.tsv. A citation whose place was
    found but whose ד"ה was not gets the place, and a title starting with ⚠.

Output (default): one "שיתוף לאוצריא" file  <pdf-stem>.topics.json
    → in the plugin: ייבוא → ייבוא מאוצריא → בחירת קובץ.
--split writes one such file per top-level heading; --format backup writes a
backup file instead (גיבוי ושחזור → שחזור מגיבוי → הוספה לנתונים הקיימים), which
adds every top-level heading as a separate topic in one go.

Needs: Python 3.8+, PyMuPDF (pip install pymupdf) or pdfplumber; otzaria-mcp
(auto-discovered, or --mcp-cmd). Everything otzaria-mcp answers is cached in
sqlite (--cache), so re-runs are fast.

    python sugyot_to_topics.py "C:\\work\\lib\\דפי סוגיות - שבת.pdf"
    python sugyot_to_topics.py file.pdf --dry-run            # layout + parse only
    python sugyot_to_topics.py file.pdf --masechet שבת --split --minor-as divider
"""
import argparse
import collections
import datetime
import json
import os
import queue
import random
import re
import sqlite3
import subprocess
import sys
import threading
import time
import unicodedata

VERSION = '0.1.0'
HEB = 'א-ת'


def log(*a):
    print(*a, file=sys.stderr, flush=True)


# ====================================================================== text
NIQQUD_RE = re.compile('[\u0591-\u05C7]')
BIDI_RE = re.compile('[\u200e\u200f\u202a-\u202e\u2066-\u2069\ufeff]')
FINALS = str.maketrans('ךםןףץ', 'כמנפצ')


def norm_he(s):
    s = unicodedata.normalize('NFKC', '' if s is None else str(s))
    s = re.sub('[״“”„]', '"', s)
    s = re.sub("[׳‘’`´]", "'", s)
    s = s.replace('–', '-').replace('—', '-').replace('\u05be', '-')
    s = BIDI_RE.sub('', s)
    return re.sub(r'\s+', ' ', s).strip()


def name_key(s):
    return norm_he(NIQQUD_RE.sub('', s or '')).replace('"', '').replace("'", '')


def plain_tokens(s):
    """Hebrew word tokens without niqqud/quotes, finals folded — for matching."""
    s = re.sub(r'<[^>]+>', ' ', s or '')
    s = NIQQUD_RE.sub('', norm_he(s))
    s = re.sub('[^%s"\' ]+' % HEB, ' ', s)
    return [t.replace('"', '').replace("'", '').translate(FINALS) for t in s.split()
            if t.replace('"', '').replace("'", '')]


HE_VALUE = {'א': 1, 'ב': 2, 'ג': 3, 'ד': 4, 'ה': 5, 'ו': 6, 'ז': 7, 'ח': 8, 'ט': 9,
            'י': 10, 'כ': 20, 'ל': 30, 'מ': 40, 'נ': 50, 'ס': 60, 'ע': 70, 'פ': 80,
            'צ': 90, 'ק': 100, 'ר': 200, 'ש': 300, 'ת': 400,
            'ך': 20, 'ם': 40, 'ן': 50, 'ף': 80, 'ץ': 90}
HE_UNITS = ['', 'א', 'ב', 'ג', 'ד', 'ה', 'ו', 'ז', 'ח', 'ט']
HE_TENS = ['', 'י', 'כ', 'ל', 'מ', 'נ', 'ס', 'ע', 'פ', 'צ']
HE_HUNDREDS = ['', 'ק', 'ר', 'ש', 'ת', 'תק', 'תר', 'תש', 'תת', 'תתק']


def heb(n):
    """int -> Hebrew numeral without geresh (טו/טז)."""
    if not n or n <= 0:
        return ''
    out = ''
    if n >= 1000:
        out += heb(n // 1000) + "'"
        n %= 1000
    out += HE_HUNDREDS[n // 100] + HE_TENS[(n % 100) // 10] + HE_UNITS[n % 10]
    return out.replace('יה', 'טו').replace('יו', 'טז')


def num(tok):
    """Lenient Hebrew numeral ("ק"ב", "ב'", "קלג", "טו") -> int, else None.
    Final letters are only accepted as the last letter, and the letter values must
    not increase (so ordinary words rarely parse)."""
    t = (tok or '').strip().strip('.:,;()[]')
    t = t.replace('"', '').replace("'", '')
    if not t or len(t) > 5:
        return None
    vals = []
    for i, ch in enumerate(t):
        if ch not in HE_VALUE or (ch in 'ךםןףץ' and i != len(t) - 1):
            return None
        vals.append(HE_VALUE[ch])
    if any(b > a for a, b in zip(vals, vals[1:])):
        return None
    return sum(vals)


def is_num_tok(tok):
    """A token that is written as a numeral (has a geresh/gershayim, or is short)."""
    t = (tok or '').strip('.:,;')
    if num(t) is None:
        return False
    bare = t.replace('"', '').replace("'", '')
    return ('"' in t or "'" in t) or len(bare) <= 3


# =============================================================== PDF -> lines
class Ch(object):
    __slots__ = ('x0', 'x1', 'top', 'bottom', 'c', 'font', 'size', 'bold')

    def __init__(self, x0, x1, top, bottom, c, font, size, bold):
        self.x0, self.x1, self.top, self.bottom = x0, x1, top, bottom
        self.c, self.font, self.size, self.bold = c, font, size, bold


def _font_base(name):
    return re.sub(r'^[A-Z]{6}\+', '', name or '')


def _is_bold(name, flags=0):
    n = (name or '').lower()
    return bool(flags & 16) or any(w in n for w in ('bold', 'black', 'heavy', 'demi'))


def pages_fitz(path):
    import fitz   # PyMuPDF
    doc = fitz.open(path)
    for pno, page in enumerate(doc):
        out = []
        d = page.get_text('rawdict')
        for b in d.get('blocks', []):
            for l in b.get('lines', []):
                for sp in l.get('spans', []):
                    font = _font_base(sp.get('font', ''))
                    size = float(sp.get('size', 0) or 0)
                    bold = _is_bold(font, sp.get('flags', 0))
                    for ch in sp.get('chars', []):
                        x0, y0, x1, y1 = ch['bbox']
                        out.append(Ch(x0, x1, y0, y1, ch.get('c', ''), font, size, bold))
        yield pno, page.rect.width, page.rect.height, out


def pages_plumber(path):
    import pdfplumber
    with pdfplumber.open(path) as pdf:
        for pno, page in enumerate(pdf.pages):
            out = []
            for c in page.chars:
                font = _font_base(c.get('fontname', ''))
                out.append(Ch(c['x0'], c['x1'], c['top'], c['bottom'], c.get('text', ''),
                              font, float(c.get('size', 0) or 0), _is_bold(font)))
            yield pno, float(page.width), float(page.height), out


def open_pages(path, backend='auto'):
    order = {'auto': ('fitz', 'pdfplumber'), 'fitz': ('fitz',), 'pdfplumber': ('pdfplumber',)}[backend]
    errs = []
    for b in order:
        try:
            if b == 'fitz':
                import fitz  # noqa: F401
                return list(pages_fitz(path)), 'PyMuPDF'
            import pdfplumber  # noqa: F401
            return list(pages_plumber(path)), 'pdfplumber'
        except ImportError as e:
            errs.append(str(e))
    raise SystemExit('No PDF library: pip install pymupdf  (or pdfplumber)\n' + '\n'.join(errs))


class Line(object):
    def __init__(self, page, text, top, bottom, x0, x1, style, size, W, H):
        self.page, self.text, self.top, self.bottom = page, text, top, bottom
        self.x0, self.x1, self.style, self.size, self.W, self.H = x0, x1, style, size, W, H

    def __repr__(self):
        return 'Line(p%d %s %r)' % (self.page + 1, self.style, self.text[:40])


LTR_RUN_RE = re.compile(r'[0-9A-Za-z]+(?:[.,:/\-][0-9A-Za-z]+)*')


def _assemble(chars):
    """Chars of one visual line -> logical text. Hebrew is RTL: order by x
    descending (this is right both for logical-order and for visual-order PDFs),
    then put digit/Latin runs back in LTR order."""
    heb_n = sum(1 for c in chars if '\u0590' <= c.c <= '\u05ff')
    lat_n = sum(1 for c in chars if c.c.isascii() and c.c.isalpha())
    rtl = heb_n >= lat_n
    chars = sorted(chars, key=lambda c: -(c.x0 + c.x1) if rtl else (c.x0 + c.x1))
    out, prev = [], None
    for c in chars:
        if prev is not None and not c.c.isspace() and not prev.c.isspace():
            gap = (prev.x0 - c.x1) if rtl else (c.x0 - prev.x1)
            if gap > 0.22 * max(c.size, prev.size, 1):
                out.append(' ')
        out.append(c.c)
        prev = c
    s = ''.join(out)
    if rtl:
        s = LTR_RUN_RE.sub(lambda m: m.group(0)[::-1], s)
    return re.sub(r'\s+', ' ', s).strip()


def page_lines(pno, W, H, chars):
    chars = [c for c in chars if c.c and c.c != '\n']
    ink = [c for c in chars if not c.c.isspace()]
    if not ink:
        return []
    ink.sort(key=lambda c: (c.top + c.bottom) / 2)
    rows = []
    for c in ink:
        mid = (c.top + c.bottom) / 2
        if rows:
            r = rows[-1]
            if abs(mid - r['mid']) <= 0.45 * max(r['size'], c.size, 1):
                r['chars'].append(c)
                r['mid'] = (r['mid'] * (len(r['chars']) - 1) + mid) / len(r['chars'])
                r['size'] = max(r['size'], c.size)
                continue
        rows.append({'mid': mid, 'size': c.size, 'chars': [c]})
    spaces = [c for c in chars if c.c.isspace()]
    for c in spaces:                       # put explicit spaces into their row
        mid = (c.top + c.bottom) / 2
        best = min(rows, key=lambda r: abs(r['mid'] - mid))
        if abs(best['mid'] - mid) <= 0.6 * max(best['size'], 1):
            best['chars'].append(c)
    out = []
    for r in rows:
        cs = r['chars']
        text = _assemble(cs)
        if not text:
            continue
        st = collections.Counter()
        for c in cs:
            if not c.c.isspace():
                st[(c.font, round(c.size * 2) / 2, c.bold)] += 1
        style = st.most_common(1)[0][0]
        ink_cs = [c for c in cs if not c.c.isspace()]
        out.append(Line(pno, text, min(c.top for c in ink_cs), max(c.bottom for c in ink_cs),
                        min(c.x0 for c in ink_cs), max(c.x1 for c in ink_cs), style, style[1], W, H))
    return out


# ================================================================== layout
ENUM_RE = re.compile(r'^\s*(?:\(?([%s]{1,3}|\d{1,3})[.)]\s|\(([%s]{1,3}|\d{1,3})\)\s?)' % (HEB, HEB))
END_PUNCT = tuple('.:;?!')


def enumerated(text):
    m = ENUM_RE.match(text)
    if not m:
        return False
    t = m.group(1) or m.group(2)
    return t.isdigit() or num(t) is not None


def fix_brackets(lines):
    """Mirror () [] if the PDF stores them mirrored (most lines open with ')')."""
    close_first = open_first = 0
    for l in lines:
        m = re.search(r'[()\[\]]', l.text)
        if m:
            if m.group(0) in ')]':
                close_first += 1
            else:
                open_first += 1
    if close_first > open_first * 2 and close_first >= 3:
        tr = str.maketrans('()[]', ')(][')
        for l in lines:
            l.text = l.text.translate(tr)


def analyse(pages, args):
    """-> list of items: {'kind': 'H', 'depth', 'text', 'page'} / {'kind': 'B', 'text', 'page'}"""
    lines = []
    for pno, W, H, chars in pages:
        lines.extend(page_lines(pno, W, H, chars))
    if not lines:
        raise SystemExit('No text layer in this PDF (scanned? run OCR first).')
    fix_brackets(lines)
    npages = len(pages)
    band = args.header_band

    def in_band(l):
        return l.top < band * l.H or l.bottom > (1 - band * 0.7) * l.H

    # --- running headers / footers
    by_style = collections.defaultdict(list)
    for l in lines:
        by_style[l.style].append(l)
    header_styles = set()
    for s, ls in by_style.items():
        inb = sum(1 for l in ls if in_band(l))
        pages_s = {l.page for l in ls}
        if inb / len(ls) >= 0.9 and len(pages_s) >= max(2, 0.3 * npages):
            header_styles.add(s)
    keyed = collections.defaultdict(set)
    for l in lines:
        if in_band(l):
            keyed[re.sub('[%s0-9"\'.\\s]{0,4}$' % HEB, '', re.sub(r'[\d]+', '', l.text))[:30]].add(l.page)
    rep = {k for k, ps in keyed.items() if k and len(ps) >= max(2, 0.3 * npages)}

    def is_header(l):
        if l.style in header_styles:
            return True
        if in_band(l):
            k = re.sub('[%s0-9"\'.\\s]{0,4}$' % HEB, '', re.sub(r'[\d]+', '', l.text))[:30]
            if k in rep:
                return True
            if re.fullmatch(r'[-–\s]*(?:\d+|[%s]{1,4}["\']?)[-–\s]*' % HEB, l.text):   # bare page number
                return True
        return False

    body_lines = [l for l in lines if not is_header(l)]
    if not body_lines:
        raise SystemExit('Everything looked like a running header; try --header-band 0.05')
    chars_by_style = collections.Counter()
    for l in body_lines:
        chars_by_style[l.style] += len(l.text.replace(' ', ''))
    body = chars_by_style.most_common(1)[0][0]
    bsize, bbold = body[1], body[2]

    def heading_style(s):
        if s == body:
            return False
        if s[1] > bsize * 1.08:
            return True
        return s[2] and not bbold and s[1] >= bsize * 0.95

    hstyles = sorted({l.style for l in body_lines if heading_style(l.style)},
                     key=lambda s: (-s[1], not s[2], s[0]))
    rank = {s: i for i, s in enumerate(hstyles)}
    numbered_styles = {s for s in hstyles if any(enumerated(l.text) for l in body_lines if l.style == s)}

    def hkey(l):
        sub = 1 if (l.style in numbered_styles and not enumerated(l.text)) else 0
        return (rank[l.style], sub)

    hkeys = sorted({hkey(l) for l in body_lines if l.style in rank})
    depth_of = {k: i + 1 for i, k in enumerate(hkeys)}
    if args.verbose:
        log('body style: %s' % (body,))
        for s in hstyles:
            log('heading style %d: %s%s' % (rank[s], s, ' (numbered)' if s in numbered_styles else ''))
        log('header styles: %s' % sorted(header_styles))

    # --- left text margin per page (wrapped lines reach it)
    left = {}
    for p in {l.page for l in body_lines}:
        xs = sorted(l.x0 for l in body_lines if l.page == p and l.style == body)
        left[p] = xs[max(0, len(xs) // 20)] if xs else 0

    items = []
    prev = None
    for l in body_lines:
        is_h = l.style in rank
        text = l.text
        if prev is not None and prev['_style'] == l.style and prev['page'] == l.page:
            gap = l.top - prev['_bottom']
            close = gap < 0.9 * l.size
            not_ended = not prev['text'].rstrip().endswith(END_PUNCT)
            if is_h:
                cont = close and not_ended and not enumerated(text)
                cont = cont and (prev['_x0'] <= left.get(l.page, 0) + 0.08 * l.W or len(prev['text']) > 40)
            else:
                cont = close and not_ended and prev['_x0'] <= left.get(l.page, 0) + 0.06 * l.W
            if cont:
                prev['text'] = prev['text'].rstrip() + ' ' + text.lstrip()
                prev['_bottom'], prev['_x0'] = l.bottom, l.x0
                continue
        it = {'kind': 'H' if is_h else 'B', 'text': text, 'page': l.page, '_style': l.style,
              '_bottom': l.bottom, '_x0': l.x0, '_sub': hkey(l)[1] if is_h else 0}
        if is_h:
            it['depth'] = depth_of[hkey(l)]
        items.append(it)
        prev = it
    for it in items:
        it['text'] = norm_he(it['text'])
        for k in ('_style', '_bottom', '_x0', '_sub'):
            it.pop(k, None)
    return [it for it in items if it['text']]


# ================================================================= lexicon
MASECHTOT = {
    'ברכות': 'ברכות', 'שבת': 'שבת', 'עירובין': 'עירובין', 'ערובין': 'עירובין', 'פסחים': 'פסחים',
    'שקלים': 'שקלים', 'יומא': 'יומא', 'סוכה': 'סוכה', 'ביצה': 'ביצה', 'ראש השנה': 'ראש השנה',
    'ר"ה': 'ראש השנה', 'תענית': 'תענית', 'מגילה': 'מגילה', 'מועד קטן': 'מועד קטן', 'מו"ק': 'מועד קטן',
    'חגיגה': 'חגיגה', 'יבמות': 'יבמות', 'כתובות': 'כתובות', 'נדרים': 'נדרים', 'נזיר': 'נזיר',
    'סוטה': 'סוטה', 'גיטין': 'גיטין', 'קידושין': 'קידושין', 'קדושין': 'קידושין',
    'בבא קמא': 'בבא קמא', 'ב"ק': 'בבא קמא', 'בבא מציעא': 'בבא מציעא', 'ב"מ': 'בבא מציעא',
    'בבא בתרא': 'בבא בתרא', 'ב"ב': 'בבא בתרא', 'סנהדרין': 'סנהדרין', 'מכות': 'מכות',
    'שבועות': 'שבועות', 'עבודה זרה': 'עבודה זרה', 'ע"ז': 'עבודה זרה', 'הוריות': 'הוריות',
    'זבחים': 'זבחים', 'מנחות': 'מנחות', 'חולין': 'חולין', 'בכורות': 'בכורות', 'ערכין': 'ערכין',
    'תמורה': 'תמורה', 'כריתות': 'כריתות', 'מעילה': 'מעילה', 'נדה': 'נדה', 'נידה': 'נדה',
}
SEDARIM = {
    'זרעים': 'ברכות פאה דמאי כלאים שביעית תרומות מעשרות חלה ערלה ביכורים',
    'מועד': 'שבת עירובין פסחים שקלים יומא סוכה ביצה תענית מגילה חגיגה',
    'נשים': 'יבמות כתובות נדרים נזיר סוטה גיטין קידושין',
    'נזיקין': 'סנהדרין מכות שבועות עדיות אבות הוריות',
    'קדשים': 'זבחים מנחות חולין בכורות ערכין תמורה כריתות מעילה תמיד מדות קנים',
    'טהרות': 'כלים אהלות נגעים פרה טהרות מקואות נדה מכשירין זבים ידים עוקצין',
}
SEDER_OF = {m: s for s, ms in SEDARIM.items() for m in ms.split()}
SEDER_OF.update({'ראש השנה': 'מועד', 'מועד קטן': 'מועד', 'מעשר שני': 'זרעים', 'בבא קמא': 'נזיקין',
                 'בבא מציעא': 'נזיקין', 'בבא בתרא': 'נזיקין', 'עבודה זרה': 'נזיקין', 'טבול יום': 'טהרות'})
for _m in ('פאה דמאי כלאים שביעית תרומות מעשרות חלה ערלה ביכורים עדיות אבות מדות קנים כלים אהלות '
           'נגעים פרה טהרות מקואות מכשירין זבים ידים עוקצין').split():
    MASECHTOT.setdefault(_m, _m)
MASECHTOT.update({'מעשר שני': 'מעשר שני', 'טבול יום': 'טבול יום', 'מידות': 'מדות'})
MISHNA_ONLY = set(('פאה דמאי כלאים שביעית תרומות מעשרות חלה ערלה ביכורים עדיות אבות מדות קנים כלים '
                   'אהלות נגעים פרה טהרות מקואות מכשירין זבים ידים עוקצין').split()) | {'מעשר שני', 'טבול יום'}
SA_SECTION_OF_SEDER = {'זרעים': 'אורח חיים', 'מועד': 'אורח חיים', 'נשים': 'אבן העזר',
                       'נזיקין': 'חושן משפט', 'קדשים': 'יורה דעה', 'טהרות': 'יורה דעה'}
SA_SECTION_OVERRIDE = {'נדרים': 'יורה דעה', 'נזיר': 'יורה דעה', 'עבודה זרה': 'יורה דעה', 'נדה': 'יורה דעה'}
RAMBAM_HILCHOT_OF = {'שבת': 'שבת', 'עירובין': 'עירובין', 'פסחים': 'חמץ ומצה', 'ביצה': 'שביתת יום טוב',
                     'סוכה': 'שופר וסוכה ולולב', 'מגילה': 'מגילה וחנוכה', 'ברכות': 'ברכות',
                     'יומא': 'שביתת עשור', 'תענית': 'תעניות'}
SECTIONS = {'או"ח': 'אורח חיים', 'אורח חיים': 'אורח חיים', 'יו"ד': 'יורה דעה', 'יורה דעה': 'יורה דעה',
            'אה"ע': 'אבן העזר', 'אהע"ז': 'אבן העזר', 'אבהע"ז': 'אבן העזר', 'אבן העזר': 'אבן העזר',
            'חו"מ': 'חושן משפט', 'חושן משפט': 'חושן משפט'}
RAMBAM_SEFARIM = set('מדע אהבה זמנים נשים קדושה הפלאה זרעים עבודה קרבנות טהרה נזיקין קנין משפטים שופטים ספר'.split())

# kind:  G gemara   T commentary on the Bavli, by daf   P by perek + siman (Rosh)
#        R Rambam   RX ספר המצוות / פירוש המשניות   RC on the Rambam
#        S Shulchan Aruch   SC on the SA / Tur   O find-ref
# KIND | key | aliases ; ... | Otzaria title forms ; ... ({M} = masechet)
WORK_LEXICON = """
G|גמרא|גמ׳;גמרא;גמ;הגמ׳;ש״ס;בבלי|
G|משנה|מתני׳;מתניתין;מתני;משנה;המשנה;מתנ׳|
T|רש״י|רש״י;פירש״י;פרש״י|רש״י על {M}
T|תוספות|תוס׳;תוספות;תוס;תו׳;התוס׳|תוספות על {M}
T|תוספות רי״ד|תוס׳ רי״ד;תוספות רי״ד;תוס׳ הרי״ד;תוספות הרי״ד|תוספות רי״ד על {M}
T|תוספות הרא״ש|תוס׳ הרא״ש;תוספות הרא״ש;תוס׳ רא״ש;תוה״ר|תוספות הרא״ש על {M}
T|תוספות ישנים|תוס׳ ישנים;תוספות ישנים|תוספות ישנים על {M}
T|רבינו חננאל|ר״ח;רבינו חננאל;רבנו חננאל;הר״ח|רבינו חננאל על מסכת {M};רבינו חננאל על {M}
T|רמב״ן|רמב״ן;חי׳ הרמב״ן;חידושי הרמב״ן;חדושי הרמב״ן;חי׳ רמב״ן;חידושי רמב״ן|חידושי רמב״ן על {M};רמב״ן על {M}
T|רשב״א|רשב״א;חי׳ הרשב״א;חידושי הרשב״א;חדושי הרשב״א;חי׳ רשב״א|רשב״א על {M};חידושי הרשב״א על {M}
T|ריטב״א|ריטב״א;חי׳ הריטב״א;חידושי הריטב״א;חדושי הריטב״א;חי׳ ריטב״א|ריטב״א על {M};חידושי הריטב״א על {M}
T|חידושי הר״ן|חי׳ הר״ן;חידושי הר״ן;חדושי הר״ן;חי׳ ר״ן|חידושי הר״ן על {M}
T|ר״ן|ר״ן;ר״ן על הרי״ף;הר״ן|ר״ן על {M}
T|מאירי|מאירי;חי׳ המאירי;חידושי המאירי;חדושי המאירי;המאירי;בית הבחירה|מאירי על {M};בית הבחירה על {M}
T|מהרש״א|מהרש״א|מהרש״א - חידושי הלכות על {M};מהרש״א על {M}
T|מהרש״ל|מהרש״ל;חכמת שלמה|חכמת שלמה על {M}
T|מהר״ם|מהר״ם;מהר״ם לובלין|מהר״ם על {M}
T|מהר״ם שיף|מהר״ם שיף|מהר״ם שיף על {M}
T|רש״ש|רש״ש|רש״ש על {M}
T|פני יהושע|פנ״י;פני יהושע;הפנ״י|פני יהושע על {M}
T|רבי עקיבא איגר|רעק״א;רע״א;חי׳ רעק״א;חידושי רעק״א;חי׳ רע״א;חידושי רע״א;חי׳ הגרע״א;חידושי הגרע״א;הגרע״א;חידושי רבי עקיבא איגר;רבי עקיבא איגר;ר׳ עקיבא איגר|חידושי רבי עקיבא איגר על מסכת {M};חידושי רבי עקיבא איגר על {M}
T|גליון הש״ס|גליון הש״ס;גליון;גיליון הש״ס|גליון הש״ס על {M}
T|שפת אמת|שפ״א;שפת אמת|שפת אמת על {M}
T|חתם סופר|חת״ס;חתם סופר;חי׳ חת״ס;חידושי חת״ס;חידושי חתם סופר|חידושי חתם סופר על {M}
T|בעל המאור|בעה״מ;בעל המאור;המאור;המאור הקטן|המאור הקטן על {M};בעל המאור על {M}
T|מלחמות|מלחמות;מלחמת ה׳;מלחמת השם;מלחמות ה׳|מלחמת השם על {M}
T|רי״ף|רי״ף;הרי״ף|רי״ף {M};רי״ף על {M}
T|ראש יוסף|ראש יוסף|ראש יוסף על {M}
T|שיטה מקובצת|שיטה מקובצת;שטמ״ק;שיטמ״ק|שיטה מקובצת על {M}
T|ציון לנפש חיה|צל״ח;ציון לנפש חיה|ציון לנפש חיה על {M}
T|תוספות הרשב״א|תוס׳ הרשב״א;תוספות הרשב״א|תוספות הרשב״א על {M}
T|קרני ראם|קרני ראם|קרני ראם על {M}
T|בית מאיר|בית מאיר|בית מאיר על {M}
T|חידושי הגר״ח סטנסיל|חידושי הגר״ח סטנסיל;הגר״ח סטנסיל|חידושי הגר״ח על {M}
T|תלמיד הרמב״ן|תלמיד הרמב״ן|חידושי תלמיד הרמב״ן על {M}
T|אבן העוזר|אבן העוזר|אבן העוזר על {M}
P|רא״ש|רא״ש;הרא״ש;פסקי הרא״ש|פסקי הרא״ש על {M}
P|הגהות אשרי|הג׳ אשרי;הגהות אשרי;הגה״א;הג״א|הגהות אשרי על {M}
P|קרבן נתנאל|קרבן נתנאל;קר״נ|קרבן נתנאל על {M}
R|רמב״ם|רמב״ם;הרמב״ם;משנה תורה|
RX|ספר המצוות|סה״מ;ספר המצוות;ספר המצות|
RX|פירוש המשניות|פיה״מ;פהמ״ש;פירוש המשניות|רמב״ם על משנה {M}
RC|מגיד משנה|מ״מ;מגיד משנה;המ״מ;הה״מ;ה״ה|
RC|כסף משנה|כס״מ;כסף משנה;הכס״מ|
RC|לחם משנה|לח״מ;לחם משנה|
RC|משנה למלך|מל״מ;משנה למלך|
RC|השגות הראב״ד|השגות הראב״ד;השגת הראב״ד;הראב״ד;ראב״ד|
RC|הגהות מיימוניות|הגמ״י;הגהות מיימוניות|
RC|אבן האזל|אבן האזל|
RC|אור שמח|אור שמח|
RC|מעשה רקח|מעשה רקח|
RC|חידושי רבנו חיים הלוי|חידושי רבינו חיים הלוי;חי׳ רבינו חיים הלוי;חדושי רבנו חיים הלוי;חידושי רבנו חיים הלוי;חדושי רבינו חיים הלוי;חי׳ הגר״ח;חידושי הגר״ח|
S|שולחן ערוך|שו״ע;שולחן ערוך;שלחן ערוך;השו״ע;טוש״ע;מחבר;המחבר|
SC|מגן אברהם|מג״א;מגן אברהם;המג״א|
SC|טורי זהב|ט״ז;טורי זהב;הט״ז|
SC|משנה ברורה|מ״ב;משנ״ב;משנה ברורה;המ״ב;המשנ״ב|
SC|ביאור הלכה|בה״ל;ביאור הלכה;ביה״ל;באור הלכה|
SC|שער הציון|שעה״צ;שער הציון|
SC|פרי מגדים|פמ״ג;פרי מגדים|
SC|מחצית השקל|מחצה״ש;מחצית השקל|
SC|אליה רבה|א״ר;אליה רבה|
SC|לבושי שרד|לבושי שרד|
SC|הגהות רבי עקיבא איגר|הג׳ רע״א;הגהות רע״א;הג׳ רעק״א;הגהות רעק״א;הג׳׳ רע״א;הג׳ רע״ק|
SC|באר היטב|באה״ט;באר היטב|
SC|ערוך השלחן|ערוה״ש;ערוך השלחן;ערוך השולחן|
SC|שולחן ערוך הרב|שו״ע הרב;שולחן ערוך הרב|
SC|שפתי כהן|ש״ך;שפתי כהן|
SC|בית יוסף|ב״י;בית יוסף|
SC|בית חדש|ב״ח;בית חדש|
SC|דרכי משה|ד״מ;דרכי משה|
SC|טור|טור;הטור|
SC|כף החיים|כה״ח;כף החיים|
SC|ישועות יעקב|ישועות יעקב|
"""
# works found through find-ref: alias -> a name Otzaria knows
O_LEXICON = """
חזו״א;חזון איש=חזון איש
קוב״ש;קובץ שיעורים=קובץ שיעורים
אבנ״ז;אבני נזר=אבני נזר
תרוה״ד;תרומת הדשן=תרומת הדשן
או״ז;אור זרוע=אור זרוע
נוב״י=נודע ביהודה
קצוה״ח=קצות החושן
מנ״ח=מנחת חינוך
שעה״מ=שער המלך
שבה״ל=שבלי הלקט
סמ״ג=ספר מצוות גדול
סמ״ק=ספר מצוות קטן
יראים;ספר יראים=ספר יראים
ריב״ש;הריב״ש=תשובות הריב״ש
אגלי טל=אגלי טל
שערי ישר=שערי ישר
אפיקי ים=אפיקי ים
תוצאות חיים=תוצאות חיים
אמרי בינה=אמרי בינה
שביתת השבת=שביתת השבת
כפות תמרים=כפות תמרים
שלטי גיבורים;שלטי הגבורים;של״ג=שלטי הגבורים
חמדת שלמה=חמדת שלמה
שנות אליהו=שנות אליהו
ספר הליקוטים=ספר הליקוטים
אליהו רבה=אליהו רבה
רע״ב;ברטנורא=ברטנורא
תוס׳ רעק״א;תוספות רעק״א;תוס׳ רע״א;תוספות רע״א=תוספות רבי עקיבא איגר
תוספות יו״ט;תוס׳ יו״ט;תוי״ט;תוספות יום טוב=תוספות יום טוב
מרדכי;המרדכי=מרדכי
תשובות רע״א;שו״ת רע״א;שו״ת רעק״א=שו״ת רבי עקיבא איגר
אחיעזר=אחיעזר
מנחת ברוך=מנחת ברוך
דברי יחזקאל=דברי יחזקאל
יש סדר למשנה=יש סדר למשנה
מלאכת שלמה=מלאכת שלמה
ספר התרומה;התרומה=ספר התרומה
"""


def _alias_key(s):
    return tuple(name_key(w) for w in norm_he(s).split())


ALIAS = {}                      # token tuple -> entry
ALIAS_QUOTED = {}               # token tuple -> True if every spelling of it is an abbreviation
for _row in WORK_LEXICON.strip().splitlines():
    _k, _key, _al, _forms = (norm_he(x) for x in _row.split('|'))
    _e = {'kind': _k, 'key': _key, 'forms': [f for f in _forms.split(';') if f]}
    for _a in _al.split(';'):
        if _a:
            ALIAS.setdefault(_alias_key(_a), _e)
            ALIAS_QUOTED[_alias_key(_a)] = ALIAS_QUOTED.get(_alias_key(_a), True) and ('"' in _a or "'" in _a)
for _row in O_LEXICON.strip().splitlines():
    _al, _full = _row.split('=')
    for _a in _al.split(';'):
        ALIAS.setdefault(_alias_key(_a), {'kind': 'O', 'key': norm_he(_full), 'forms': []})
        _n = norm_he(_a)
        ALIAS_QUOTED[_alias_key(_a)] = ALIAS_QUOTED.get(_alias_key(_a), True) and ('"' in _n or "'" in _n)
MAS_ALIAS = {_alias_key(k): v for k, v in MASECHTOT.items()}
SECTION_ALIAS = {_alias_key(k): v for k, v in SECTIONS.items()}
MAX_ALIAS = max(len(k) for k in list(ALIAS) + list(MAS_ALIAS) + list(SECTION_ALIAS))
PREFIXES = ('', 'ו', 'ב', 'וב', 'ה', 'וה', 'ל', 'ול', 'ד', 'וד', 'מ', 'ומ', 'כ', 'ש', 'בה', 'ובה', 'דה')


def match_table(table, toks, i, allow_prefix=True, prefixes=None):
    """Longest alias of `table` starting at toks[i] -> (value, n_tokens, prefix) or None.
    An abbreviation ("מנ"ח") only matches a token written as one (not "דמנח")."""
    for n in range(min(MAX_ALIAS, len(toks) - i), 0, -1):
        quoted = any('"' in x or "'" in x for x in toks[i:i + n])
        words = [name_key(t) for t in toks[i:i + n]]
        if any(not w for w in words):
            continue
        for p in ((prefixes or PREFIXES) if allow_prefix else ('',)):
            if p and not words[0].startswith(p):
                continue
            first = words[0][len(p):]
            if len(first) < 2:
                continue
            key = tuple([first] + words[1:])
            if key in table:
                if table is ALIAS and ALIAS_QUOTED.get(key) and not quoted:
                    continue
                return table[key], n, p
    return None


def _keys(words):
    return {name_key(x) for x in norm_he(words).split()}


MARKERS = _keys("עי׳ עיין ועי׳ ועיין וע׳ ע׳ ראה וראה עיי׳ ועיי׳ ועוד וכן וכ״כ כ״כ וע״ע ע״ע עיי״ש ועי״ש עי״ש "
                "וע״ש ע״ש עין ועין היטב כעי״ז וכעי״ז מש״כ במש״כ מה שכתב בשם הובא שהובא דהובא בדברי ובדברי "
                "דברי בדבריו ובדבריו על מ״ש ומ״ש בד׳ בדעת ובדעת בביאור בשיטת שיטת נמי גם ונו״כ נו״כ ונושאי "
                "נושאי כלים כמו שכתב שכ׳ וכ׳ כ׳ ובזה יש להבין")
BACKREF = _keys('הנ״ל דלעיל לעיל להלן דלהלן לקמן דלקמן')
SUGYA_WORDS = _keys('בסוגיין בסוגיא סוגיין בסוגייתנו כאן בסוה״ס')
DH_OPEN = {'דה': 'dh', 'בדה': 'dh', 'ודה': 'vdh', 'ובדה': 'vdh', 'מדה': 'dh', 'ומדה': 'vdh',
           'תודה': 'tdh', 'ותודה': 'tdh', 'בתודה': 'tdh', 'ובתודה': 'tdh', 'סדה': 'edh', 'סודה': 'edh',
           'בסדה': 'edh', 'וסדה': 'edh'}
ENDERS = _keys('וכו׳ כו׳ וגו׳ וכו כו וגו ואילך')
MID = _keys('באה״ד בא״ד באמצע בסה״ד בסוה״ד בסוד״ה בסו״ד')
UNTIL = _keys('עד ע״ד ועד')
END_WORDS = _keys('סוף הסוף בסוף סוה״ס המשנה הסוגיא העמוד הסעיף התוס׳ הדיבור')
ORDINALS = {name_key(x): n for x, n in (('הראשון', 1), ('קמא', 1), ('השני', 2), ('בתרא', 2), ('השלישי', 3))}
RIF_PAGE = _keys('בדפי מדפי בדפיו דפי מדפיו לדפי')
EDITION = _keys('דפוס דפו״י בדפו״י הוצאת מודפס עמוד עמ׳ מהדורת מהדו״ק מהדו״ת וילנא ווילנא זכרון')
WORK_NOISE = _keys('בענין ובענין ספר בספר ובספר בפירושו פירוש בפי׳ פי׳ בתשובת תשובת ז״ל זצ״ל במערכה '
                   'בהגהותיו שם וגם אמנם נ״ב חי׳ חידושי חדושי על')
CHIDUSHEI = _keys('חי׳ חידושי חדושי')
LOC_WORDS = {
    'סימן': 'siman', 'סי': 'siman', 'ס': 'siman?', 'סעיף': 'seif', 'סע': 'seif', 'סעי': 'seif',
    'סק': 'sk', 'אות': 'os', 'אותיות': 'os', 'ענף': 'anaf', 'כלל': 'klal', 'חלק': 'chelek', 'ח': 'chelek',
    'פרק': 'perek', 'פ': 'perek', 'הלכה': 'halacha', 'הל': 'halacha', 'משנה': 'mishna', 'מצוה': 'mitzva',
    'שורש': 'shoresh', 'שער': 'shaar', 'דף': 'daf', 'בדף': 'daf', 'ובדף': 'daf', 'לדף': 'daf',
    'מדף': 'daf', 'ודף': 'daf', 'דפים': 'daf', 'בסי': 'siman', 'ובסי': 'siman', 'בסימן': 'siman',
    'סוסי': 'siman', 'ריש': None,
}
COMPACT_RE = re.compile('^(?:ו?ב?)(סק|סי|ס|פ|ה|ח)([%s]?)"([%s])$' % (HEB, HEB))
COMPACT_FIELD = {'סק': 'sk', 'סי': 'siman?', 'ס': 'siman?', 'פ': 'perek', 'ה': 'halacha', 'ח': 'chelek'}
AMUD_TOK = {'עא': 'a', 'עב': 'b', 'בעא': 'a', 'בעב': 'b', 'ובעא': 'a', 'ובעב': 'b'}
NOT_DAF = _keys('שם כן לא זה זו מה הם הן אך רק בה בו לו לה כך כל גם אם עם את אל על')
TOK_RE = re.compile(r'[()\[\],;]|[^\s()\[\],;]+')


def tokenize(s):
    out = []
    for t in TOK_RE.findall(s):
        if t in ('.', ':', '-', '–'):
            out.append(';' if t != '-' else ',')
        elif len(t) > 1 and t[-1] in '.:' and (not re.fullmatch('[%s]{1,4}[.:]' % HEB, t)
                                               or name_key(t[:-1]) in NOT_DAF or num(t[:-1]) is None):
            out += [t[:-1], ';']
        elif len(t) > 1 and t[0] in '.:':
            out += [';', t[1:]]
        else:
            out.append(t)
    return [t for t in out if t]


def amud_of(tok):
    return AMUD_TOK.get(name_key(tok))


def parse_daf(toks, i, after_daf_word=False):
    """daf at toks[i] -> ((n, amud|None), n_tokens) or None.
    ב' ע"א | צ"ז ע"ב | מ"ב ב' | קנ"ד א' | קלג. | ה ע"א | after דף: ע' / צ"ט"""
    t = toks[i]
    for _ in range(2):
        if num(t) is None and len(t) > 2 and t[0] in 'ובלמ' and ('"' in t or "'" in t):
            t = t[1:]
    m = re.fullmatch('([%s]{1,4})([.:])' % HEB, t)
    if m and num(m.group(1)) is not None:
        return (num(m.group(1)), 'a' if m.group(2) == '.' else 'b'), 1
    n = num(t)
    if n is None or n > 180 or not (is_num_tok(t) or after_daf_word):
        return None
    if i + 1 < len(toks):
        nk = name_key(toks[i + 1])
        if nk in ('עא', 'עב'):
            return (n, AMUD_TOK[nk]), 2
        if nk == 'עמוד' and i + 2 < len(toks) and name_key(toks[i + 2]) in ('א', 'ב'):
            return (n, 'a' if name_key(toks[i + 2]) == 'א' else 'b'), 3
        if nk in ('א', 'ב') and (after_daf_word or "'" in toks[i + 1]):
            return (n, 'a' if nk == 'א' else 'b'), 2
    if after_daf_word:
        return (n, None), 1
    return None


def strict_num(t):
    """a numeral written as one: with geresh/gershayim, or a single letter"""
    bare = (t or '').replace('"', '').replace("'", '').strip('.:,')
    return num(t) is not None and ('"' in t or "'" in t or len(bare) == 1)


MAS_PREFIXES = ('', 'ו', 'ב', 'וב', 'ד', 'מ', 'במ', 'ובמ')


def _numbers_after(toks, j):
    if j < len(toks) and num(toks[j]) is not None and (is_num_tok(toks[j]) or len(toks[j]) <= 4):
        return num(toks[j]), 1
    return None, 0


class Cite(object):
    def __init__(self, entry=None, work_words=None):
        self.kind = entry['kind'] if entry else ('?' if work_words else None)
        self.key = entry['key'] if entry else (' '.join(work_words) if work_words else None)
        self.forms = list(entry['forms']) if entry else []
        self.mas = None
        self.dafs = []            # [(n, 'a'|'b'|None)]
        self.loc = {}             # siman seif sk os perek halacha mishna mitzva anaf klal chelek
        self.section = None
        self.hilchot = None
        self.perek_name = None
        self.rosh = False         # ריש פרק
        self.dhs = []             # [[words], ...]
        self.ordinal = {}         # dh index -> n  ("(השני)")
        self.anchor = []          # gemara / mishna: the opening words
        self.until = None         # {'dh': [...], 'words': [...], 'end': True, 'daf': (n, a)}
        self.ibid = False
        self.backref = False
        self.rif_pages = False
        self.sugya = False
        self.toks = []
        self.on_mishna = False
        self.pmg = None
        self.mitzva_kind = None

    def clone_work(self):
        c = Cite()
        c.kind, c.key, c.forms, c.mas = self.kind, self.key, list(self.forms), self.mas
        c.section, c.hilchot = self.section, self.hilchot
        return c

    def has_place(self):
        return bool(self.dafs or self.loc or self.hilchot or self.perek_name)

    def text(self):
        s = ' '.join(self.toks)
        s = s.replace(' ,', ',').replace(' ;', ';').replace('( ', '(').replace(' )', ')')
        return s.replace('[ ', '[').replace(' ]', ']').strip(' ,;')

    def __repr__(self):
        bits = [self.kind or '-', self.key or '?']
        if self.mas:
            bits.append('mas=' + self.mas)
        if self.dafs:
            bits.append('daf=' + ','.join(heb(n) + {'a': '.', 'b': ':', None: ''}[a] for n, a in self.dafs))
        for k, v in self.loc.items():
            bits.append('%s=%s' % (k, heb(v)))
        for f in ('section', 'hilchot', 'perek_name'):
            if getattr(self, f):
                bits.append('%s=%s' % (f, getattr(self, f)))
        for d in self.dhs:
            bits.append('dh=' + ' '.join(d))
        if self.anchor:
            bits.append('anchor=' + ' '.join(self.anchor))
        if self.until:
            bits.append('until=%s' % ({k: (' '.join(v) if isinstance(v, list) else v)
                                         for k, v in self.until.items()},))
        for f in ('ibid', 'backref', 'rif_pages', 'sugya', 'rosh'):
            if getattr(self, f):
                bits.append(f)
        return '<' + ' '.join(bits) + '>'


class Parser(object):
    """One source line -> [Cite]."""

    def try_loc(self, toks, i, cur):
        """-> (field, value, n_tokens); field 'daf' has value (n, amud)."""
        t = toks[i]
        k = name_key(t)
        if i + 1 < len(toks) and name_key(toks[i + 1]) in ('עא', 'עב'):
            r = parse_daf(toks, i)
            if r:
                return 'daf', r[0], r[1]
        if k not in LOC_WORDS and len(k) > 2 and k[0] in 'וב' and k[1:] in LOC_WORDS:
            k = k[1:]
        if t.count('"') == 1:
            m = COMPACT_RE.match(t)
            if m:
                lead, pre, last = m.group(1), m.group(2), m.group(3)
                if lead == 'ס' and pre == '' and last == 'ק':          # ס"ק <n>
                    v, n2 = _numbers_after(toks, i + 1)
                    return ('sk', v, 1 + n2) if v else None
                v = num(pre + last)
                if v is not None:
                    f = COMPACT_FIELD[lead]
                    if f == 'siman?':
                        f = 'siman' if (cur is None or (cur.loc.get('siman') is None and not cur.ibid)) else \
                            ('sk' if cur.kind == 'SC' else 'seif')
                    return f, v, 1
        if t in ('ס"ק', 'סק"') or k == 'סק':
            v, n2 = _numbers_after(toks, i + 1)
            return ('sk', v, 1 + n2) if v else None
        if k == 'סעיף' and i + 1 < len(toks) and name_key(toks[i + 1]) == 'קטן':
            v, n2 = _numbers_after(toks, i + 2)
            return ('sk', v, 2 + n2) if v else None
        field = LOC_WORDS.get(k)
        if field == 'daf':
            if i + 1 < len(toks):
                r = parse_daf(toks, i + 1, after_daf_word=True)
                if r:
                    return 'daf', r[0], 1 + r[1]
            return None
        if field:
            if field == 'chelek' and k == 'ח' and "'" not in t:
                return None
            if field == 'mishna' and (cur is None or cur.kind not in ('G', 'RX', 'O', '?')):
                return None
            v, n2 = _numbers_after(toks, i + 1)
            if v is None:
                return None
            if field == 'siman?':
                field = 'siman' if (cur is None or (cur.loc.get('siman') is None and not cur.ibid)) else \
                    ('sk' if cur.kind == 'SC' else 'seif')
            return field, v, 1 + n2
        r = parse_daf(toks, i)
        if r:
            return 'daf', r[0], r[1]
        return None

    def parse(self, text):
        s = re.sub(r'[.:\s]+$', '', norm_he(text).strip())
        while len(s) > 2 and s[0] in '([' and s[-1] in ')]' and self._balanced(s[1:-1]):
            s = s[1:-1].strip(' .')
        cites = []
        self._run(tokenize(s), cites)
        return [c for c in cites if c.kind or c.has_place() or c.dhs]

    @staticmethod
    def _balanced(s):
        d = 0
        for ch in s:
            d += ch in '(['
            d -= ch in ')]'
            if d < 0:
                return False
        return d == 0

    QUAL = _keys('בדעת ובדעת בשם ובשם בשיטת ובשיטת בביאור שהביא והביא הביא מביא בדברי ובדברי על כתב שכתב '
                 'כתבו שכתבו כמ״ש וכמ״ש שהקשה הקשה')
    HIL = _keys('הלכות מהלכות מהל׳ הל׳ בהלכות דהלכות בהל׳ ובהל׳ מהלכ׳ והל׳')

    def _run(self, toks, cites):
        S = {'cur': None, 'mode': None, 'pend': [], 'pa': 0, 'comma': False, 'qual': False}
        i = 0

        def new(entry=None, words=None, start=None):
            c = Cite(entry, words)
            c.a = c.b = i if start is None else start
            cites.append(c)
            S['cur'] = c
            return c

        def clone(at):
            c = S['cur'].clone_work() if S['cur'] is not None else cites[-1].clone_work()
            c.a = c.b = at
            cites.append(c)
            S['cur'] = c
            return c

        def from_pending():
            ww = [w for w in S['pend'] if name_key(w) not in WORK_NOISE]
            if ww and ww[0].startswith('ו') and len(ww[0]) > 3:
                ww[0] = ww[0][1:]
            return ww or None

        while i < len(toks):
            t = toks[i]
            k = name_key(t)
            cur, mode = S['cur'], S['mode']

            def take(n=1):
                if S['cur'] is not None:
                    S['cur'].b = i + n
                return i + n

            # ---------------------------------------------------- brackets
            if t in ('(', '['):
                close = ')' if t == '(' else ']'
                depth, j = 1, i + 1
                while j < len(toks) and depth:
                    depth += (toks[j] == t) - (toks[j] == close)
                    j += 1
                inner = toks[i + 1:j - 1] if depth == 0 else toks[i + 1:]
                ik = [name_key(x) for x in inner]
                has_daf = any(parse_daf(inner, q) for q in range(len(inner)))
                if any(x in RIF_PAGE or x in EDITION for x in ik):
                    if cur is not None and not has_daf and any(x in RIF_PAGE for x in ik):
                        cur.rif_pages = True
                elif len(ik) == 1 and ik[0] in ORDINALS and cur is not None and cur.dhs:
                    cur.ordinal[len(cur.dhs) - 1] = ORDINALS[ik[0]]
                elif cur is not None and inner and self._all_locs(inner, cur):
                    pass
                elif any(match_table(ALIAS, inner, q) for q in range(len(inner))):
                    sub = []
                    self._run(inner, sub)
                    sub = [c for c in sub if c.has_place() or c.dhs]
                    for c in sub:
                        c.a += i + 1
                        c.b += i + 1
                    cites.extend(sub)
                    if sub:
                        S['cur'], S['pend'] = None, []
                i = take(j - i) if S['cur'] is not None else j
                S['mode'] = None
                continue
            if t in (',', ';', ')', ']'):
                S['mode'], S['pend'], S['qual'] = None, [], False
                S['comma'] = t == ','
                i += 1
                continue

            # ---------------------------------------------------- ד"ה family
            dk = DH_OPEN.get(k)
            if dk:
                if mode == 'dh' and dk in ('dh', 'tdh') and cur.dhs:
                    cur.dhs[-1].append(t)                 # "ד"ה תוס' ד"ה מקצת" = one dibbur
                    i = take()
                    continue
                if dk == 'edh' or mode in ('until', 'until_dh'):
                    if cur is not None:
                        cur.until = cur.until or {}
                        cur.until['dh'] = []
                        S['mode'] = 'until_dh'
                    i = take()
                    continue
                if dk == 'tdh':
                    prev = cur
                    c = new(ALIAS[_alias_key("תוס'")])
                    if prev is not None and prev.mas and prev.kind in ('G', 'T'):
                        c.mas = prev.mas
                elif S['pend']:
                    new(None, from_pending(), S['pa'])
                elif cur is None:
                    new()
                S['cur'].dhs.append([])
                S['mode'], S['pend'], S['comma'] = 'dh', [], False
                i = take()
                continue
            # ---------------------------------------------------- small words
            if mode == 'dh' and cur.dhs and not cur.dhs[-1]:
                cur.dhs[-1].append(t)                     # "ד"ה בא"ד ...", "ד"ה גמ'"
                i = take()
                continue
            if k in ENDERS:
                if mode in ('dh', 'anchor'):
                    S['mode'] = None
                i = take()
                continue
            if k in MID:
                S['mode'] = 'mid'
                i = take()
                continue
            if k in UNTIL and cur is not None:
                cur.until = cur.until or {}
                S['mode'] = 'until'
                i = take()
                continue
            if k in ('שם', 'ושם') and mode not in ('until', 'until_dh', 'anchor'):
                if cur is None or S['pend']:
                    new(None, from_pending(), S['pa'] if S['pend'] else i)
                elif S['comma'] and cur.has_place():
                    clone(i)                              # "..., ושם פ"ז הל' י"א"
                c = S['cur']
                if mode == 'dh' and c.dhs and not c.dhs[-1]:
                    c.dhs.pop()
                c.ibid = True
                S['mode'], S['pend'] = None, []
                i = take()
                continue
            if k in BACKREF or k in SUGYA_WORDS:
                if cur is not None:
                    if k in BACKREF:
                        cur.backref = True
                    else:
                        cur.sugya = True
                S['mode'] = None
                i = take()
                continue
            if mode == 'mid':
                i = take()
                continue
            if mode in ('dh', 'until_dh'):
                lst = cur.dhs[-1] if mode == 'dh' else cur.until['dh']
                hit = match_table(ALIAS, toks, i) if lst else None
                if lst and (k in MARKERS and k.startswith('ו')):
                    S['mode'] = None                       # "ד"ה פטורי ועיין רש"ש"
                elif lst and i + 1 < len(toks) and name_key(toks[i + 1]) in ('עא', 'עב') and parse_daf(toks, i):
                    S['mode'] = None                       # "ד"ה שניהם וצ"ג ע"א ד"ה חד"
                elif hit and self._new_work_here(toks, i, hit):
                    S['mode'] = None                       # "ד"ה עקירת ומהר"ם שם", "ד"ה X פנ"י שם"
                elif k in ORDINALS and lst and mode == 'dh':
                    cur.ordinal[len(cur.dhs) - 1] = ORDINALS[k]
                    i = take()
                    continue
                elif len(lst) >= 12 or (lst and (LOC_WORDS.get(k) == 'daf' or LOC_WORDS.get(k[1:]) == 'daf')
                                        and self.try_loc(toks, i, cur)):
                    S['mode'] = None
                else:
                    lst.append(t)
                    i = take()
                    continue
            mode = S['mode']
            if mode in ('until', 'pname', 'hil'):
                h = match_table(ALIAS, toks, i)
                if h and h[2].startswith('ו'):
                    S['mode'] = mode = None
            if mode == 'anchor':
                hit = match_table(ALIAS, toks, i)
                if (hit and self._new_work_here(toks, i, hit)) \
                        or k in MARKERS or (i + 1 < len(toks) and name_key(toks[i + 1]) in ('עא', 'עב')):
                    S['mode'] = mode = None
                else:
                    cur.anchor.append(t)
                    i = take()
                    continue
            # ---------------------------------------------------- places
            if S['comma'] and cur is not None and cur.loc and strict_num(t) and not (
                    i + 1 < len(toks) and (name_key(toks[i + 1]) in ('עא', 'עב') or toks[i + 1] in ("א'", "ב'"))):
                i = take()                                # one more number of the same list
                continue
            if cur is not None and cur.key == 'פרי מגדים' and k in ('משבז', 'אא', 'משבצות', 'אשל', 'זהב', 'אברהם',
                                                                   'במשבז', 'באא', 'באשל'):
                cur.pmg = 'משבצות זהב' if k.endswith('משבז') or k in ('משבצות', 'זהב') else 'אשל אברהם'
                i = take()
                continue
            sec = match_table(SECTION_ALIAS, toks, i)
            if sec and cur is not None and cur.kind in ('S', 'SC', 'O', '?', 'T', 'P'):
                cur.section = sec[0]
                i = take(sec[1])
                continue
            if cur is not None and cur.kind in ('R', 'RC') and k in self.HIL and i + 1 < len(toks) \
                    and not strict_num(toks[i + 1]):
                S['mode'], cur.hilchot = 'hil', ''
                i = take()
                continue
            if mode == 'hil':
                if self.try_loc(toks, i, cur) or match_table(ALIAS, toks, i) or DH_OPEN.get(k):
                    S['mode'] = None
                else:
                    cur.hilchot = (cur.hilchot + ' ' + t).strip()
                    if len(cur.hilchot.split()) >= 4:
                        S['mode'] = None
                    i = take()
                    continue
            if cur is not None and cur.kind in ('R', 'RC') and not cur.hilchot and not cur.loc \
                    and k in RAMBAM_SEFARIM:
                i = take()
                continue
            if cur is not None and k in ('ריש', 'בריש', 'וריש', 'סוף', 'בסוף', 'ובסוף') and i + 1 < len(toks) \
                    and mode not in ('until', 'until_dh'):
                cur.rosh = cur.rosh or k.endswith('ריש')
                i = take()
                nk = name_key(toks[i]) if i < len(toks) else ''
                if nk not in ('מכילתין', 'דמכילתין', 'פרק', 'פ', 'הסימן', 'הסוגיא') and not self.try_loc(toks, i, cur) \
                        and not match_table(ALIAS, toks, i) and i < len(toks) and toks[i] not in (',', ';'):
                    S['mode'], cur.perek_name = 'pname', ''
                continue
            if cur is not None and k in ('פרק', 'פ', 'בפרק', 'בפ', 'ופרק', 'ובפ', 'בפר') and i + 1 < len(toks) \
                    and not strict_num(toks[i + 1]) and not match_table(ALIAS, toks, i + 1):
                S['mode'], cur.perek_name = 'pname', ''
                i = take()
                continue
            if mode == 'pname':
                if self.try_loc(toks, i, cur) or match_table(ALIAS, toks, i) or DH_OPEN.get(k) or k in MARKERS:
                    S['mode'] = None
                else:
                    cur.perek_name = (cur.perek_name + ' ' + t).strip()
                    if len(cur.perek_name.split()) >= 3:
                        S['mode'] = None
                    i = take()
                    continue
            if k in ('מכילתין', 'דמכילתין') and cur is not None:
                if not cur.dafs:
                    cur.dafs = [(2, 'a')]
                i = take()
                continue
            alias_here = match_table(ALIAS, toks, i)
            prev_k = name_key(toks[i - 1]) if i else ''
            loc = None
            amud_next = i + 1 < len(toks) and (name_key(toks[i + 1]) in ('עא', 'עב') or (
                toks[i + 1] in ("א'", "ב'") and cur is not None and cur.kind in ('T', 'G', 'P', '?', None)
                and not cur.dafs))
            if not alias_here or prev_k in LOC_WORDS or prev_k[1:] in LOC_WORDS or amud_next:
                loc = self.try_loc(toks, i, cur)
            if loc is None and cur is not None and cur.kind in ('S', 'SC') and 'siman' not in cur.loc \
                    and strict_num(t) and num(t) and not alias_here and not match_table(MAS_ALIAS, toks, i):
                before = [c for c in cites if c is not cur]
                if cur.kind == 'SC' and before and before[-1].kind in ('S', 'SC') and before[-1].loc.get('siman'):
                    loc = ('sk', num(t), 1)               # "שו"ע סי' רס"ו ומג"א י"ג"
                else:
                    loc = ('siman', num(t), 1)
            if loc is not None:
                field, val, n = loc
                if mode in ('until', 'until_dh') and cur is not None:
                    if field == 'daf':
                        cur.until['daf'] = val
                    i = take(n)
                    continue
                if cur is None:
                    if S['pend']:
                        new(None, from_pending(), S['pa'])
                    elif cites:
                        clone(i)
                    else:
                        new()
                elif field == 'daf' and (cur.dhs or cur.anchor or cur.sugya or (cur.dafs and cur.dafs[-1][1] is not None
                                                                   and val[0] != cur.dafs[-1][0])):
                    clone(i)
                elif field in cur.loc and field in ('siman', 'perek', 'mitzva', 'chelek', 'klal') and \
                        (cur.dhs or S['comma'] or k.startswith('ו')):
                    clone(i)
                c = S['cur']
                if field == 'daf':
                    if c.dafs and c.dafs[-1][0] == val[0] and c.dafs[-1][1] is None:
                        c.dafs[-1] = val
                    elif val not in c.dafs:
                        c.dafs.append(val)
                elif field not in c.loc:
                    c.loc[field] = val
                S['pend'], S['comma'] = [], False
                i = take(n)
                if c.kind == 'G' or (c.kind not in ('S', 'SC') and not c.dhs and self._anchor_ahead(toks, i)):
                    S['mode'] = 'anchor'
                continue
            if cur is not None and k in ('בעא', 'בעב', 'ובעא', 'ובעב'):
                base = cur.dafs[-1][0] if cur.dafs else None
                if base:
                    c = clone(i)
                    c.dafs = [(base, AMUD_TOK[k])]
                i = take()
                continue
            if mode == 'until' and cur is not None:
                cur.until.setdefault('words', []).append(t)
                if k in END_WORDS:
                    cur.until['end'] = True
                i = take()
                continue
            # ---------------------------------------------------- works
            mas = match_table(MAS_ALIAS, toks, i, prefixes=MAS_PREFIXES)
            if mas and not (cur is not None and cur.kind in ('T', 'G', 'P', '?', 'RX', 'O', 'R', 'RC')
                            and not cur.dafs and not cur.dhs and not cur.mas and not cur.loc) \
                    and not self._place_follows(toks, i + mas[1]):
                mas = None
            if mas and (not alias_here or mas[1] >= alias_here[1]):
                m_name = mas[0]
                if cur is not None and S['qual']:
                    pass
                elif cur is not None and cur.kind in ('T', 'G', 'P', '?', 'RX', 'O') and not cur.dafs \
                        and not cur.dhs and not cur.mas and not cur.loc:
                    cur.mas = m_name
                elif cur is not None and cur.kind in ('R', 'RC') and not cur.hilchot:
                    cur.hilchot = m_name
                elif S['pend']:
                    c = new(None, from_pending(), S['pa'])
                    c.mas = m_name
                else:
                    c = new(ALIAS[_alias_key('משנה' if m_name in MISHNA_ONLY else 'גמרא')])
                    c.mas = m_name
                S['pend'], S['comma'], S['mode'], S['qual'] = [], False, None, False
                i = take(mas[1])
                continue
            if cur is not None and k in RIF_PAGE:
                cur.rif_pages = True
                h = match_table(ALIAS, toks, i + 1) if i + 1 < len(toks) else None
                i = take(1 + (h[1] if h and h[0]['key'] == 'רי"ף' else 0))
                continue
            if alias_here:
                e, n, p = alias_here
                if e['kind'] == 'G' and cur is not None and cur.kind is None:
                    cur.kind, cur.key = 'G', e['key']      # "ק' ע"א מתני'"
                    i = take(n)
                    continue
                if e['kind'] == 'G' and cur is not None and cur.kind == 'G' and cur.dafs and not S['comma'] \
                        and not self._place_follows(toks, i + n):
                    cur.on_mishna = cur.on_mishna or e['key'] == 'משנה'
                    i = take(n)                           # "עירובין צ"ז ע"ב מתני' וגמ'"
                    continue
                if k in CHIDUSHEI and n == 1 and i + 1 < len(toks):
                    h2 = match_table(ALIAS, toks, i + 1)
                    if h2:
                        e, n = h2[0], 1 + h2[1]
                if cur is not None and S['qual']:
                    S['qual'] = False
                    i = take(n)                               # "ריטב"א בדעת התוס'": not a new source
                    continue
                if p.startswith('ל') and cur is not None and not cur.has_place() and not cur.dhs:
                    i = take(n)                           # "פירוש המשניות לרמב"ם", "הגהות רע"א לשו"ע"
                    continue
                if e['kind'] == 'G' and p.startswith('ב') and cur is not None and cur.kind in ('T', 'P') \
                        and not cur.has_place() and not cur.dhs:
                    cur.on_mishna = e['key'] == 'משנה'     # "רש"י במתני' ד"ה ..."
                    i = take(n)
                    continue
                if e['key'] == 'ספר המצוות' and cur is not None and cur.kind == 'R' and not cur.has_place():
                    cur.kind, cur.key = 'RX', e['key']
                    i = take(n)
                    continue
                new(e)
                S['pend'], S['comma'], S['mode'] = [], False, None
                i = take(n)
                continue
            # ---------------------------------------------------- plain words
            if cur is not None and cur.kind == 'RX' and k in ('לאוין', 'לת', 'לאו', 'עשה', 'עשין', 'מצוות', 'מ'):
                cur.mitzva_kind = cur.mitzva_kind or ('עשה' if k.startswith('עש') else 'לא תעשה')
                i = take()
                continue
            if k in self.QUAL and cur is not None:
                S['qual'] = True
                i = take()
                continue
            if k in MARKERS:
                S['pend'], S['mode'], S['comma'] = [], None, False
                i += 1
                continue
            if mode == 'anchor' and cur is not None:
                cur.anchor.append(t)
                i = take()
                continue
            if mode == 'until' and cur is not None and not (alias_here and alias_here[2].startswith('ו')):
                cur.until.setdefault('words', []).append(t)
                if k in END_WORDS:
                    cur.until['end'] = True
                i = take()
                continue
            if cur is not None and S['comma'] and cur.dhs and not strict_num(t) and len(k) > 1 and not cur.anchor \
                    and not any(self._place_follows(toks, q) for q in range(i + 1, min(len(toks), i + 4))):
                cur.dhs.append([t])                           # "ד"ה ואם, ואין"
                S['mode'], S['comma'] = 'dh', False
                i = take()
                continue
            if cur is not None and cur.kind and not cur.has_place() and not cur.dhs and not S['comma']:
                i = take()                                    # a qualifier of the work just named
                continue
            if not S['pend']:
                S['pa'] = i
            S['pend'].append(t)
            if len(S['pend']) > 5:
                S['pend'] = S['pend'][-5:]
                S['pa'] = i - 4
            S['cur'] = None
            i += 1
        for c in cites:
            c.toks = toks[c.a:c.b]
            c.dhs = [d for d in c.dhs if d]
        return cites

    def _new_work_here(self, toks, i, hit):
        """inside a dibbur / an opening phrase: does a new source start here?"""
        if hit[2].startswith('ו'):
            return True
        if not ('"' in toks[i] or "'" in toks[i]):
            return False
        j = i + hit[1]
        return j < len(toks) and (name_key(toks[j]) in ('שם', 'ושם') or bool(DH_OPEN.get(name_key(toks[j])))
                                  or self._place_follows(toks, j))

    def _place_follows(self, toks, j):
        if j >= len(toks):
            return False
        if DH_OPEN.get(name_key(toks[j])):
            return True
        return bool(self.try_loc(toks, j, None)) or (j + 1 < len(toks) and self.try_loc(toks, j + 1, None)
                                                     and name_key(toks[j]) in ('דף', 'פרק', 'פ'))

    def _all_locs(self, inner, cur):
        """[אות קט]: a bracket holding only places -> applied to the current cite"""
        j, got = 0, []
        while j < len(inner):
            r = self.try_loc(inner, j, cur)
            if not r:
                return False
            got.append(r)
            j += r[2]
        for f, v, _ in got:
            if f != 'daf':
                cur.loc.setdefault(f, v)
        return bool(got)

    @staticmethod
    def _anchor_ahead(toks, i):
        """words up to an עד before the next boundary: "מאיתמר ... עד ..." is an opening phrase."""
        for j in range(i, min(len(toks), i + 10)):
            if toks[j] in (',', ';', '(', '['):
                return False
            if name_key(toks[j]) in UNTIL:
                return j > i
            if DH_OPEN.get(name_key(toks[j])):
                return False
        return False


# ============================================================ otzaria-mcp
class Cache(object):
    MISSING = object()

    def __init__(self, path):
        d = os.path.dirname(os.path.abspath(path))
        os.makedirs(d, exist_ok=True)
        self.db = sqlite3.connect(path)
        self.db.execute('CREATE TABLE IF NOT EXISTS kv (ns TEXT, k TEXT, v TEXT, PRIMARY KEY (ns, k))')
        self.db.commit()
        self.n = 0

    def get(self, ns, k):
        row = self.db.execute('SELECT v FROM kv WHERE ns=? AND k=?', (ns, k)).fetchone()
        return Cache.MISSING if row is None else json.loads(row[0])

    def put(self, ns, k, v):
        self.db.execute('INSERT OR REPLACE INTO kv VALUES (?,?,?)', (ns, k, json.dumps(v, ensure_ascii=False)))
        self.n += 1
        if self.n % 100 == 0:
            self.db.commit()

    def close(self):
        self.db.commit()
        self.db.close()


class McpError(Exception):
    pass


class McpClient(object):
    """Minimal MCP stdio client (newline-delimited JSON-RPC), as in ency_pairs.py."""

    def __init__(self, command, args=(), env=None, timeout=90):
        cmd = [command] + list(args)
        if os.name == 'nt' and command.lower().endswith(('.cmd', '.bat')):
            cmd = ['cmd', '/c'] + cmd
        e = dict(os.environ)
        e.update(env or {})
        self.timeout = timeout
        self.p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                                  stderr=subprocess.DEVNULL, env=e, bufsize=0)
        self.q = queue.Queue()
        threading.Thread(target=self._pump, daemon=True).start()
        self.seq = 0
        init = self._request('initialize', {'protocolVersion': '2024-11-05', 'capabilities': {},
                                            'clientInfo': {'name': 'sugyot-to-topics', 'version': VERSION}})
        self.server_version = ((init or {}).get('serverInfo') or {}).get('version') or ''
        self._send({'jsonrpc': '2.0', 'method': 'notifications/initialized'})

    def _pump(self):
        for raw in iter(self.p.stdout.readline, b''):
            raw = raw.strip()
            if raw:
                try:
                    self.q.put(json.loads(raw.decode('utf-8')))
                except ValueError:
                    pass
        self.q.put(None)

    def _send(self, obj):
        self.p.stdin.write((json.dumps(obj, ensure_ascii=False) + '\n').encode('utf-8'))
        self.p.stdin.flush()

    def _request(self, method, params):
        self.seq += 1
        want = self.seq
        self._send({'jsonrpc': '2.0', 'id': want, 'method': method, 'params': params})
        while True:
            try:
                msg = self.q.get(timeout=self.timeout)
            except queue.Empty:
                raise McpError('otzaria-mcp did not answer %s within %ss' % (method, self.timeout))
            if msg is None:
                raise McpError('otzaria-mcp exited')
            if msg.get('id') == want:
                if 'error' in msg:
                    raise McpError(str(msg['error']))
                return msg.get('result')

    def call(self, name, arguments):
        res = self._request('tools/call', {'name': name, 'arguments': arguments}) or {}
        text = ''.join(c.get('text', '') for c in res.get('content') or [] if c.get('type') == 'text')
        if res.get('isError'):
            raise McpError(text[:300])
        try:
            return json.loads(text)
        except ValueError:
            return {'_text': text}

    def close(self):
        try:
            self.p.stdin.close()
            self.p.terminate()
        except Exception:
            pass


def discover_otzaria_mcp():
    """The otzaria-mcp entry of Claude Desktop's config: (command, args, env) — as ency_pairs.py."""
    cands = []
    appdata = os.environ.get('APPDATA')
    if appdata:
        cands.append(os.path.join(appdata, 'Claude', 'claude_desktop_config.json'))
    local = os.environ.get('LOCALAPPDATA')
    if local:
        pk = os.path.join(local, 'Packages')
        if os.path.isdir(pk):
            for d in os.listdir(pk):
                if d.lower().startswith('claude'):
                    cands.append(os.path.join(pk, d, 'LocalCache', 'Roaming', 'Claude', 'claude_desktop_config.json'))
    home = os.path.expanduser('~')
    cands += [os.path.join(home, 'Library', 'Application Support', 'Claude', 'claude_desktop_config.json'),
              os.path.join(home, '.config', 'Claude', 'claude_desktop_config.json')]
    for path in cands:
        try:
            with open(path, encoding='utf-8') as f:
                cfg = json.load(f)
        except (OSError, ValueError):
            continue
        servers = cfg.get('mcpServers') or {}
        ranked = sorted(servers.items(), key=lambda kv: (kv[0] != 'otzaria-library', kv[0]))
        for name, s in ranked:
            blob = (name + ' ' + str(s.get('command', '')) + ' ' + ' '.join(map(str, s.get('args') or []))).lower()
            if ('otzaria' in blob and 'mcp' in blob and 'semantic' not in blob and 'sources' not in blob) \
                    or name == 'otzaria-library':
                return s.get('command'), list(s.get('args') or []), dict(s.get('env') or {})
    return None


class Library(object):
    """The calls this script makes, cached in sqlite per library version."""
    PAGE = 100

    def __init__(self, client, cache):
        self.c, self.cache = client, cache
        try:
            info = client.call('library_info', {}) or {}
        except McpError:
            info = {}
        self.ver = '%s|%s' % (client.server_version, info.get('db_version') or info.get('data_version') or '')
        self.lines_mem = collections.OrderedDict()
        self.totals = {}
        self.calls = 0

    def _call(self, ns, name, args):
        key = json.dumps(args, ensure_ascii=False, sort_keys=True)
        hit = self.cache.get(ns + '@' + self.ver, key)
        if hit is not Cache.MISSING:
            return hit
        self.calls += 1
        try:
            r = self.c.call(name, args)
        except McpError as e:
            log('  %s %s: %s' % (name, key[:80], e))
            return None                                       # not cached: asked again next run
        self.cache.put(ns + '@' + self.ver, key, r)
        return r

    def search_books(self, q, limit=20):
        r = self._call('books', 'search_books', {'query': q, 'limit': limit, 'sources': ['library']}) or {}
        return r.get('books') or []

    def lookup(self, name, limit=30):
        r = self._call('lookup', 'lookup_title', {'name': name, 'limit': limit}) or {}
        return r.get('books') or []

    def toc(self, book_id, parent_id=None, alt=None):
        a = {'book_id': book_id, 'limit': 1000}
        if parent_id is not None:
            a['parent_id'] = parent_id
        if alt:
            a['alt_structure_key'] = alt
        return self._call('toc', 'get_book_toc', a) or {}

    def resolve(self, ref):
        r = self._call('ref', 'resolve_ref', {'reference': ref, 'limit': 8, 'snippets': False,
                                              'heref_fallback': False, 'exclude_book_level': True}) or {}
        return [c for c in r.get('candidates') or [] if c.get('source', 'library') == 'library']

    def page(self, book_id, start):
        s0 = start // self.PAGE * self.PAGE
        r = self._call('text', 'get_text', {'book_id': book_id, 'line_index': s0, 'limit': self.PAGE,
                                            'strip_html': True}) or {}
        if r.get('total_lines') is not None:
            self.totals[book_id] = r['total_lines']
        return {l.get('line_index'): (l.get('he_ref'), l.get('content') or '') for l in r.get('lines') or []}

    def lines(self, book_id, start, end):
        """{line: (he_ref, text)} for [start, end)"""
        out = {}
        s = max(0, start)
        while s < end:
            tot = self.totals.get(book_id)
            if tot is not None and s >= tot:
                break
            key = (book_id, s // self.PAGE)
            pg = self.lines_mem.get(key)
            if pg is None:
                pg = self.page(book_id, s)
                self.lines_mem[key] = pg
                while len(self.lines_mem) > 400:
                    self.lines_mem.popitem(last=False)
            if not pg:
                break
            for li, v in pg.items():
                if s <= li < end:
                    out[li] = v
            s = (s // self.PAGE + 1) * self.PAGE
        return out

    def line(self, book_id, li):
        return self.lines(book_id, li, li + 1).get(li)


# ============================================================ matching
DH_SKIP = {'מתני', 'מתניתין', 'גמ', 'גמרא', 'דה', 'בדה', 'תוד', 'שם', 'וכו', 'כו', 'באד', 'באהד', 'בא'}


def tok_eq(a, b):
    if a == b:
        return True
    for x, y in ((a, b), (b, a)):
        if len(y) >= 2 and len(x) > len(y) and x.endswith(y) and len(x) - len(y) <= 2 \
                and all(ch in 'והבלמשדכ' for ch in x[:len(x) - len(y)]):
            return True
    if len(a) >= 3 and len(b) >= 3 and a[0] == b[0] and a[-1] == b[-1] and \
            re.sub('[וי]', '', a[1:-1]) == re.sub('[וי]', '', b[1:-1]):
        return True                                   # plene / defective spelling inside the word
    return False


def dh_score(dh_words, text):
    """How well a dibbur hamatchil opens this line (0..1)."""
    d = [t for t in plain_tokens(' '.join(dh_words)) if t not in ('וכו', 'כו', 'וגו')][:5]
    if not d:
        return 0.0
    L = plain_tokens(text)[:len(d) + 8]
    best = 0.0
    for start in range(0, min(4, len(L))):
        if start and not all(x in DH_SKIP for x in L[:start]):
            break
        if not tok_eq(d[0], L[start]):
            continue
        got, j = 1, start + 1
        for w in d[1:]:
            for jj in range(j, min(j + 2, len(L))):
                if tok_eq(w, L[jj]):
                    got += 1
                    j = jj + 1
                    break
        best = max(best, got / len(d))
    return best


def dh_ok(dh_words, score):
    n = len([t for t in plain_tokens(' '.join(dh_words)) if t not in ('וכו', 'כו')][:5])
    return score >= (1.0 if n <= 2 else 0.66)


def phrase_in(words, text):
    """the opening words of a gemara passage, contiguous (one gap allowed), anywhere in the line"""
    d = [t for t in plain_tokens(' '.join(words)) if t not in ('וכו', 'כו', 'וגו')][:5]
    if not d:
        return False
    L = plain_tokens(text)
    need = len(d) if len(d) <= 2 else len(d) - 1
    for s0 in range(len(L)):
        if not tok_eq(d[0], L[s0]):
            continue
        got, j = 1, s0 + 1
        for w in d[1:]:
            for jj in range(j, min(j + 2, len(L))):
                if tok_eq(w, L[jj]):
                    got += 1
                    j = jj + 1
                    break
        if got >= need:
            return True
    return False


HEADING_RE = re.compile('^(?:דף|סימן|סעיף|אות|פרק|הלכה|ס"ק|ענף|כלל|חלק|שער|מצוה|משנה|תשובה|שאלה)\\b')


def heading_like(text):
    t = norm_he(re.sub(r'<[^>]+>', ' ', text or ''))
    if not t:
        return False
    w = t.split()
    return len(w) <= 6 and (HEADING_RE.match(t) is not None or (len(w) <= 2 and all(num(x) for x in w)))


def skel(s):
    return re.sub('[וי ]', '', name_key(s).translate(FINALS))


GENERIC_TITLE_WORDS = _keys('ספר על שו״ת שות חידושי חדושי פירוש הלכות רבינו רבנו מסכת תלמוד קונטרס תשובות')


def title_has(title, name):
    """every significant word of `name` is in `title` (abbreviations as whole words, ה/ל prefixes
    allowed), or `name` is the acronym of consecutive title words"""
    T = skel(title)
    tw = [name_key(w) for w in norm_he(title).replace(',', ' ').replace('[', ' ').replace(']', ' ').split()]
    words = [w for w in norm_he(name).split() if name_key(w) not in GENERIC_TITLE_WORDS and re.search('[%s]' % HEB, w)]

    def has(w):
        if '"' in w or "'" in w:
            k = name_key(w)
            return any(x == k or (x.endswith(k) and len(x) - len(k) <= 2 and all(ch in 'הלדובמ' for ch in x[:len(x) - len(k)]))
                       for x in tw)
        w2 = w[1:] if w.startswith('ה') and len(w) > 3 else w
        return len(skel(w2)) >= 2 and skel(w2) in T
    if words and all(has(w) for w in words):
        return True
    if '"' in name and len(name.split()) == 1:
        letters = name_key(name).translate(FINALS)
        tws = [w.translate(FINALS) for w in tw]

        def rec(i, j):
            if i == len(letters):
                return True
            if j >= len(tws):
                return False
            for k in range(min(len(tws[j]), len(letters) - i), 0, -1):
                if letters[i:i + k] == tws[j][:k] and rec(i + k, j + 1):
                    return True
            return False
        return len(letters) >= 2 and any(rec(0, j) for j in range(len(tws)))
    return False


# ============================================================ resolution
class Hit(object):
    def __init__(self, book_id, title, start, end=None, status='ok', ref='', single=False):
        self.book_id, self.title, self.start = book_id, title, start
        self.end = start + 1 if end is None else end
        self.status, self.ref, self.single = status, ref, single

    def __repr__(self):
        return 'Hit(%s %s %d-%d %s)' % (self.status, self.title, self.start, self.end, self.ref)


def toc_daf(text):
    t = norm_he(re.sub(r'<[^>]+>', ' ', text or ''))
    t = re.sub(r'^(?:דף|ד\')\s*', '', t)
    m = re.fullmatch('([%s"\']{1,6})\\s*(\\.|:|ע"א|ע"ב|עמוד א|עמוד ב)?' % HEB, t)
    if not m or num(m.group(1)) is None:
        return None
    a = m.group(2)
    return num(m.group(1)), (None if not a else 'a' if a in ('.', 'ע"א', 'עמוד א') else 'b')


def heref_daf(he_ref):
    for comp in (he_ref or '').split(',')[1:3]:
        m = re.fullmatch('\\s*([%s]{1,4})([.:])\\s*' % HEB, comp)
        if m and num(m.group(1)):
            return num(m.group(1)), 'a' if m.group(2) == '.' else 'b'
    return None


def mas_in(title, mas):
    T = ' ' + norm_he(title).replace(',', ' ') + ' '
    return (' %s ' % mas) in T or (' מסכת %s ' % mas) in T


class Ctx(object):
    """What a dafless / "שם" citation leans on, per top-level topic."""

    def __init__(self, mas):
        self.mas = mas
        self.topic_dafs = []
        self.history = []          # resolved cites, in order
        self.hits_by_key = {}
        self.sa_section = None
        self.sa_siman = None
        self.rambam = {}

    def last(self, pred):
        for c in reversed(self.history):
            if pred(c):
                return c
        return None


class Resolver(object):
    def __init__(self, lib, args):
        self.lib, self.args = lib, args
        self.books = {}
        self.dafidx = {}
        self.chap = {}

    # ---------------------------------------------------------------- books
    def bavli(self, mas):
        k = ('bavli', mas)
        if k not in self.books:
            b = None
            for x in self.lib.search_books(mas, 10):
                if norm_he(x.get('title')) == mas and x.get('is_base_book', True) and not x.get('is_pdf'):
                    b = x
                    break
            self.books[k] = b
        return self.books[k]

    def talmud_book(self, c, mas, forms=None):
        forms = forms or [f.replace('{M}', mas) for f in c.forms] or [c.key + ' על ' + mas, c.key + ' ' + mas]
        k = (c.key, mas, tuple(forms))
        if k in self.books:
            return self.books[k]
        def ok(b, f):
            t = norm_he(b.get('title', ''))
            if b.get('is_pdf') or not mas_in(t, mas):
                return False
            if 'ירושלמי' in t and 'ירושלמי' not in f:
                return False
            if re.search(r'משנה %s(\s|$)' % re.escape(mas), t) and 'משנה' not in f:
                return False
            if 'משנה תורה' in t:
                return False
            return title_has(t, f) or bool(c.key and title_has(t, c.key + ' ' + mas))

        best = None
        for f in forms:
            cands = [b for b in self.lib.lookup(f, 10) + self.lib.search_books(f, 15) if ok(b, f)]
            if cands:
                best = sorted(cands, key=lambda b: (norm_he(b['title']) != norm_he(f), len(b['title'])))[0]
                break
        if best is None and c.key:
            # the acronym index, then every book "על <masechet>" (e.g. Shabbat's ריטב"א is printed as
            # "חידושי הרן על שבת [מיוחס להריטבא]")
            pool = self.lib.lookup(c.key, 50) + self.catalogue(mas)
            cands = [b for b in pool if ok(b, c.key + ' על ' + mas)]
            if cands:
                best = sorted(cands, key=lambda b: len(b['title']))[0]
        self.books[k] = best
        return best

    def catalogue(self, mas):
        k = ('cat', mas)
        if k not in self.books:
            seen, out = set(), []
            for q in ('על ' + mas, 'על מסכת ' + mas):
                for b in self.lib.search_books(q, 100):
                    if b['book_id'] not in seen:
                        seen.add(b['book_id'])
                        out.append(b)
            self.books[k] = out
        return self.books[k]

    def find_books(self, name):
        k = ('name', name)
        if k not in self.books:
            out, seen = [], set()
            for b in self.lib.lookup(name, 30) + self.lib.search_books(name, 30):
                if not b.get('is_pdf') and b['book_id'] not in seen and title_has(b.get('title', ''), name):
                    seen.add(b['book_id'])
                    out.append(b)
            self.books[k] = out
        return self.books[k]

    # ----------------------------------------------------------------- dafs
    def dafs_of(self, book_id):
        if book_id in self.dafidx:
            return self.dafidx[book_id]
        marks = []
        for e in self.lib.toc(book_id).get('entries') or []:
            d = toc_daf(e.get('text'))
            if d and e.get('line_index') is not None:
                marks.append((e['line_index'], d))
        marks.sort()
        out = {}
        for i, (li, d) in enumerate(marks):
            end = marks[i + 1][0] if i + 1 < len(marks) else li + 400
            out.setdefault(d, (li, end))
        self.dafidx[book_id] = out
        return out

    def daf_range(self, book, daf):
        bid = book['book_id']
        idx = self.dafs_of(bid)
        n, a = daf
        if a:
            r = idx.get((n, a)) or idx.get((n, None))
        else:
            ra, rb = idx.get((n, 'a')), idx.get((n, 'b'))
            r = (ra[0], rb[1]) if ra and rb else (ra or rb or idx.get((n, None)))
        if r or idx:
            return r
        # no daf TOC: find-ref "<title> ב." and walk the he_refs
        for amud in ([a] if a else ['a', 'b']):
            for c in self.lib.resolve('%s %s%s' % (book['title'], heb(n), '.' if amud == 'a' else ':')):
                if c.get('book_id') != bid:
                    continue
                s = c.get('line_index')
                ls = self.lib.lines(bid, s, s + 300)
                e = s + 1
                for li in sorted(ls):
                    d = heref_daf(ls[li][0])
                    if li > s and d and d != (n, amud) and (a or d[0] != n):
                        break
                    e = li + 1
                return s, e
        return None

    def daf_of_line(self, book, li):
        for d, (s, e) in self.dafs_of(book['book_id']).items():
            if s <= li < e and d[1]:
                return d
        return None

    # -------------------------------------------------------------- perakim
    def chapters(self, book_id):
        if book_id not in self.chap:
            out = []
            r = self.lib.toc(book_id, alt='Chapters') or {}

            def walk(es):
                for e in es or []:
                    if e.get('line_index') is not None:
                        out.append((e['line_index'], norm_he(e.get('text') or e.get('title') or '')))
                    walk(e.get('children'))
            walk(r.get('alt_entries'))
            if not out:
                for e in self.lib.toc(book_id).get('entries') or []:
                    if norm_he(e.get('text', '')).startswith('פרק') and e.get('line_index') is not None:
                        out.append((e['line_index'], norm_he(e['text'])))
            self.chap[book_id] = sorted(out)
        return self.chap[book_id]

    def perek_of_daf(self, mas, daf):
        b = self.bavli(mas)
        if not b:
            return None
        r = self.daf_range(b, daf)
        ch = self.chapters(b['book_id'])
        if not r or not ch:
            return None
        return sum(1 for li, _ in ch if li <= r[0]) or None

    def perek_by_name(self, mas, name):
        b = self.bavli(mas)
        if not b or not name:
            return None
        want = plain_tokens(name)
        for i, (li, t) in enumerate(self.chapters(b['book_id'])):
            tt = plain_tokens(t)
            if want and all(any(tok_eq(w, x) for x in tt) for w in want):
                return i + 1
        return None

    def perek_start_daf(self, mas, p):
        b = self.bavli(mas)
        ch = self.chapters(b['book_id']) if b else []
        if not ch or not p or p > len(ch):
            return None
        return self.daf_of_line(b, ch[p - 1][0])

    # ------------------------------------------------------ TOC navigation
    def _children(self, book_id, parent_id):
        r = self.lib.toc(book_id, parent_id=parent_id) or {}
        es = r.get('entries') or []
        return [e for e in es if e.get('parent_id') == parent_id] if parent_id is not None else \
            [e for e in es if e.get('parent_id') is None]

    @staticmethod
    def _entry_num(text):
        t = norm_he(re.sub(r'<[^>]+>', ' ', text or ''))
        t = re.sub(r'^(?:סימן|סי\'|סעיף|ס"ק|סק|אות|פרק|הלכה|הל\'|דף|ענף|כלל|חלק|שער|מצוה|משנה|תשובה|שורש)\s*', '', t)
        w = t.split()
        return num(w[0]) if w and len(w) <= 3 else None

    def toc_find(self, book_id, nums):
        """walk the TOC by numbers (siman, ס"ק ...) -> (line, end) of the deepest entry found"""
        level = self._children(book_id, None)
        found, found_end = None, None
        for depth, n in enumerate(nums):
            hit = None
            for wrap in range(3):
                for j, e in enumerate(level):
                    if self._entry_num(e.get('text')) == n:
                        hit = (e, level[j + 1].get('line_index') if j + 1 < len(level) else None)
                        break
                if hit or not level:
                    break
                wrappers = [e for e in level if e.get('has_children') and self._entry_num(e.get('text')) is None]
                if len(wrappers) != 1 and not (wrap == 0 and len(level) == 1):
                    break
                level = self._children(book_id, (wrappers or level)[0].get('id'))
            if not hit:
                break
            e, nxt = hit
            found = e.get('line_index')
            found_end = nxt if nxt is not None else (found_end if found_end and found_end > found else None)
            level = self._children(book_id, e.get('id')) if e.get('has_children') else []
            if depth + 1 < len(nums) and not level:
                break
        if found is None:
            return None
        return found, (found_end if found_end and found_end > found else found + 400)

    # ------------------------------------------------------------- units
    def unit_of(self, cand, nums):
        """lines a find-ref candidate stands for"""
        bid, li = cand['book_id'], cand['line_index']
        hr = cand.get('he_ref')
        if cand.get('match_kind') in ('exact', 'dibur_in_ref'):
            return li, li + 1
        if hr:
            comps = [x.strip() for x in hr.split(',')]
            cut = len(comps) - 1
            want = [heb(n) for n in nums]
            for i2 in range(len(comps) - 1, 0, -1):
                if want and name_key(comps[i2]) == want[-1]:
                    cut = i2 + 1
                    break
            prefix = ', '.join(comps[:cut])
            ls = self.lib.lines(bid, li, li + 400)
            end = li + 1
            for x in sorted(ls):
                h = ls[x][0]
                if x > li and h and not norm_he(h).startswith(norm_he(prefix)):
                    break
                end = x + 1
            return li, end
        ls = self.lib.lines(bid, li + 1, li + 400)
        end = li + 1
        for x in sorted(ls):
            if heading_like(ls[x][1]):
                break
            end = x + 1
        return li, end

    def pick(self, cands, name, nums, prefer=()):
        def nums_ok(c):
            if not nums:
                return True
            ref = ' '.join(x for x in (c.get('reference'), c.get('he_ref'), c.get('toc_path')) if x)
            toks = {name_key(t) for t in re.split(r'[\s,>]+', norm_he(ref)) if t}
            return heb(nums[0]) in toks
        good = [c for c in cands if title_has(c.get('book_title', ''), name) and nums_ok(c)]
        if not good:
            return None
        best = min(c.get('rank', 0) for c in good)
        top = [c for c in good if c.get('rank', 0) == best]
        if len({c['book_id'] for c in top}) > 1:
            pref = [c for c in top if any(w and w in norm_he(c.get('book_title', '')) for w in prefer)]
            if len({c['book_id'] for c in pref}) == 1:
                top = pref
            else:
                return None
        if len({c['line_index'] for c in top}) > 1 and len({c.get('toc_path') for c in top}) > 1:
            pref = [c for c in top if any(w and w in norm_he(c.get('toc_path') or c.get('reference') or '')
                                          for w in prefer)]
            if len({c['line_index'] for c in pref}) == 1:
                return pref[0]
            return None                                           # "תרומת הדשן נז": חלק א or ב
        return top[0]

    # ----------------------------------------------------------------- text
    def find_dh(self, bid, start, end, dh, ordinal=1):
        ls = self.lib.lines(bid, start, min(end, start + 600))
        good = []
        for li in sorted(ls):
            tx = ls[li][1]
            if heading_like(tx):
                continue
            if dh_ok(dh, dh_score(dh, tx)):
                good.append(li)
        if not good:
            return None
        return good[min(ordinal, len(good)) - 1]

    def find_phrase(self, bid, start, end, words):
        ls = self.lib.lines(bid, start, min(end, start + 600))
        for li in sorted(ls):
            if phrase_in(words, ls[li][1]):
                return li
        return None

    def text_of(self, hit, max_words):
        ls = self.lib.lines(hit.book_id, hit.start, hit.end)
        idx = [li for li in sorted(ls) if not (heading_like(ls[li][1]) and len(ls) > 1)]
        out, used, words = [], [], 0
        for li in idx:
            t = norm_he(re.sub(r'<[^>]+>', ' ', ls[li][1]))
            if not t:
                continue
            w = t.split()
            if max_words and words + len(w) > max_words:
                left = max_words - words
                if left > 15 or not out:
                    out.append(' '.join(w[:max(left, 15)]) + ' …')
                    used.append(li)
                break
            out.append(t)
            used.append(li)
            words += len(w)
        he = ls[used[0]][0] if used else None
        return '\n'.join(out), used, he

    # ------------------------------------------------------------ context
    def fill(self, c, ctx, line_prev):
        prev_all = list(line_prev) + list(reversed(ctx.history))
        if c.kind is None:
            src = line_prev[-1] if line_prev else (ctx.history[-1] if ctx.history else None)
            if src is None:
                return False
            c.kind, c.key, c.forms = src.kind, src.key, list(src.forms)
            c.mas = c.mas or src.mas
            if not c.ibid and not c.has_place():
                c.ibid = True
        if c.ibid:
            same = [p for p in reversed(line_prev) if p.has_place()] + \
                   [p for p in reversed(ctx.history) if p.has_place()]
            base = None
            if c.kind in ('T', 'G', 'P'):
                base = next((p for p in same if p.dafs and p.kind in ('T', 'G', 'P', '?', 'O')), None)
            elif c.kind in ('S', 'SC'):
                base = next((p for p in same if p.kind in ('S', 'SC') and p.loc.get('siman')), None)
            elif c.kind in ('R', 'RC'):
                base = next((p for p in same if p.kind in ('R', 'RC') and p.loc.get('perek')), None)
            else:
                base = next((p for p in same if p.key == c.key), None) or (same[0] if same else None)
            if base is not None:
                if not c.dafs and base.dafs:
                    c.dafs = list(base.dafs)
                    c.mas = c.mas or base.mas
                    c.rif_pages = c.rif_pages or base.rif_pages
                for k2, v in base.loc.items():
                    if k2 not in c.loc and not (c.kind == 'SC' and k2 == 'seif' and base.kind == 'S'):
                        c.loc[k2] = v
                c.hilchot = c.hilchot or base.hilchot
                c.section = c.section or base.section
        if c.kind in ('S', 'SC'):
            c.section = c.section or ctx.sa_section or SA_SECTION_OVERRIDE.get(ctx.mas) or \
                SA_SECTION_OF_SEDER.get(SEDER_OF.get(ctx.mas or '', ''), 'אורח חיים')
            if not c.loc.get('siman') and ctx.sa_siman and c.kind == 'SC':
                c.loc['siman'] = ctx.sa_siman
        if c.kind in ('R', 'RC'):
            if not c.hilchot:
                c.hilchot = ctx.rambam.get('hilchot') or RAMBAM_HILCHOT_OF.get(ctx.mas or '')
            if c.kind == 'RC' and not c.loc.get('perek') and ctx.rambam.get('perek'):
                c.loc.setdefault('perek', ctx.rambam['perek'])
                if ctx.rambam.get('halacha'):
                    c.loc.setdefault('halacha', ctx.rambam['halacha'])
        if c.hilchot:
            c.hilchot = re.sub(r'^(?:ה?הלכות|הל\'|מהל\'|ד)\s*', '', norm_he(c.hilchot)).strip()
            if c.hilchot.startswith('ד') and c.hilchot[1:] in RAMBAM_HILCHOT_OF.values():
                c.hilchot = c.hilchot[1:]
        return True

    def remember(self, c, ctx, hits):
        ctx.history.append(c)
        if any(hits):
            ctx.hits_by_key[c.key] = [h for h in hits if h]
        if c.kind in ('S', 'SC'):
            if c.section:
                ctx.sa_section = c.section
            if c.loc.get('siman') and c.kind == 'S':
                ctx.sa_siman = c.loc['siman']
        if c.kind == 'R' and c.loc.get('perek'):
            ctx.rambam = {'hilchot': c.hilchot, 'perek': c.loc.get('perek'), 'halacha': c.loc.get('halacha')}

    # ------------------------------------------------------------ resolve
    def resolve(self, c, ctx, line_prev):
        """-> [Hit or None] (one per dibbur, or one)"""
        if not self.fill(c, ctx, line_prev):
            return [None]
        if c.backref and not c.has_place() and not c.dhs and ctx.hits_by_key.get(c.key):
            return list(ctx.hits_by_key[c.key])
        k = c.kind
        try:
            if k == 'G':
                return [self.r_gemara(c, ctx, line_prev)]
            if k == 'T' and (c.dafs or c.dhs or not c.loc or c.loc.get('perek') or c.perek_name):
                return self.r_talmud(c, ctx, line_prev)
            if k == 'P':
                return self.r_rosh(c, ctx, line_prev)
            if k == 'R':
                return [self.r_rambam(c)]
            if k == 'RC':
                return self.r_rambam_comm(c)
            if k == 'RX':
                return self.r_rx(c, ctx, line_prev)
            if k == 'S':
                return [self.r_sa(c)]
            if k == 'SC':
                return self.r_sc(c)
            if c.dafs and (c.mas or ctx.mas):
                r = self.r_talmud(c, ctx, line_prev)
                if any(r):
                    return r
            return self.r_other(c, ctx)
        except McpError as e:
            log('  ! %s: %s' % (c.text(), e))
            return [None]

    def daf_candidates(self, c, ctx, line_prev, mas):
        if c.dafs and not c.rif_pages:
            return list(c.dafs), False
        cands = []
        p = c.loc.get('perek') or (self.perek_by_name(mas, c.perek_name) if c.perek_name else None)
        if p and (c.rosh or not c.dhs):
            d = self.perek_start_daf(mas, p)
            if d:
                return [d], not c.rosh
        if mas == ctx.mas:
            for q in reversed(line_prev):
                if q.dafs and (q.mas or ctx.mas) == mas and not q.rif_pages:
                    cands += q.dafs
                    break
            cands += ctx.topic_dafs
        seen, out = set(), []
        for d in cands:
            if d not in seen:
                seen.add(d)
                out.append(d)
        return out, True

    def _dh_hits(self, c, book, ranges, loose, until_ok=True):
        bid, title = book['book_id'], book['title']
        out = []
        for k, dh in enumerate(c.dhs or [None]):
            found = None
            for (s, e) in ranges:
                if dh is None:
                    found = Hit(bid, title, s, e, 'loose' if loose else 'ok')
                    break
                li = self.find_dh(bid, s, e, dh, c.ordinal.get(k, 1))
                if li is not None:
                    found = Hit(bid, title, li, li + 1, 'ok', single=True)
                    if until_ok and c.until and k == len(c.dhs) - 1:
                        lim = e if not c.until.get('daf') else (self.daf_range(book, c.until['daf']) or (s, e))[1]
                        if c.until.get('dh'):
                            j = self.find_dh(bid, li + 1, max(lim, e + 200), c.until['dh'])
                            if j is not None:
                                found.end, found.single = j + 1, False
                        elif c.until.get('end'):
                            found.end, found.single = lim, False
                        elif c.until.get('words'):
                            j = self.find_phrase(bid, li, lim, [w for w in c.until['words']
                                                                if name_key(w) not in END_WORDS])
                            if j is not None and j > li:
                                found.end, found.single = j + 1, False
                    break
            if found is None and dh is not None and ranges and not loose:
                s, e = ranges[0]
                found = Hit(bid, title, s, e, 'partial')
            out.append(found)
        return out

    def r_talmud(self, c, ctx, line_prev, forms=None):
        mas = c.mas or ctx.mas
        if not mas:
            return [None]
        book = self.talmud_book(c, mas, forms)
        if book is None and c.kind in ('O', '?'):
            book = self.talmud_book(c, mas, [c.key + ' על ' + mas, c.key + ' על מסכת ' + mas, c.key + ' ' + mas])
        if book is None:
            return [None]
        dafs, loose = self.daf_candidates(c, ctx, line_prev, mas)
        ranges = [r for r in (self.daf_range(book, d) for d in dafs) if r]
        if not ranges:
            return [None]
        return self._dh_hits(c, book, ranges, loose)

    def r_gemara(self, c, ctx, line_prev):
        mas = c.mas or ctx.mas
        if not mas:
            return None
        if mas in MISHNA_ONLY or (c.key == 'משנה' and c.loc.get('perek')):
            nums = [c.loc[k] for k in ('perek', 'mishna') if c.loc.get(k)]
            if not nums:
                return None
            ref = 'משנה %s %s' % (mas, ' '.join(heb(n) for n in nums))
            cand = self.pick(self.lib.resolve(ref), 'משנה ' + mas, nums)
            if not cand:
                return None
            s, e = self.unit_of(cand, nums)
            return Hit(cand['book_id'], cand['book_title'], s, e, 'ok')
        book = self.bavli(mas)
        if not book:
            return None
        dafs, loose = self.daf_candidates(c, ctx, line_prev, mas)
        ranges = [(d, self.daf_range(book, d)) for d in dafs]
        ranges = [(d, r) for d, r in ranges if r]
        if not ranges:
            return None
        bid, title = book['book_id'], book['title']
        mishna = c.key == 'משנה' or c.on_mishna
        for d, (s, e) in ranges:
            start, ok = s, not (mishna or c.anchor)
            ls = self.lib.lines(bid, s, e + 60)
            order = sorted(ls)
            if mishna:
                for li in order:
                    if li < e and plain_tokens(ls[li][1])[:1] in (['מתני'], ['מתניתין']):
                        start, ok = li, True
                        break
            if c.anchor:
                li = self.find_phrase(bid, start, e, c.anchor)
                if li is not None:
                    start, ok = li, True
                elif mishna and ok:
                    pass
                else:
                    ok = False
            if not ok and len(ranges) > 1:
                continue
            # where it ends
            end, lim = (start + 1 if (mishna or c.anchor) else e), e
            u = c.until or {}
            if u.get('daf'):
                ur = self.daf_range(book, u['daf'])
                if ur:
                    lim = ur[1]
            words = [w for w in u.get('words', []) if name_key(w) not in END_WORDS and name_key(w) not in UNTIL]
            if words:
                j = self.find_phrase(bid, start + 1, max(lim, e) + (0 if u.get('daf') else 60), words)
                end = j + 1 if j is not None else max(end, lim if u.get('daf') else end)
            elif u.get('end') or u.get('daf'):
                end = lim
                ls2 = self.lib.lines(bid, start + 1, lim)
                for li in sorted(ls2):
                    if plain_tokens(ls2[li][1])[:1] in (['מתני'], ['מתניתין']) and \
                            any(name_key(w) == 'המשנה' for w in u.get('words', [])):
                        end = li
                        break
            elif mishna and not c.anchor:
                ls2 = self.lib.lines(bid, start + 1, start + 20)
                end = start + 1
                for li in sorted(ls2):
                    if plain_tokens(ls2[li][1])[:1] in (['גמ'], ['גמרא']):
                        break
                    end = li + 1
            status = 'ok' if ok and not loose else ('loose' if ok else 'partial')
            return Hit(bid, title, start, max(end, start + 1), status)
        d, (s, e) = ranges[0]
        return Hit(bid, title, s, e, 'partial' if not loose else 'loose')

    def r_rosh(self, c, ctx, line_prev):
        mas = c.mas or ctx.mas
        book = self.talmud_book(c, mas) if mas else None
        if not book:
            return [None]
        if c.dafs and not c.rif_pages and not c.loc.get('siman'):
            return self.r_talmud(c, ctx, line_prev)
        p = c.loc.get('perek') or (self.perek_by_name(mas, c.perek_name) if c.perek_name else None)
        loose = False
        if not p:
            dafs, _ = self.daf_candidates(c, ctx, line_prev, mas)
            p = self.perek_of_daf(mas, dafs[0]) if dafs else None
            loose = True
        if not p:
            return [None]
        nums = [p] + ([c.loc['siman']] if c.loc.get('siman') else [])
        cand = self.pick(self.lib.resolve('%s %s' % (book['title'], ' '.join(heb(n) for n in nums))),
                         book['title'], [])
        se = None
        if cand and cand['book_id'] == book['book_id']:
            se = self.unit_of(cand, nums)
        if not se:
            se = self.toc_find(book['book_id'], nums)
        if not se:
            return [None]
        unit = Hit(book['book_id'], book['title'], se[0], se[1], 'loose' if (loose and not c.loc.get('siman')) else 'ok')
        return self._in_unit(c, unit)

    def _in_unit(self, c, unit):
        """dibburim inside a located unit (siman / halacha / perek)"""
        if not c.dhs:
            return [unit]
        out = []
        for k, dh in enumerate(c.dhs):
            li = self.find_dh(unit.book_id, unit.start, unit.end, dh, c.ordinal.get(k, 1))
            out.append(Hit(unit.book_id, unit.title, li, li + 1, unit.status, single=True) if li is not None
                       else Hit(unit.book_id, unit.title, unit.start, unit.end, 'partial'))
        return out

    def _ref_hit(self, refs, name, nums, prefer=(), status='ok', must=()):
        for ref in refs:
            cand = self.pick(self.lib.resolve(ref), name, nums, prefer)
            if cand and all(m in norm_he(cand.get('book_title', '')) for m in must):
                s, e = self.unit_of(cand, nums)
                return Hit(cand['book_id'], cand['book_title'], s, e, status)
        return None

    def r_rambam(self, c):
        hil, p, h = c.hilchot, c.loc.get('perek'), c.loc.get('halacha')
        if not hil or not p:
            return None
        nums = [p] + ([h] if h else [])
        refs = ['רמב"ם הלכות %s %s' % (hil, ' '.join(heb(n) for n in nums))]
        return self._ref_hit(refs, 'משנה תורה', nums, must=(hil.split()[0],))

    def r_rambam_comm(self, c):
        hil, p, h = c.hilchot, c.loc.get('perek'), c.loc.get('halacha')
        if not hil or not p:
            return [None]
        nums = [p] + ([h] if h else [])
        names = [c.key] + (['השגות הראב"ד'] if c.key == 'השגות הראב"ד' else [])
        if c.key == 'חידושי רבנו חיים הלוי':
            names = ['חידושי רבנו חיים הלוי', 'חידושי רבינו חיים הלוי', 'חידושי הגר"ח']
        for nm in names:
            unit = self._ref_hit(['%s הלכות %s %s' % (nm, hil, ' '.join(heb(n) for n in nums)),
                                  '%s על משנה תורה, הלכות %s %s' % (nm, hil, ' '.join(heb(n) for n in nums))],
                                 nm, nums, must=(hil.split()[0],))
            if unit:
                return self._in_unit(c, unit)
        return [None]

    def r_rx(self, c, ctx, line_prev):
        if c.key == 'ספר המצוות':
            n = c.loc.get('mitzva') or c.loc.get('siman')
            if not n:
                return [None]
            kinds = [c.mitzva_kind] if c.mitzva_kind else ['לא תעשה', 'עשה']
            for kd in kinds:
                h = self._ref_hit(['ספר המצוות לרמב"ם %s %s' % (kd, heb(n))], 'ספר המצוות', [n])
                if h:
                    return [h]
            return [None]
        mas = c.mas or ctx.mas
        book = self.talmud_book(c, mas, ['רמב"ם על משנה ' + mas]) if mas else None
        if not book:
            return [None]
        p = c.loc.get('perek')
        if not p:
            dafs, _ = self.daf_candidates(c, ctx, line_prev, mas)
            p = self.perek_of_daf(mas, dafs[0]) if dafs else None
        if not p:
            return [None]
        se = self.toc_find(book['book_id'], [p] + ([c.loc['mishna']] if c.loc.get('mishna') else []))
        if not se:
            return [None]
        return self._in_unit(c, Hit(book['book_id'], book['title'], se[0], se[1], 'ok'))

    def r_sa(self, c):
        s, f = c.loc.get('siman'), c.loc.get('seif')
        if not s:
            return None
        sec = c.section or 'אורח חיים'
        refs = ['שולחן ערוך %s סימן %s%s' % (sec, heb(s), (' סעיף %s' % heb(f)) if f else '')]
        return self._ref_hit(refs, 'שולחן ערוך', [s] + ([f] if f else []), must=(sec.split()[0],))

    SC_REFS = {
        'מגן אברהם': ['מגן אברהם {siman} {sk}', 'מגן אברהם סימן {siman}'],
        'משנה ברורה': ['משנה ברורה {siman} {sk}', 'משנה ברורה סימן {siman} ס"ק {sk}', 'משנה ברורה סימן {siman}'],
        'ביאור הלכה': ['ביאור הלכה סימן {siman} סעיף {seif}', 'ביאור הלכה סימן {siman}'],
        'שער הציון': ['שער הציון סימן {siman} ס"ק {sk}', 'שער הציון {siman} {sk}', 'שער הציון סימן {siman}'],
        'טור': ['טור {sec} {siman}', 'טור {sec} סימן {siman}'],
        'בית יוסף': ['בית יוסף {sec} {siman}', 'בית יוסף {sec} סימן {siman}'],
        'פרי מגדים': ['{pmg} {sec} {siman} {sk}', 'פרי מגדים {pmg} {siman} {sk}', 'פרי מגדים {sec} {siman} {sk}'],
    }
    SC_DEFAULT = ['{name} {sec} סימן {siman} ס"ק {sk}', '{name} {sec} {siman} {sk}', '{name} סימן {siman} ס"ק {sk}',
                  '{name} {siman} {sk}', '{name} {sec} סימן {siman}', '{name} סימן {siman}']

    def r_sc(self, c):
        s = c.loc.get('siman')
        if not s:
            return [None]
        sk = c.loc.get('sk') or c.loc.get('os') or (c.loc.get('seif') if c.key not in ('ביאור הלכה',) else None)
        vals = {'name': c.key, 'sec': c.section or 'אורח חיים', 'siman': heb(s), 'sk': heb(sk) if sk else '',
                'seif': heb(c.loc.get('seif')) if c.loc.get('seif') else '', 'pmg': c.pmg or ''}
        refs = []
        for tpl in self.SC_REFS.get(c.key, []) + self.SC_DEFAULT:
            need = re.findall(r'{(\w+)}', tpl)
            if all(vals.get(n) for n in need):
                refs.append(re.sub(r'\s+', ' ', tpl.format(**vals)))
        nums = [s] + ([sk] if sk and c.key != 'ביאור הלכה' else [])
        name = c.pmg if c.pmg else c.key
        other_secs = [x for x in set(SECTIONS.values()) if x != vals['sec']]
        for ref in dict.fromkeys(refs):
            for cand in [self.pick(self.lib.resolve(ref), name, nums, prefer=(vals['sec'],))]:
                if cand and not any(o in norm_he(cand.get('book_title', '')) for o in other_secs):
                    se = self.unit_of(cand, nums if '{sk}' in ref or '{seif}' in ref else [s])
                    unit = Hit(cand['book_id'], cand['book_title'], se[0], se[1], 'ok')
                    return self._in_unit(c, unit)
        return self.r_other(c, None)

    def r_other(self, c, ctx):
        name = c.key
        if not name:
            return [None]
        order = ('chelek', 'shaar', 'siman', 'anaf', 'klal', 'perek', 'halacha', 'mishna', 'mitzva', 'seif', 'sk', 'os')
        word = {'chelek': 'חלק', 'shaar': 'שער', 'siman': 'סימן', 'anaf': 'ענף', 'klal': 'כלל', 'perek': 'פרק',
                'halacha': 'הלכה', 'mishna': 'משנה', 'mitzva': 'מצוה', 'seif': 'סעיף', 'sk': 'ס"ק', 'os': 'אות'}
        keys = [k for k in order if c.loc.get(k)]
        nums = [c.loc[k] for k in keys]
        if not nums and not c.dafs:
            return [None]
        names = [name]
        if name.startswith('שו"ת '):
            names.append(name[5:])
        elif c.kind in ('O', '?'):
            names.append('שו"ת ' + name)
        sec = c.section
        if not sec and ctx is not None and 'siman' in c.loc:
            sec = SA_SECTION_OVERRIDE.get(ctx.mas) or SA_SECTION_OF_SEDER.get(SEDER_OF.get(ctx.mas or '', ''))
        seder = SEDER_OF.get(ctx.mas or '', '') if ctx is not None else ''
        prefer = tuple(x for x in (sec, seder, c.mas, ctx.mas if ctx else None) if x)
        locs = [' '.join('%s %s' % (word[k], heb(c.loc[k])) for k in keys),
                ' '.join(heb(c.loc[k]) for k in keys)]
        if c.dafs:
            d = c.dafs[0]
            locs = ['%s %s%s' % (c.mas or (ctx.mas if ctx else ''), heb(d[0]), '.' if d[1] != 'b' else ':')] + locs
        refs = []
        for nm in names:
            for lc in locs:
                if sec:
                    refs.append('%s %s %s' % (nm, sec, lc))
                refs.append('%s %s' % (nm, lc))
        for ref in dict.fromkeys(r.strip() for r in refs):
            cand = self.pick(self.lib.resolve(ref), names[-1] if name.startswith('שו"ת ') else name, nums[:1],
                             prefer)
            if cand:
                se = self.unit_of(cand, nums)
                return self._in_unit(c, Hit(cand['book_id'], cand['book_title'], se[0], se[1], 'ok'))
        # TOC walk in the book (or the part of a multi-part work that fits the context)
        if nums:
            books = []
            for nm in names:
                books += [b for b in self.find_books(nm) if b not in books]
            if c.mas:
                books = [b for b in books if mas_in(b.get('title', ''), c.mas)] or books
            if len(books) > 1:
                pref = [b for b in books if any(p in norm_he(b.get('title', '')) for p in prefer)]
                books = pref if len(pref) == 1 else (books if len({b['title'].split(',')[0] for b in books}) == 1
                                                     and len(books) <= 1 else pref[:1])
            for b in books[:1]:
                se = self.toc_find(b['book_id'], ([c.loc['chelek']] if c.loc.get('chelek') else []) +
                                   [n for k, n in zip(keys, nums) if k != 'chelek'])
                if se:
                    return self._in_unit(c, Hit(b['book_id'], b['title'], se[0], se[1], 'ok'))
        return [None]


# ================================================================ output
_SEQ = [0]


def uid():
    _SEQ[0] += 1
    n = int(time.time() * 1000) + _SEQ[0]
    s = ''
    while n:
        n, r = divmod(n, 36)
        s = '0123456789abcdefghijklmnopqrstuvwxyz'[r] + s
    return s + ''.join(random.choice('abcdefghijklmnopqrstuvwxyz0123456789') for _ in range(4))


MARK = {'ok': '', 'loose': '≈ ', 'partial': '⚠ '}


def new_node(name):
    return {'id': uid(), 'name': name, 'chrono': False, 'noDivAt': 0, 'dividers': [], 'sources': [],
            'images': {}, 'children': []}


def ref_of(he_ref, title):
    r = norm_he(he_ref or '')
    t = norm_he(title or '')
    if t and r.startswith(t):
        r = r[len(t):]
    return re.sub(r'\s+', ' ', r).strip(' ,')


def expand_dafs(c):
    """the amudim a sugya's opening line covers ("ב' ע"א ... עד ג' ע"א")"""
    out = []
    for n, a in c.dafs:
        out += [(n, a or 'a')] + ([(n, 'b')] if a != 'b' else [])
    u = (c.until or {}).get('daf')
    if u and c.dafs:
        n, a = c.dafs[-1]
        cur = (n, a or 'a')
        for _ in range(12):
            if cur not in out:
                out.append(cur)
            if cur[0] > u[0] or (cur[0] == u[0] and (cur[1] == (u[1] or 'b') or cur[1] == 'b')):
                break
            cur = (cur[0], 'b') if cur[1] == 'a' else (cur[0] + 1, 'a')
    return out


def guess_masechet(args, items, stem):
    if args.masechet:
        return norm_he(args.masechet)
    for src in (args.title or '', stem):
        for k in sorted(MASECHTOT, key=len, reverse=True):
            if re.search(r'(^|[\s\-–_,])%s($|[\s\-–_,.])' % re.escape(k), norm_he(src)):
                return MASECHTOT[k]
    cnt = collections.Counter()
    for it in items:
        for w in norm_he(it['text']).split():
            if w in MASECHTOT:
                cnt[MASECHTOT[w]] += 1
    return cnt.most_common(1)[0][0] if cnt else None


class Builder(object):
    def __init__(self, args, resolver, mas):
        self.args, self.res, self.mas = args, resolver, mas
        self.parser = Parser()
        self.review = []
        self.stats = collections.Counter()
        self.ts = int(time.time() * 1000)

    def _ts(self):
        self.ts += 1
        return self.ts

    def manual(self, book, text=''):
        self.stats['manual'] += 1
        return {'id': uid(), 'bookId': None, 'book': book, 'ref': '', 'index': 0, 'text': text, 'imgs': [],
                'range': None, 'para': False, 'manual': True, 'added': self._ts()}

    def from_hit(self, hit, cite_text):
        single = hit.single or hit.end - hit.start <= 1
        if hit.status == 'ok':
            text, used, he = self.res.text_of(hit, self.args.max_words if single else self.args.max_range_words)
        else:                    # ≈ / ⚠: the place opens, but its text would be a guess
            ls = self.res.lib.lines(hit.book_id, hit.start, hit.start + 3)
            first = next((li for li in sorted(ls) if not heading_like(ls[li][1])), hit.start)
            text, used, he = '', [first], (ls.get(first) or (None,))[0]
        idx = used[0] if used else hit.start
        s = {'id': uid(), 'bookId': hit.title, 'book': hit.title, 'ref': ref_of(he or hit.ref, hit.title),
             'index': idx, 'text': text, 'range': None,
             'parts': [{'index': i, 'range': None} for i in used] if len(used) > 1 else None,
             'para': len(used) == 1, 'added': self._ts(), 'imgs': []}
        mode = self.args.titles
        if mode == 'all' or (mode == 'flagged' and hit.status != 'ok'):
            s['title'] = MARK[hit.status] + cite_text
        self.stats[hit.status] += 1
        return s

    def topic_dafs(self, items, i):
        """dafim of the סוגיא: the base-text citations of its opening lines"""
        out = []
        for j in range(i + 1, len(items)):
            it = items[j]
            if it['kind'] == 'H':
                if out:
                    break
                if it['depth'] <= items[i]['depth']:
                    break
                continue
            for c in self.parser.parse(it['text']):
                if c.dafs and c.kind in ('G', 'T') and (c.mas or self.mas) == self.mas and not c.rif_pages:
                    for d in expand_dafs(c):
                        if d not in out:
                            out.append(d)
            if out and j > i + 3:
                break
        return out

    def line(self, text, page, ctx, path):
        cites = self.parser.parse(text)
        out, prev, seen = [], [], set()
        multi = len(cites) > 1
        for c in cites:
            ct = c.text() or text
            hits = [None]
            if self.res is not None:
                try:
                    hits = self.res.resolve(c, ctx, prev) or [None]
                except Exception as e:                       # one bad citation never stops the run
                    log('  ! %s: %s: %s' % (ct, type(e).__name__, e))
                    if self.args.verbose:
                        import traceback
                        traceback.print_exc()
                    hits = [None]
                self.res.remember(c, ctx, hits)
                if self.args.verbose:
                    log('    %s  %s' % (ct, ' | '.join(repr(h) for h in hits)))
            prev.append(c)
            for h in hits:
                if h is None:
                    out.append(self.manual(ct, text if multi else ''))
                    self.review.append((page + 1, path, text, ct, repr(c), 'unmatched', '', '', '', ''))
                    continue
                key = (h.book_id, h.start, h.end)
                if key in seen:
                    continue
                seen.add(key)
                if h.status == 'loose' and self.args.loose == 'manual':
                    out.append(self.manual(ct, text if multi else ''))
                    continue
                s = self.from_hit(h, ct)
                out.append(s)
                self.review.append((page + 1, path, text, ct, repr(c), h.status, h.title, s['ref'], s['index'],
                                    'otzaria://open/book/%s?index=%s' % (h.book_id, s['index'])))
        if not cites:
            out.append(self.manual(text))
            self.review.append((page + 1, path, text, '', '', 'no citation', '', '', '', ''))
        return out

    def build(self, items, root_name):
        root = new_node(root_name)
        stack = [(0, root)]
        heads = [it['depth'] for it in items if it['kind'] == 'H']
        top = min(heads) if heads else 1
        maxd = max(heads) if heads else 1
        div_depth = maxd if (self.args.minor_as == 'divider' and maxd > top + 1) else None
        ctx = Ctx(self.mas)
        cur_div = None
        n_top = 0
        for i, it in enumerate(items):
            if it['kind'] == 'H':
                d = it['depth']
                name = re.sub(r'[.:]\s*$', '', it['text']).strip()
                if div_depth and d >= div_depth:
                    dv = {'id': uid(), 'name': name}
                    stack[-1][1]['dividers'].append(dv)
                    cur_div = dv['id']
                    continue
                cur_div = None
                while len(stack) > 1 and stack[-1][0] >= d:
                    stack.pop()
                node = new_node(name)
                stack[-1][1]['children'].append(node)
                stack.append((d, node))
                if d == top:
                    n_top += 1
                    if self.args.limit_topics and n_top > self.args.limit_topics:
                        stack[-2][1]['children'].pop()
                        break
                    ctx = Ctx(self.mas)
                    ctx.topic_dafs = self.topic_dafs(items, i)
                    log('[%d] %s%s' % (n_top, name, ('   (דפים: %s)' % ' '.join(
                        heb(n) + ('.' if a == 'a' else ':') for n, a in ctx.topic_dafs)) if ctx.topic_dafs else ''))
                continue
            node = stack[-1][1]
            path = ' › '.join(n['name'] for _, n in stack[1:])
            for s in self.line(it['text'], it['page'], ctx, path):
                if cur_div:
                    s['div'] = cur_div
                node['sources'].append(s)
        return root


def share_doc(node):
    def strip(n):
        return {'name': n['name'], 'chrono': False, 'noDivAt': 0, 'dividers': n['dividers'],
                'sources': n['sources'], 'images': {}, 'children': [strip(c) for c in n['children']]}
    return {'app': 'otzaria-topic-sources', 'v': 1, 'exported': datetime.datetime.utcnow().isoformat() + 'Z',
            'topic': strip(node)}


def backup_doc(nodes):
    data = {}

    def walk(n):
        data['src:' + n['id']] = n['sources']
        return {'id': n['id'], 'name': n['name'], 'chrono': False, 'noDivAt': 0, 'dividers': n['dividers'],
                'children': [walk(c) for c in n['children']]}
    data['tree'] = [walk(n) for n in nodes]
    return {'app': 'otzaria-topic-sources-backup', 'v': 1,
            'exported': datetime.datetime.utcnow().isoformat() + 'Z', 'data': data}


def safe_name(s):
    return re.sub(r'[\\/:*?"<>|]+', '', s).strip()[:80] or 'נושא'


def write_json(path, obj):
    with open(path, 'w', encoding='utf-8') as f:
        json.dump(obj, f, ensure_ascii=False, indent=1)


# ================================================================== main
def build_args(argv=None):
    ap = argparse.ArgumentParser(description='מראי מקומות PDF -> topic file for Otzaria\'s "מקורות ונושאים" plugin')
    ap.add_argument('pdf')
    ap.add_argument('--out', help='output file (default: <pdf>.topics.json; with --split: a folder)')
    ap.add_argument('--title', help='name of the top topic (default: the PDF file name)')
    ap.add_argument('--masechet', help='the masechet the sources refer to (default: guessed from the name)')
    ap.add_argument('--split', action='store_true', help='one file per top-level heading')
    ap.add_argument('--format', choices=('share', 'backup'), default='share',
                    help='share = ייבוא → ייבוא מאוצריא (default); backup = גיבוי ושחזור → הוספה')
    ap.add_argument('--minor-as', choices=('topic', 'divider'), default='topic',
                    help='the lowest heading level: sub-topics (default) or חוצצים')
    ap.add_argument('--titles', choices=('all', 'flagged', 'none'), default='all',
                    help='put the citation as written in each source\'s title: always (default), only for '
                         'approximate (≈) / ד"ה-not-found (⚠) ones, or never')
    ap.add_argument('--loose', choices=('keep', 'manual'), default='keep',
                    help='citations placed only by the sugya\'s daf (no daf/ד"ה of their own): keep with ≈, '
                         'or copy as manual sources')
    ap.add_argument('--max-words', type=int, default=400, help='words in-lined for one line (0 = all)')
    ap.add_argument('--max-range-words', type=int, default=1500, help='words in-lined for a range (0 = all)')
    ap.add_argument('--dry-run', action='store_true', help='layout + parse only, no Otzaria lookups')
    ap.add_argument('--backend', choices=('auto', 'fitz', 'pdfplumber'), default='auto')
    ap.add_argument('--header-band', type=float, default=0.12,
                    help='top fraction of the page where running headers live')
    ap.add_argument('--mcp-cmd', help='otzaria-mcp command (default: from claude_desktop_config.json)')
    ap.add_argument('--mcp-args', default='', help='its arguments, space separated')
    ap.add_argument('--cache', help='sqlite cache (default: next to this script)')
    ap.add_argument('--limit-topics', type=int, default=0, help='only the first N top-level topics (testing)')
    ap.add_argument('--verbose', '-v', action='store_true')
    return ap.parse_args(argv)


def main(argv=None):
    if hasattr(sys.stdout, 'reconfigure'):
        for s in (sys.stdout, sys.stderr):
            try:
                s.reconfigure(encoding='utf-8')
            except Exception:
                pass
    args = build_args(argv)
    pdf = os.path.abspath(args.pdf)
    stem = os.path.splitext(os.path.basename(pdf))[0]
    pages, backend = open_pages(pdf, args.backend)
    items = analyse(pages, args)
    nh = sum(1 for it in items if it['kind'] == 'H')
    log('%s: %d pages, %d headings, %d source lines (%s)' % (os.path.basename(pdf), len(pages), nh,
                                                              len(items) - nh, backend))
    mas = guess_masechet(args, items, stem)
    log('masechet: %s' % (mas or '?  (use --masechet)'))
    client = lib = res = cache = None
    if not args.dry_run:
        if args.mcp_cmd:
            spec = (args.mcp_cmd, args.mcp_args.split(), {})
        else:
            spec = discover_otzaria_mcp()
            if not spec:
                raise SystemExit('otzaria-mcp not found in claude_desktop_config.json; pass --mcp-cmd (or --dry-run)')
        client = McpClient(*spec)
        cache = Cache(args.cache or os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sugyot_cache.sqlite'))
        lib = Library(client, cache)
        res = Resolver(lib, args)
        log('otzaria-mcp %s' % (client.server_version or ''))
    b = Builder(args, res, mas)
    try:
        root = b.build(items, args.title or stem)
    finally:
        if cache:
            cache.close()
        if client:
            client.close()

    base = args.out or os.path.join(os.path.dirname(pdf), stem + '.topics' + ('' if args.split else '.json'))
    written = []
    if args.split:
        os.makedirs(base, exist_ok=True)
        if args.format == 'backup':
            p = os.path.join(base, stem + ' - גיבוי.json')
            write_json(p, backup_doc(root['children']))
            written.append(p)
        else:
            for k, n in enumerate(root['children'], 1):
                p = os.path.join(base, '%02d %s.json' % (k, safe_name(n['name'])))
                write_json(p, share_doc(n))
                written.append(p)
    else:
        write_json(base, share_doc(root) if args.format == 'share' else backup_doc([root]))
        written.append(base)
    rev = os.path.splitext(base if not args.split else os.path.join(base, stem))[0] + '.review.tsv'
    with open(rev, 'w', encoding='utf-8-sig') as f:
        f.write('page\ttopic\tline\tcitation\tparse\tstatus\tbook\tref\tindex\tlink\n')
        for r in b.review:
            f.write('\t'.join(str(x).replace('\t', ' ').replace('\n', ' ') for x in r) + '\n')
    st = b.stats
    log('sources: %d located (%d ≈ by the sugya\'s daf, %d ⚠ ד"ה not found), %d copied as manual'
        % (st['ok'] + st['loose'] + st['partial'], st['loose'], st['partial'], st['manual']))
    if lib:
        log('otzaria-mcp calls: %d (the rest from cache)' % lib.calls)
    for p in written:
        log('-> ' + p)
    log('-> ' + rev)


if __name__ == '__main__':
    main()