import os
import re
import json
import math
import time
import sqlite3
import shutil
import threading
import zipfile
import hashlib
from dataclasses import dataclass
from collections import deque
from pathlib import Path
from typing import Dict, Optional, Set, List, Tuple

import tkinter as tk
from tkinter import filedialog
import requests
import numpy as np
import faiss
from flask import Flask, request, render_template_string, redirect, jsonify, flash
from huggingface_hub import snapshot_download

# =========================
# CONFIG
# =========================
HF_REPO = "ArieLLL123/otzaria-embeddings"
DEFAULT_DB_PATH = r"C:\אוצריא\אוצריא\seforim.db"
DB_DOWNLOAD_URL = "https://github.com/Otzaria/otzaria-library/releases/download/library-db-1/seforim.zip"

EDITION_PATHS = {
    "v1": "editions/otzaria_embeddings_v1",
    "v2": "editions/otzaria_embeddings_v2",
    "v3": "editions/otzaria_embeddings_v3",
}

BASE_DIR = os.path.dirname(__file__)
CACHE_DIR = os.path.join(BASE_DIR, "hf_cache")
RUNTIME_DIR = os.path.join(BASE_DIR, "runtime")

DB_DIR = os.path.join(BASE_DIR, "db")
MODELS_ZIPS_DIR = os.path.join(BASE_DIR, "models_zips")   # כאן המשתמשים שמים ZIP
LOCAL_MODELS_DIR = os.path.join(BASE_DIR, "local_models") # כאן מחלצים פעם אחת

SETTINGS_PATH = os.path.join(RUNTIME_DIR, "settings.json")

# =========================
# 🔹 תוספת: ברירות מחדל חדשות
# =========================
DEFAULT_TOP_K = 20
DEFAULT_MIN_SCORE = 0.0

os.makedirs(CACHE_DIR, exist_ok=True)
os.makedirs(RUNTIME_DIR, exist_ok=True)
os.makedirs(DB_DIR, exist_ok=True)
os.makedirs(MODELS_ZIPS_DIR, exist_ok=True)
os.makedirs(LOCAL_MODELS_DIR, exist_ok=True)

try:
    from werkzeug.utils import secure_filename
except ImportError:
    def secure_filename(filename): return filename

# פרמטרים לחיתוך טקסט
DEFAULT_WINDOW_LINES = 6
DEFAULT_STRIDE = 3

# =========================
# TEXT TOOLS & HEBREW NLP
# =========================
NIQQUD_RE   = re.compile(r"[\u0591-\u05C7]")
HTML_TAG_RE = re.compile(r"<[^>]+>")
NON_WORD_RE = re.compile(r"[^0-9A-Za-z\u0590-\u05FF\"']+")

def clean_text(s: str) -> str:
    """ניקוי טקסט בסיסי - מסיר ניקוד וHTML"""
    if not s:
        return ""
    s = HTML_TAG_RE.sub(" ", s)
    s = NIQQUD_RE.sub("", s)
    s = s.replace('״', '"').replace('׳', "'")
    s = NON_WORD_RE.sub(" ", s)
    return " ".join(s.split())

def hebrew_stem(word: str) -> str:
    """סטמינג נאיבי לעברית (קידומות נפוצות)"""
    if len(word) < 4:
        return word
    prefixes = ['וכש', 'וש', 'וה', 'וב', 'ול', 'ומ', 'כש', 'שב', 'שה', 'מש', 'מה', 'ו', 'ה', 'ב', 'ל', 'מ', 'ש', 'כ']
    for p in prefixes:
        if word.startswith(p) and len(word) > len(p) + 2:
            return word[len(p):]
    return word

def get_tokens(text: str) -> Set[str]:
    words = clean_text(text).split()
    return {hebrew_stem(w) for w in words if w}

def fts_query_from_text(q_clean: str) -> str:
    toks = [t for t in clean_text(q_clean).split() if len(t) > 1]
    return " ".join(toks) if toks else ""

# =========================
# SETTINGS PERSISTENCE
# =========================
def load_settings() -> dict:
    if not os.path.exists(SETTINGS_PATH):
        return {}
    try:
        with open(SETTINGS_PATH, "r", encoding="utf-8") as f:
            return json.load(f) or {}
    except:
        return {}

def save_settings(data: dict) -> None:
    try:
        with open(SETTINGS_PATH, "w", encoding="utf-8") as f:
            json.dump(data, f, ensure_ascii=False, indent=2)
    except:
        pass

# =========================
# ZIP MODEL SUPPORT
# =========================
def sha256_file(path: str) -> str:
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for chunk in iter(lambda: f.read(1024 * 1024), b""):
            h.update(chunk)
    return h.hexdigest()

def ensure_zip_extracted(zip_path: str) -> str:
    """
    מחלץ ZIP לתיקיית local_models לפי hash,
    כדי שאם מחליפים ZIP – זה לא ידרוס אלא ייצור תיקייה חדשה.
    """
    if not os.path.exists(zip_path):
        raise FileNotFoundError(f"ZIP לא נמצא: {zip_path}")

    zhash = sha256_file(zip_path)[:16]
    target_dir = os.path.join(LOCAL_MODELS_DIR, zhash)
    marker = os.path.join(target_dir, ".extracted_ok")

    if os.path.exists(marker):
        return target_dir

    os.makedirs(target_dir, exist_ok=True)

    # חילוץ בטוח: מונע Zip Slip
    with zipfile.ZipFile(zip_path, "r") as z:
        for member in z.infolist():
            member_path = os.path.join(target_dir, member.filename)
            abs_target = os.path.abspath(target_dir)
            abs_member = os.path.abspath(member_path)
            if not abs_member.startswith(abs_target + os.sep) and abs_member != abs_target:
                raise RuntimeError("ZIP לא תקין (path traversal).")
        z.extractall(target_dir)

    with open(marker, "w", encoding="utf-8") as f:
        f.write(time.strftime("%Y-%m-%d %H:%M:%S"))

    return target_dir

def find_model_files(root_dir: str, edition: str) -> tuple[str, str]:
    """
    מחפש בתוך root_dir את:
    - vocab.json
    - embeddings_last.npy
    Prefer: נתיב שמכיל את שם ה-edition folder אם קיים
    """
    candidates_vocab = list(Path(root_dir).rglob("vocab.json"))
    candidates_emb   = list(Path(root_dir).rglob("embeddings_last.npy"))

    if not candidates_vocab or not candidates_emb:
        raise FileNotFoundError(
            "לא מצאתי בתוך ה-ZIP את vocab.json ו/או embeddings_last.npy.\n"
            "ודא שהקבצים קיימים בזיפ."
        )

    # עדיפות לתיקייה שמכילה את otzaria_embeddings_vX אם יש
    prefer_key = f"otzaria_embeddings_{edition}".lower()

    def pick(cands):
        for p in cands:
            if prefer_key in str(p).lower():
                return str(p)
        return str(cands[0])

    return pick(candidates_vocab), pick(candidates_emb)

# =========================
# DATABASE & STREAMING
# =========================
def get_book_titles(db_path: str) -> Dict[int, str]:
    titles = {}
    if not os.path.exists(db_path):
        return titles

    try:
        con = sqlite3.connect(db_path)
        cur = con.execute("SELECT id, title FROM book")
        for r in cur:
            titles[r[0]] = r[1]
        con.close()
    except Exception as e:
        print(f"שגיאה בטעינת שמות ספרים: {e}")

    return titles

def iter_rows_ordered(db_path: str, chunk_rows: int = 20000):
    if not os.path.exists(db_path):
        raise FileNotFoundError(f"קובץ מסד הנתונים לא נמצא: {db_path}")

    con = sqlite3.connect(db_path)
    con.row_factory = sqlite3.Row
    con.execute("PRAGMA journal_mode=OFF;")

    table_name = "line"
    try:
        con.execute("SELECT 1 FROM lines LIMIT 1")
        table_name = "lines"
    except:
        pass

    try:
        con.execute(f"SELECT 1 FROM {table_name} LIMIT 1")
    except:
        con.close()
        return

    q = f"""
    SELECT id, bookId, lineIndex, content
    FROM {table_name}
    WHERE content IS NOT NULL AND content != ''
    ORDER BY bookId, lineIndex
    """
    cur = con.execute(q)
    while True:
        rows = cur.fetchmany(chunk_rows)
        if not rows:
            break
        yield rows
    con.close()

def iter_chunks(db_path: str, max_chunks: int, window_lines: int, stride: int):
    rows_iter = iter_rows_ordered(db_path)
    buf = deque()
    cur_book = None
    produced = 0

    for batch in rows_iter:
        for r in batch:
            b_id = r["bookId"]
            if b_id != cur_book:
                cur_book = b_id
                buf.clear()

            txt = str(r["content"])
            if len(txt) < 3:
                continue

            buf.append({"id": r["id"], "idx": r["lineIndex"], "txt": txt})

            if len(buf) >= window_lines:
                window = list(buf)[-window_lines:]
                full_text = " ".join(w["txt"] for w in window)
                cln_text = clean_text(full_text)

                if len(cln_text) > 30:
                    yield {
                        "bookId": cur_book,
                        "startLine": window[0]["idx"],
                        "text": full_text,
                        "clean": cln_text
                    }
                    produced += 1
                    if produced >= max_chunks:
                        return

                for _ in range(stride):
                    if buf:
                        buf.popleft()

# =========================
# ENGINE CORE
# =========================
@dataclass
class LoadedModel:
    edition: str
    vocab: Dict[str, int]
    emb_norm: np.ndarray
    idf: np.ndarray

@dataclass
class BuiltIndex:
    faiss_index: faiss.Index
    meta_db_path: str
    count: int

class Engine:
    def __init__(self):
        self.model: Optional[LoadedModel] = None
        self.built: Optional[BuiltIndex] = None
        self.book_map: Dict[int, str] = {}
        self.status = {"state": "idle", "msg": "המערכת מוכנה", "progress": 0}
        self._lock = threading.RLock()
        self.last_cfg = load_settings()  # persist

    def _update(self, state, msg, progress):
        with self._lock:
            self.status = {"state": state, "msg": msg, "progress": int(progress)}
        print(f"[{state}] {msg} ({progress}%)")

    def _hf_snapshot_offline_first(self, allow_patterns: List[str]) -> str:
        try:
            return snapshot_download(
                repo_id=HF_REPO,
                repo_type="model",
                cache_dir=CACHE_DIR,
                allow_patterns=allow_patterns,
                local_files_only=True,
            )
        except Exception:
            return snapshot_download(
                repo_id=HF_REPO,
                repo_type="model",
                cache_dir=CACHE_DIR,
                allow_patterns=allow_patterns,
                local_files_only=False,
            )

    def load_resources(
        self,
        db_path: str,
        edition: str = "v3",
        model_source: str = "hf",  # hf / zip
        zip_path: str = ""
    ):
        if db_path and os.path.exists(db_path):
            self.book_map = get_book_titles(db_path)

        try:
            self._update("downloading", f"טוען מודל {edition} ({model_source})...", 5)

            if model_source == "zip":
                # ברירת מחדל: מחפש zip בשם סטנדרטי בתיקיית models_zips
                if not zip_path:
                    zip_path = os.path.join(MODELS_ZIPS_DIR, f"otzaria_embeddings_{edition}.zip")

                extracted_root = ensure_zip_extracted(zip_path)
                vocab_path, emb_path = find_model_files(extracted_root, edition)

            else:
                path = EDITION_PATHS.get(edition, EDITION_PATHS["v3"])
                local_dir = self._hf_snapshot_offline_first([
                    f"{path}/vocab.json",
                    f"{path}/embeddings_last.npy",
                ])
                base = os.path.join(local_dir, path)
                vocab_path = os.path.join(base, "vocab.json")
                emb_path = os.path.join(base, "embeddings_last.npy")

            with open(vocab_path, "r", encoding="utf-8") as f:
                meta = json.load(f)

            emb = np.load(emb_path).astype(np.float32)
            norms = np.linalg.norm(emb, axis=1, keepdims=True)
            norms[norms == 0] = 1
            emb_norm = emb / norms

            vocab = meta["vocab"]
            freqs = np.array(meta.get("freqs", []), dtype=np.float64)
            if len(freqs) == len(vocab):
                idf = np.log((np.sum(freqs) + 1) / (freqs + 1)) + 1
            else:
                idf = np.ones(len(vocab), dtype=np.float32)

            self.model = LoadedModel(edition, vocab, emb_norm, idf.astype(np.float32))
            self._update("idle", "המודל נטען בהצלחה", 100)

        except Exception as e:
            self._update("error", f"שגיאה בטעינת מודל: {e}", 0)
            raise

    def _stamp(self, edition: str, max_chunks: int) -> str:
        # חשוב: stamp תלוי edition + chunks + window/stride כדי לטעון בדיוק מה שנבנה
        return f"{edition}_N{max_chunks}_W{DEFAULT_WINDOW_LINES}_S{DEFAULT_STRIDE}"

    def build_index(self, db_path: str, max_chunks: int):
        if not self.model or self.status["state"] == "indexing":
            return

        stamp = self._stamp(self.model.edition, max_chunks)
        idx_path = os.path.join(RUNTIME_DIR, f"{stamp}.index")
        meta_db_path = os.path.join(RUNTIME_DIR, f"{stamp}.sqlite")

        # אם קיים - טוען
        if os.path.exists(idx_path) and os.path.exists(meta_db_path):
            self._update("loading", "טוען אינדקס קיים...", 50)
            idx = faiss.read_index(idx_path)
            self.built = BuiltIndex(idx, meta_db_path, idx.ntotal)
            if not self.book_map:
                self.book_map = get_book_titles(db_path)
            self._update("ready", f"מוכן לחיפוש ({idx.ntotal:,} רשומות)", 100)
            return

        self._update("indexing", "מתחיל בבניית אינדקס (זה יקח זמן)...", 0)

        if os.path.exists(meta_db_path):
            os.remove(meta_db_path)

        con = sqlite3.connect(meta_db_path)
        con.execute("PRAGMA synchronous = OFF")
        con.execute("PRAGMA journal_mode = MEMORY")

        con.execute("""
            CREATE TABLE chunks (
                rowid INTEGER PRIMARY KEY,
                bookId INTEGER,
                startLine INTEGER,
                text TEXT
            )
        """)
        con.execute("CREATE INDEX idx_book ON chunks(bookId)")

        # FTS5 (BM25)
        # נשמור ב-FTS את הטקסט הנקי
        con.execute("""
            CREATE VIRTUAL TABLE chunks_fts
            USING fts5(text, content='');
        """)

        d = self.model.emb_norm.shape[1]
        index = faiss.IndexIDMap(faiss.IndexFlatIP(d))

        vectors = []
        ids = []
        db_buffer = []
        fts_buffer = []

        batch_size = 5000
        total_processed = 0
        start_time = time.time()

        for chunk in iter_chunks(db_path, max_chunks, DEFAULT_WINDOW_LINES, DEFAULT_STRIDE):
            vec = self._text_to_vec(chunk["clean"])
            if vec is None:
                continue

            current_id = total_processed
            vectors.append(vec)
            ids.append(current_id)

            db_buffer.append((current_id, chunk["bookId"], chunk["startLine"], chunk["text"]))
            fts_buffer.append((current_id, chunk["clean"]))

            total_processed += 1

            if len(vectors) >= batch_size:
                index.add_with_ids(np.vstack(vectors), np.array(ids).astype("int64"))
                con.executemany("INSERT INTO chunks VALUES (?,?,?,?)", db_buffer)
                con.executemany("INSERT INTO chunks_fts(rowid, text) VALUES (?,?)", fts_buffer)
                con.commit()
                vectors, ids, db_buffer, fts_buffer = [], [], [], []

                elapsed = time.time() - start_time
                rate = total_processed / (elapsed + 0.1)
                pct = min(95, int((total_processed / max_chunks) * 100))
                self._update("indexing", f"עובדו {total_processed:,} רשומות ({int(rate)} לשנייה)", pct)

        if vectors:
            index.add_with_ids(np.vstack(vectors), np.array(ids).astype("int64"))
            con.executemany("INSERT INTO chunks VALUES (?,?,?,?)", db_buffer)
            con.executemany("INSERT INTO chunks_fts(rowid, text) VALUES (?,?)", fts_buffer)
            con.commit()

        con.close()
        faiss.write_index(index, idx_path)

        self.built = BuiltIndex(index, meta_db_path, total_processed)
        self._update("ready", "הבנייה הושלמה בהצלחה!", 100)

    def _text_to_vec(self, text: str):
        if not self.model:
            return None
        words = text.split()
        if not words:
            return None

        indices = [self.model.vocab[w] for w in words if w in self.model.vocab]
        if not indices:
            return None

        idfs = self.model.idf[indices]
        vecs = self.model.emb_norm[indices]

        weighted = vecs * idfs[:, None]
        avg_vec = np.sum(weighted, axis=0)

        norm = np.linalg.norm(avg_vec)
        if norm < 1e-9:
            return None
        return avg_vec / norm

    def _fts_candidates(self, q_clean: str, limit: int) -> List[Tuple[int, float]]:
        """
        מחזיר רשימת (rowid, bm25_raw)
        bm25 ב-FTS5: קטן יותר = יותר רלוונטי.
        """
        if not self.built:
            return []
        fts_q = fts_query_from_text(q_clean)
        if not fts_q:
            return []
        con = sqlite3.connect(self.built.meta_db_path)
        con.row_factory = sqlite3.Row
        try:
            rows = con.execute(
                "SELECT rowid, bm25(chunks_fts) AS bm FROM chunks_fts WHERE chunks_fts MATCH ? LIMIT ?",
                (fts_q, int(limit)),
            ).fetchall()
            return [(int(r["rowid"]), float(r["bm"])) for r in rows]
        except:
            return []
        finally:
            con.close()

    def search(self, query: str, book_filter: Optional[int] = None, top_k: int = 20):
        if not self.model or not self.built:
            return []

        q_clean = clean_text(query)
        q_vec = self._text_to_vec(q_clean)
        if q_vec is None:
            return []

        # 1) מועמדים וקטוריים רחבים
        vec_candidates_k = max(top_k * 20, 200)
        scores, ids = self.built.faiss_index.search(np.array([q_vec]), vec_candidates_k)
        vec_found_ids = [int(i) for i in ids[0] if i >= 0]

        # 2) מועמדי FTS (BM25)
        fts_candidates_k = max(top_k * 20, 200)
        fts_rows = self._fts_candidates(q_clean, fts_candidates_k)
        fts_found_ids = [rid for rid, _ in fts_rows]

        # 3) איחוד
        union_ids = []
        seen = set()
        for rid in vec_found_ids + fts_found_ids:
            if rid not in seen:
                union_ids.append(rid)
                seen.add(rid)

        if not union_ids:
            return []

        # 4) שליפת מטא
        con = sqlite3.connect(self.built.meta_db_path)
        con.row_factory = sqlite3.Row

        placeholders = ",".join(["?"] * len(union_ids))
        sql = f"SELECT rowid, bookId, startLine, text FROM chunks WHERE rowid IN ({placeholders})"
        params: List = list(union_ids)

        if book_filter:
            sql += " AND bookId = ?"
            params.append(int(book_filter))

        rows = con.execute(sql, params).fetchall()
        con.close()

        if not rows:
            return []

        # מפות ציונים
        vec_scores = {int(fid): float(scr) for fid, scr in zip(ids[0], scores[0]) if int(fid) >= 0}
        fts_bm = {rid: bm for rid, bm in fts_rows}

        # נרמול BM25 -> [0..1] (היפוך גס; bm25 קטן=טוב)
        def bm_to_rel(bm: Optional[float]) -> float:
            if bm is None:
                return 0.0
            bm_pos = max(0.0, bm)
            return 1.0 / (1.0 + bm_pos)

        q_tokens = get_tokens(q_clean)

        results = []
        for r in rows:
            rid = int(r["rowid"])
            chunk_txt = r["text"]
            chunk_clean = clean_text(chunk_txt)
            chunk_tokens = get_tokens(chunk_clean)
            chunk_words = chunk_clean.split()

            base_vec = vec_scores.get(rid, 0.0)
            bm_rel = bm_to_rel(fts_bm.get(rid))

            q_clean = clean_text(query)
            intersection = len(q_tokens & chunk_tokens)
            overlap = (intersection / len(q_tokens)) if q_tokens else 0.0

            phrase = 1.0 if (q_clean and q_clean in chunk_clean) else 0.0

            proximity = 0.0
            if intersection > 1 and q_tokens:
                found_indices = []
                for qw in q_tokens:
                    for i, cw in enumerate(chunk_words):
                        if hebrew_stem(cw) == qw:
                            found_indices.append(i)
                            break
                if found_indices:
                    span = max(found_indices) - min(found_indices)
                    density = len(found_indices) / (span + 1)
                    proximity = min(density, 1.0)

            # שקלול Hybrid (אפשר לכוונן)
            final_score = (
                (base_vec * 0.35) +
                (bm_rel   * 0.25) +
                (overlap  * 0.25) +
                (phrase   * 0.10) +
                (proximity* 0.05)
            )

            book_title = self.book_map.get(int(r["bookId"]), f"ספר {int(r['bookId'])}")
            results.append({
                "score": float(final_score),
                "text": chunk_txt,
                "source": f"{book_title}, שורה {int(r['startLine'])}",
                "book_id": int(r["bookId"]),
                "book_title": book_title
            })

        results.sort(key=lambda x: x["score"], reverse=True)
        return results[:top_k]

ENGINE = Engine()

# =========================
# FLASK WEB APP
# =========================
app = Flask(__name__)
app.secret_key = "otzaria_ai_secret_v3"

HTML_TEMPLATE = """
<!DOCTYPE html>
<html lang="he" dir="rtl">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>אוצריא AI</title>
    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
    <link href="https://fonts.googleapis.com/css2?family=Heebo:wght@300;400;500;700&family=Frank+Ruhl+Libre:wght@400;700&display=swap" rel="stylesheet">
    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.10.5/font/bootstrap-icons.css">
    <style>
        :root { --primary-color: #0d6efd; --bg-color: #f8f9fa; --card-bg: #ffffff; }
        body { background-color: var(--bg-color); font-family: 'Heebo', sans-serif; color: #333; }
        .serif-text { font-family: 'Frank Ruhl Libre', serif; }
        .navbar { background: var(--card-bg); border-bottom: 1px solid #eee; }
        .search-container { max-width: 800px; margin: 0 auto; }
        .search-box { transition: all 0.3s; border: 1px solid #dfe1e5; border-radius: 24px; overflow: hidden; background: var(--card-bg); }
        .search-box:hover, .search-box:focus-within { box-shadow: 0 1px 6px rgba(32,33,36,.28); border-color: rgba(223,225,229,0); }
        .search-input { border: none; box-shadow: none !important; font-size: 1.1rem; padding: 12px 20px; }
        .result-card { background: var(--card-bg); border-radius: 12px; padding: 20px; margin-bottom: 16px; border: 1px solid #eee; transition: transform 0.2s; }
        .result-card:hover { transform: translateY(-2px); box-shadow: 0 4px 12px rgba(0,0,0,0.05); }
        .source-badge { background-color: #e7f1ff; color: #0d6efd; padding: 4px 10px; border-radius: 6px; font-size: 0.85rem; font-weight: 600; }
        mark { background-color: #fff3cd; padding: 0 2px; border-radius: 2px; color: #333; }
        .settings-btn { color: #6c757d; transition: color 0.2s; }
        .settings-btn:hover { color: #333; }
        .upload-area { border: 2px dashed #dee2e6; border-radius: 12px; padding: 2rem; text-align: center; transition: all 0.2s; background: #fff; }
        .upload-area:hover { border-color: var(--primary-color); background-color: #f8f9ff; }
        /* Scrollbar for modal */
        ::-webkit-scrollbar { width: 8px; }
        ::-webkit-scrollbar-track { background: #f1f1f1; }
        ::-webkit-scrollbar-thumb { background: #ccc; border-radius: 4px; }
    </style>
</head>
<body>

<!-- Navbar -->
<nav class="navbar navbar-expand-lg sticky-top">
    <div class="container">
        <a class="navbar-brand fw-bold d-flex align-items-center gap-2" href="/">
            <i class="bi bi-book-half text-primary"></i>
            <span>אוצריא <small class="text-muted fw-normal">AI</small></span>
        </a>
        
        <div class="d-flex align-items-center gap-3">
            <!-- Status Indicator -->
            <div class="d-flex align-items-center small bg-light px-3 py-1 rounded-pill border" id="status-pill" title="סטטוס מערכת">
                <div id="status-dot" class="rounded-circle bg-secondary me-2" style="width: 8px; height: 8px;"></div>
                <span id="status-text" class="text-muted">טוען...</span>
                <span id="idx-count" class="ms-2 fw-bold text-dark">{{ idx_count }}</span>
            </div>

            {% if model_loaded %}
            <!-- Settings Toggle -->
            <button class="btn btn-link settings-btn p-0 fs-5" data-bs-toggle="modal" data-bs-target="#settingsModal" title="הגדרות">
                <i class="bi bi-gear"></i>
            </button>
            {% endif %}
        </div>
    </div>
</nav>

<!-- Main Content -->
<div class="container py-5 search-container">
    
    {% if not model_loaded or not db_exists %}
    <!-- Setup Interface -->
    <div class="row justify-content-center">
        <div class="col-md-8">
            <div class="text-center mb-5">
                <h2 class="fw-bold mb-3">הגדרת מערכת</h2>
                <p class="text-muted">המערכת זקוקה להגדרות ראשוניות כדי לפעול.</p>
            </div>

            {% if not db_exists %}
            <!-- DB Card -->
            <div class="card shadow-sm mb-4 border-0">
                <div class="card-body p-4">
                    <h5 class="card-title mb-4 fw-bold"><i class="bi bi-database me-2 text-primary"></i>טעינת מאגר ספרים (DB)</h5>
                    <p class="text-muted small mb-3">לא נמצא קובץ מסד נתונים. ניתן להוריד, להעלות, או להזין נתיב ידני למטה.</p>
                    
                    <div class="row g-3">
                        <div class="col-md-4">
                            <form action="/download_db" method="post">
                                <button type="submit" class="btn btn-outline-primary w-100 py-3 h-100">
                                    <i class="bi bi-cloud-download display-6 d-block mb-2"></i>
                                    הורד מהשרת
                                </button>
                            </form>
                        </div>
                        <div class="col-md-4">
                            <a href="/select_local_db" class="btn btn-outline-success w-100 py-3 h-100">
                                <i class="bi bi-folder2-open display-6 d-block mb-2"></i>
                                בחר קובץ מקומי
                            </a>
                        </div>
                        <div class="col-md-4">
                            <form action="/upload_db" method="post" enctype="multipart/form-data" class="h-100">
                                <div class="upload-area h-100 d-flex flex-column justify-content-center" style="padding: 1rem; cursor: pointer;" onclick="document.getElementById('db_file').click()">
                                    <i class="bi bi-database-add fs-1 text-muted mb-2"></i>
                                    <span class="small">גרירת קובץ DB/ZIP או לחץ לבחירה</span>
                                    <input type="file" id="db_file" name="file" class="d-none" accept=".zip,.db,.sqlite" required onchange="this.form.submit()">
                                </div>
                            </form>
                        </div>
                    </div>
                </div>
            </div>
            {% endif %}

            {% if not model_loaded %}
            <!-- Upload Card -->
            <div class="card shadow-sm mb-4 border-0">
                <div class="card-body p-4">
                    <h5 class="card-title mb-4 fw-bold"><i class="bi bi-cpu me-2 text-primary"></i>טעינת מודל (ZIP)</h5>
                    <div class="row g-3">
                        <div class="col-md-6">
                            <a href="/select_local_zip" class="btn btn-outline-primary w-100 py-4">
                                <i class="bi bi-file-earmark-zip fs-1 d-block mb-2"></i>
                                בחר קובץ ZIP מהמחשב
                            </a>
                        </div>
                        <div class="col-md-6">
                            <form action="/upload_model" method="post" enctype="multipart/form-data">
                                <div class="upload-area" onclick="document.getElementById('model_file').click()" style="cursor: pointer; padding: 1.2rem;">
                                    <i class="bi bi-upload fs-2 text-muted mb-2"></i>
                                    <p class="small mb-0">העלאת קובץ (העתקה)</p>
                                    <input type="file" id="model_file" name="file" class="d-none" accept=".zip" required onchange="this.form.submit()">
                                </div>
                            </form>
                        </div>
                    </div>
                </div>
            </div>
            {% endif %}

            <!-- Manual Config -->
            <div class="card shadow-sm border-0">
                <div class="card-body p-4">
                    <h5 class="card-title mb-3 fw-bold text-secondary"><i class="bi bi-sliders me-2"></i>אפשרויות נוספות</h5>
                    <form action="/setup" method="post">
                        <div class="row g-3">
                            <div class="col-12">
                                <label class="form-label small fw-bold text-muted">נתיב בסיס נתונים (DB)</label>
                                <input type="text" name="db_path" class="form-control bg-light" value="{{ db_path }}">
                            </div>
                            <div class="col-md-6">
                                <label class="form-label small fw-bold text-muted">מקור מודל</label>
                                <select name="model_source" class="form-select bg-light">
                                    <option value="zip" {% if model_source == "zip" %}selected{% endif %}>ZIP מקומי</option>
                                    <option value="hf" {% if model_source == "hf" %}selected{% endif %}>HuggingFace (הורדה)</option>
                                </select>
                            </div>
                            <div class="col-md-6">
                                <label class="form-label small fw-bold text-muted">נתיב ZIP קיים</label>
                                <input type="text" name="zip_path" class="form-control bg-light" value="{{ zip_path or '' }}" placeholder="models_zips/...">
                            </div>
                            <div class="col-12 mt-3">
                                <button type="submit" class="btn btn-outline-secondary w-100">טען הגדרות</button>
                            </div>
                        </div>
                    </form>
                </div>
            </div>
        </div>
    </div>

    {% else %}
    <!-- Search Interface -->
    <!-- Hero / Search Header -->
    <div class="text-center mb-4 {% if query %}d-none{% endif %}">
        <h1 class="display-5 fw-bold mb-3">חיפוש במקורות</h1>
        <p class="text-muted mb-4">הקלד שאלה או נושא לחיפוש בארון הספרים היהודי</p>
    </div>

    <!-- Search Form -->
    <form action="/" method="get" class="mb-5">
        <div class="search-box d-flex align-items-center shadow-sm">
            <button type="submit" class="btn border-0 text-muted ps-3"><i class="bi bi-search"></i></button>
            <input type="text" name="q" class="form-control search-input" placeholder="חיפוש חופשי..." value="{{ query or '' }}" autofocus>
            {% if query %}
            <a href="/" class="btn border-0 text-muted pe-3" title="נקה"><i class="bi bi-x-lg"></i></a>
            {% endif %}
        </div>
        
        <!-- Filters -->
        <div class="d-flex justify-content-center mt-2">
            <select name="book_id" class="form-select form-select-sm w-auto border-0 bg-transparent text-muted" style="cursor: pointer;">
                <option value="">📚 כל הספרים</option>
                {% for bid, title in books.items() %}
                    <option value="{{ bid }}" {% if selected_book|int == bid %}selected{% endif %}>{{ title }}</option>
                {% endfor %}
            </select>
        </div>
    </form>

    <!-- Results -->
    {% if query %}
        <div class="d-flex justify-content-between align-items-center mb-3 px-1">
            <h5 class="mb-0 fw-normal">תוצאות עבור: <strong>{{ query }}</strong></h5>
            <span class="badge bg-light text-secondary border">{{ results|length }} תוצאות</span>
        </div>

        {% if not results %}
             <div class="text-center py-5">
                <div class="mb-3"><i class="bi bi-search display-1 text-light"></i></div>
                <h4 class="text-muted">לא נמצאו תוצאות</h4>
                <p class="text-muted small">נסה לשנות את מילות החיפוש או להסיר סינונים</p>
             </div>
        {% endif %}

        {% for r in results %}
        <div class="result-card">
            <div class="d-flex justify-content-between align-items-start mb-2">
                <span class="source-badge"><i class="bi bi-journal-text me-1"></i> {{ r.source }}</span>
                <small class="text-muted" title="ציון רלוונטיות">{{ "%.0f"|format(r.score * 100) }}%</small>
            </div>
            <div class="serif-text fs-5 lh-base text-dark">
                {{ r.text | highlight(query) | safe }}
            </div>
        </div>
        {% endfor %}
    {% endif %}
    {% endif %}
</div>

<!-- Settings Modal -->
<div class="modal fade" id="settingsModal" tabindex="-1" aria-hidden="true">
    <div class="modal-dialog modal-dialog-centered">
        <div class="modal-content border-0 shadow">
            <form action="/setup" method="post">
                <div class="modal-header border-bottom-0 pb-0">
                    <h5 class="modal-title fw-bold">הגדרות מערכת</h5>
                    <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
                </div>
                <div class="modal-body pt-3">
                    <div class="mb-3">
                        <label class="form-label small fw-bold text-muted">נתיב בסיס נתונים (DB)</label>
                        <input type="text" name="db_path" class="form-control form-control-sm bg-light" value="{{ db_path }}">
                    </div>

                    <div class="row g-2 mb-3">
                        <div class="col-6">
                            <label class="form-label small fw-bold text-muted">גרסת מודל</label>
                            <select name="edition" class="form-select form-select-sm bg-light">
                                {% for e in ["v1","v2","v3"] %}
                                    <option value="{{ e }}" {% if e == edition %}selected{% endif %}>{{ e }}</option>
                                {% endfor %}
                            </select>
                        </div>
                        <div class="col-6">
                            <label class="form-label small fw-bold text-muted">מקור מודל</label>
                            <select name="model_source" class="form-select form-select-sm bg-light">
                                <option value="zip" {% if model_source == "zip" %}selected{% endif %}>ZIP מקומי</option>
                                <option value="hf" {% if model_source == "hf" %}selected{% endif %}>HuggingFace</option>
                            </select>
                        </div>
                    </div>

                    <div class="mb-3">
                        <label class="form-label small fw-bold text-muted">נתיב ZIP (אופציונלי)</label>
                        <input type="text" name="zip_path" class="form-control form-control-sm bg-light" value="{{ zip_path or '' }}" placeholder="למשל: C:\\Downloads\\otzaria_embeddings_v3.zip">
                    </div>

                    <div class="mb-3">
                        <label class="form-label small fw-bold text-muted">מספר רשומות לאינדקס</label>
                        <input type="number" name="max_chunks" class="form-control form-control-sm bg-light" value="{{ max_chunks }}">
                        <div class="form-text small">שינוי דורש בנייה מחדש של האינדקס.</div>
                    </div>
                    <div class="mb-3">
                        <label class="form-label small fw-bold text-muted">מספר תוצאות להצגה</label>
                        <input type="number" name="top_k" min="1" max="200"
                            class="form-control form-control-sm bg-light"
                            value="{{ top_k }}">
                    </div>

                    <div class="mb-3">
                        <label class="form-label small fw-bold text-muted">סף דמיון מינימלי (%)</label>
                        <input type="number" name="min_score"
                            min="0" max="100" step="1"
                            class="form-control form-control-sm bg-light"
                            value="{{ min_score }}">
                        <div class="form-text small">
                            תוצאות מתחת לסף זה לא יוצגו.
                        </div>
                    </div>
                </div>
                <div class="modal-footer border-top-0 pt-0">
                    <button type="button" class="btn btn-light btn-sm" data-bs-dismiss="modal">ביטול</button>
                    <button type="submit" class="btn btn-primary btn-sm px-4">שמור וטען</button>
                </div>
            </form>
        </div>
    </div>
</div>

<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
<script>
    function updateStatus() {
        fetch('/status')
            .then(r => r.json())
            .then(data => {
                const textEl = document.getElementById('status-text');
                const dotEl = document.getElementById('status-dot');
                const countEl = document.getElementById('idx-count');
                
                textEl.innerText = data.msg;
                
                // Update dot color based on state
                if (data.state === 'ready') {
                    dotEl.className = 'rounded-circle bg-success me-2';
                } else if (data.state === 'indexing' || data.state === 'downloading') {
                    dotEl.className = 'rounded-circle bg-warning me-2 spinner-grow spinner-grow-sm';
                } else if (data.state === 'error') {
                    dotEl.className = 'rounded-circle bg-danger me-2';
                } else {
                    dotEl.className = 'rounded-circle bg-secondary me-2';
                }

                if(data.count && countEl) countEl.innerText = data.count.toLocaleString();
                
                let interval = (data.state === 'indexing' || data.state === 'downloading') ? 1000 : 5000;
                setTimeout(updateStatus, interval);
            })
            .catch(e => setTimeout(updateStatus, 5000));
    }
    document.addEventListener('DOMContentLoaded', updateStatus);
</script>

</body>
</html>
"""

# =========================
# HELPER FILTERS
# =========================
def highlight_text(text, query):
    if not query:
        return text
    q_words = [hebrew_stem(w) for w in clean_text(query).split() if len(w) > 1]
    if not q_words:
        return text

    patterns = []
    for w in q_words:
        pat = r'(?:^|[\s\"\'\-])([ו|מ|ש|ה|ל|ב|כ]?' + re.escape(w) + r')(?=[\s\"\'\.\,\-]|$)'
        patterns.append(pat)

    combined_pattern = "|".join(patterns)

    def replacer(match):
        full_match = match.group(0)
        word_match = re.search(r'[א-ת]+', full_match)
        if word_match:
            wm = word_match.group(0)
            return full_match.replace(wm, f'<mark>{wm}</mark>')
        return full_match

    try:
        return re.sub(combined_pattern, replacer, text)
    except:
        return text

@app.template_filter('highlight')
def highlight_filter(text, query):
    return highlight_text(text, query)

# =========================
# ROUTES
# =========================
@app.route("/")
def index():
    q = request.args.get("q", "").strip()
    book_id_str = request.args.get("book_id", "")
    book_id = int(book_id_str) if book_id_str.isdigit() else None

    cfg = ENGINE.last_cfg or {}

    top_k = int(cfg.get("top_k", DEFAULT_TOP_K))
    min_score = float(cfg.get("min_score", DEFAULT_MIN_SCORE)) / 100.0

    results = []
    all_books = ENGINE.book_map.copy()
    sorted_books = dict(sorted(all_books.items(), key=lambda item: item[1])[:800])

    if q:
        if not ENGINE.built and ENGINE.status["state"] not in ("indexing", "downloading"):
            def task():
                try:
                    ENGINE.load_resources(
                        cfg.get("db_path", DEFAULT_DB_PATH),
                        cfg.get("edition", "v3"),
                        model_source=cfg.get("model_source", "zip"),
                        zip_path=cfg.get("zip_path", "")
                    )
                    ENGINE.build_index(
                        cfg.get("db_path", DEFAULT_DB_PATH),
                        int(cfg.get("max_chunks", 100000))
                    )
                except Exception as e:
                    ENGINE._update("error", f"שגיאה: {e}", 0)

            threading.Thread(target=task, daemon=True).start()
        else:
            raw = ENGINE.search(q, book_filter=book_id, top_k=top_k * 3)
            filtered = [r for r in raw if r["score"] >= min_score]
            results = filtered[:top_k]

    idx_c = ENGINE.built.count if ENGINE.built else 0
    current_db = cfg.get("db_path", DEFAULT_DB_PATH)
    db_exists = os.path.exists(current_db)

    model_loaded = (ENGINE.model is not None)
    is_ready = (ENGINE.built is not None)
    is_busy = ENGINE.status["state"] in ("indexing", "downloading", "loading")
    show_setup = not is_ready and not is_busy and (not model_loaded or not db_exists)

    return render_template_string(
        HTML_TEMPLATE,
        model_loaded=model_loaded,
        db_exists=db_exists,
        show_setup=show_setup,
        query=q,
        results=results,
        db_path=current_db,
        edition=cfg.get("edition", "v3"),
        max_chunks=int(cfg.get("max_chunks", 100000)),
        model_source=cfg.get("model_source", "zip"),
        zip_path=cfg.get("zip_path", ""),
        idx_count=idx_c,
        books=sorted_books,
        selected_book=book_id,
        top_k=top_k,
        min_score=int(cfg.get("min_score", 0))
    )
@app.route("/setup", methods=["POST"])
def setup():
    db = request.form.get("db_path", DEFAULT_DB_PATH).strip()
    edition = request.form.get("edition", "v3").strip()
    mc = int(request.form.get("max_chunks", 100000))
    model_source = request.form.get("model_source", "zip").strip()
    zip_path = request.form.get("zip_path", "").strip()

    top_k = int(request.form.get("top_k", DEFAULT_TOP_K))
    min_score = float(request.form.get("min_score", 0))

    ENGINE.last_cfg = {
        "db_path": db,
        "edition": edition,
        "max_chunks": mc,
        "model_source": model_source,
        "zip_path": zip_path,
        "top_k": top_k,
        "min_score": min_score,
    }

    save_settings(ENGINE.last_cfg)

    def task():
        try:
            ENGINE.load_resources(db, edition,
                                  model_source=model_source,
                                  zip_path=zip_path)
            ENGINE.build_index(db, mc)
        except Exception as e:
            ENGINE._update("error", f"שגיאה: {e}", 0)

    threading.Thread(target=task, daemon=True).start()
    return redirect("/")
@app.route("/upload_model", methods=["POST"])
def upload_model():
    file = request.files.get('file')
    if not file or not file.filename:
        return redirect("/")
    
    filename = secure_filename(file.filename)
    target_path = os.path.join(MODELS_ZIPS_DIR, filename)
    
    try:
        with open(target_path, "wb") as buffer:
            shutil.copyfileobj(file.stream, buffer)
        
        # Update settings to use this zip
        cfg = load_settings()
        cfg["model_source"] = "zip"
        cfg["zip_path"] = target_path
        save_settings(cfg)
        
        # Trigger load
        def task():
            try:
                ENGINE.load_resources(cfg.get("db_path", DEFAULT_DB_PATH), 
                                    cfg.get("edition", "v3"), 
                                    model_source="zip", 
                                    zip_path=target_path)
                ENGINE.build_index(cfg.get("db_path", DEFAULT_DB_PATH), 
                                 int(cfg.get("max_chunks", 100000)))
            except Exception as e:
                ENGINE._update("error", f"שגיאה: {e}", 0)
                print(f"Error loading uploaded model: {e}")

        threading.Thread(target=task, daemon=True).start()
        
    except Exception as e:
        print(f"Upload failed: {e}")
        
    return redirect("/")

@app.route("/download_db", methods=["POST"])
def download_db():
    def task():
        try:
            ENGINE._update("downloading", "מוריד קובץ מסד נתונים...", 0)
            zip_path = os.path.join(DB_DIR, "seforim.zip")

            with requests.get(DB_DOWNLOAD_URL, stream=True) as r:
                r.raise_for_status()
                total_length = int(r.headers.get('content-length', 0))
                dl = 0

                with open(zip_path, 'wb') as f:
                    for chunk in r.iter_content(chunk_size=8192):
                        if not chunk:
                            continue
                        dl += len(chunk)
                        f.write(chunk)
                        if total_length:
                            pct = int((dl / total_length) * 100)
                            ENGINE._update("downloading", "מוריד קובץ מסד נתונים...", pct)

            # חילוץ
            ENGINE._update("indexing", "מחלץ מסד נתונים...", 0)

            with zipfile.ZipFile(zip_path, "r") as z:
                z.extractall(DB_DIR)

            # חיפוש קובץ DB בתוך התיקייה
            db_file = None
            for root, dirs, files in os.walk(DB_DIR):
                for file in files:
                    if file.endswith(".db") or file.endswith(".sqlite"):
                        db_file = os.path.join(root, file)
                        break

            if not db_file:
                raise RuntimeError("לא נמצא קובץ DB לאחר החילוץ.")

            # שמירת נתיב בהגדרות
            cfg = load_settings()
            cfg["db_path"] = db_file
            save_settings(cfg)
            ENGINE.last_cfg = cfg

            ENGINE._update("ready", "מסד הנתונים נטען בהצלחה", 100)

        except Exception as e:
            ENGINE._update("error", f"שגיאה בהורדת DB: {e}", 0)

    threading.Thread(target=task, daemon=True).start()
    return redirect("/")
@app.route("/upload_db", methods=["POST"])
def upload_db():
    file = request.files.get('file')
    if not file or not file.filename:
        return redirect("/")
    
    filename = secure_filename(file.filename)
    is_zip = filename.lower().endswith(".zip")
    
    try:
        target_path = os.path.join(DB_DIR, filename)
        file.save(target_path)

        def task():
            try:
                db_file = None
                if is_zip:
                    ENGINE._update("indexing", "מחלץ מסד נתונים...", 0)
                    with zipfile.ZipFile(target_path, 'r') as z:
                        z.extractall(DB_DIR)
                    
                    # חיפוש קובץ ה-DB בתוך התיקייה שחולצה - נחפש את הכי חדש או לפי השם
                    for root, dirs, files in os.walk(DB_DIR):
                        for f in files:
                            if f.endswith(".db") or f.endswith(".sqlite"):
                                db_file = os.path.join(root, f)
                                break
                        if db_file: break
                else:
                    db_file = target_path

                if db_file:
                    cfg = load_settings()
                    cfg["db_path"] = db_file
                    save_settings(cfg)
                    ENGINE.book_map = get_book_titles(db_file)

                    # טעינת מודל ובניית אינדקס רק אם יש מודל מוגדר
                    model_source = cfg.get("model_source")
                    if model_source:
                        ENGINE.load_resources(db_file, cfg.get("edition", "v3"), 
                                            model_source=model_source, 
                                            zip_path=cfg.get("zip_path", ""))
                        ENGINE.build_index(db_file, int(cfg.get("max_chunks", 100000)))
                    else:
                        ENGINE._update("idle", "מסד הנתונים הועלה. כעת טען מודל כדי לאנדקס.", 100)
                else:
                    ENGINE._update("error", "לא נמצא קובץ DB", 0)
            except Exception as e:
                ENGINE._update("error", f"שגיאה בטעינת DB: {e}", 0)

        threading.Thread(target=task, daemon=True).start()
        
    except Exception as e:
        print(f"Upload failed: {e}")
        
    return redirect("/")

@app.route("/select_local_db")
def select_local_db():
    root = tk.Tk()
    root.withdraw()
    root.attributes("-topmost", True)
    file_path = filedialog.askopenfilename(
        title="בחר קובץ מסד נתונים (seforim.db)",
        filetypes=[("SQLite Database", "*.db *.sqlite"), ("ZIP files", "*.zip"), ("All files", "*.*")]
    )
    root.destroy()
    
    if file_path:
        cfg = load_settings()
        cfg["db_path"] = file_path
        save_settings(cfg)
        ENGINE.last_cfg = cfg
        ENGINE.book_map = get_book_titles(file_path)
        flash(f"נבחר מסד נתונים: {file_path}")
    return redirect("/")

@app.route("/select_local_zip")
def select_local_zip():
    root = tk.Tk()
    root.withdraw()
    root.attributes("-topmost", True)
    file_path = filedialog.askopenfilename(
        title="בחר קובץ מודל (ZIP)",
        filetypes=[("ZIP files", "*.zip"), ("All files", "*.*")]
    )
    root.destroy()
    
    if file_path:
        cfg = load_settings()
        cfg["model_source"] = "zip"
        cfg["zip_path"] = file_path
        save_settings(cfg)
        ENGINE.last_cfg = cfg
        flash(f"נבחר קובץ מודל: {file_path}")
    return redirect("/")

@app.route("/status")
def status_api():
    s = ENGINE.status.copy()
    if ENGINE.built:
        s["count"] = ENGINE.built.count
    return jsonify(s)

# =========================
# MAIN
# =========================
if __name__ == "__main__":
    # טעינה אוטומטית על בסיס settings
    cfg = load_settings()
    dbp = cfg.get("db_path", DEFAULT_DB_PATH)
    ed = cfg.get("edition", "v3")
    mc = int(cfg.get("max_chunks", 100000))
    model_source = cfg.get("model_source", "zip")
    zip_path = cfg.get("zip_path", "")

    ENGINE.last_cfg = {
        "db_path": dbp,
        "edition": ed,
        "max_chunks": mc,
        "model_source": model_source,
        "zip_path": zip_path,
    }

    def boot():
        try:
            ENGINE.load_resources(dbp, ed, model_source=model_source, zip_path=zip_path)
            ENGINE.build_index(dbp, mc)
        except Exception as e:
            ENGINE._update("error", f"שגיאה: {e}", 0)
            print("Boot warning:", e)

    threading.Thread(target=boot, daemon=True).start()

    print("Starting Enhanced Server at http://127.0.0.1:8000")
    app.run(host="127.0.0.1", port=8000, debug=True)