#!/usr/bin/env python3
"""Diagnostic lexical de la couverture de preuves, sans génération ni dépendance."""
import argparse
import copy
import hashlib
import json
import platform
import re
import sqlite3
from datetime import datetime, timezone
from pathlib import Path

# Ces trois valeurs ont été fixées avant la première exécution du jeu.
VARIANTS = (
    {"id": "fixe_48_0", "size_words": 48, "overlap_words": 0},
    {"id": "fixe_96_0", "size_words": 96, "overlap_words": 0},
    {"id": "fixe_48_16", "size_words": 48, "overlap_words": 16},
    {"id": "paragraphes", "size_words": None, "overlap_words": 0},
)
FILTERS = {"country": "FR", "status": "current"}
TOP_K = 3
WORD_BUDGET = 120
# Ce traitement lexical n'effectue ni lemmatisation, ni recherche de synonymes.
STOPWORDS = set("a à au aux avec ce ces cet cette dans de des du d en est et il la le les l ma mon ne pas par pendant plus pour qu que quel quelle quelles quels qui se si son sont sur un une y t on s elle nous vous après avant fait passe".split())


def words(text):
    """Un mot = une suite non vide de caractères séparée par des blancs."""
    return text.split()


def require(condition, message):
    if not condition:
        raise ValueError(message)


def unique_ids(items, kind):
    require(isinstance(items, list) and items, f"{kind}: liste non vide requise")
    ids = [item.get("id") for item in items if isinstance(item, dict)]
    require(len(ids) == len(items), f"{kind}: objet requis")
    require(all(isinstance(i, str) and i for i in ids), f"{kind}: identifiant invalide")
    require(len(set(ids)) == len(ids), f"{kind}: identifiant dupliqué")
    return {item["id"]: item for item in items}


def validate(data):
    require(isinstance(data, dict) and data.get("schema_version") == 1, "schema_version doit valoir 1")
    docs = unique_ids(data.get("documents"), "documents")
    facts = unique_ids(data.get("facts"), "facts")
    unique_ids(data.get("questions"), "questions")
    for doc in docs.values():
        require(isinstance(doc.get("text"), str) and words(doc["text"]), "document vide")
        require(all(isinstance(doc.get(key), str) and doc[key] for key in FILTERS), "métadonnées absentes")
    spans = {}
    for fact in facts.values():
        doc_id = fact.get("document_id")
        require(doc_id in docs, f"{fact['id']}: document inconnu")
        require(isinstance(fact.get("quote"), str) and words(fact["quote"]), "citation vide")
        source, quote = words(docs[doc_id]["text"]), words(fact["quote"])
        starts = [i for i in range(len(source) - len(quote) + 1) if source[i:i + len(quote)] == quote]
        require(len(starts) == 1, f"{fact['id']}: la citation doit apparaître exactement une fois")
        spans[fact["id"]] = {"document_id": doc_id, "start": starts[0], "end": starts[0] + len(quote)}
    for question in data["questions"]:
        require(isinstance(question.get("text"), str) and question["text"].strip(), "question vide")
        require(type(question.get("answerable")) is bool, "answerable doit être booléen")
        refs = question.get("required_facts")
        require(isinstance(refs, list) and all(isinstance(x, str) for x in refs), "required_facts doit être une liste d'identifiants")
        require(len(refs) == len(set(refs)), "faits requis dupliqués")
        require(all(x in facts for x in refs), "fait requis inconnu")
        require(bool(refs) == question["answerable"], "question répondable sans faits ou question absente avec faits")
    return docs, spans


def split_document(doc, variant):
    source = words(doc["text"])
    ranges = []
    if variant["size_words"] is None:
        start = 0
        for paragraph in re.split(r"\n\s*\n", doc["text"].strip()):
            end = start + len(words(paragraph))
            ranges.append((start, end))
            start = end
    else:
        size, overlap = variant["size_words"], variant["overlap_words"]
        require(type(size) is int and type(overlap) is int and size > 0 and 0 <= overlap < size, "taille ou recouvrement invalide")
        for start in range(0, len(source), size - overlap):
            end = min(start + size, len(source))
            ranges.append((start, end))
            if end == len(source):
                break  # pas de dernier passage entièrement inclus dans le précédent
    return [{"id": f"{doc['id']}:{i:03d}", "document_id": doc["id"], "country": doc["country"],
             "status": doc["status"], "start": start, "end": end,
             "text": " ".join(source[start:end])} for i, (start, end) in enumerate(ranges)]


def lexical_query(question):
    terms = list(dict.fromkeys(term.lower() for term in re.findall(r"\w+", question, re.UNICODE)
                               if len(term) > 1 and term.lower() not in STOPWORDS))
    require(bool(terms), "question sans terme de recherche")
    return " OR ".join('"' + term.replace('"', '""') + '"' for term in terms)


def search(connection, query):
    rows = connection.execute(
        "SELECT id, bm25(chunks) AS score FROM chunks WHERE chunks MATCH ? "
        "AND country = ? AND status = ? ORDER BY score ASC, id ASC LIMIT ?",
        (query, FILTERS["country"], FILTERS["status"], TOP_K),
    ).fetchall()
    return [{"id": chunk_id, "bm25": score} for chunk_id, score in rows]


def assemble(hits, chunks, budget=WORD_BUDGET):
    require(type(budget) is int and budget >= 0, "budget invalide")
    selected, remaining = [], budget
    for rank, hit in enumerate(hits, 1):
        require(hit.get("id") in chunks, "passage retrouvé inconnu")
        if not remaining:
            break
        chunk = chunks[hit["id"]]
        take = min(chunk["end"] - chunk["start"], remaining)
        selected.append({"rank": rank, "id": chunk["id"], "document_id": chunk["document_id"],
                         "start": chunk["start"], "end": chunk["start"] + take,
                         "bm25": hit["bm25"], "words": take,
                         "clipped": take < chunk["end"] - chunk["start"],
                         "text": " ".join(words(chunk["text"])[:take])})
        remaining -= take
    return selected


def coverage(required, passages, docs, spans):
    positions = set()
    for passage in passages:
        doc_id, start, end = passage["document_id"], passage["start"], passage["end"]
        require(doc_id in docs, "contexte: document inconnu")
        require(type(start) is int and type(end) is int and 0 <= start <= end <= len(words(docs[doc_id]["text"])), "contexte: intervalle invalide")
        expected = words(docs[doc_id]["text"])[start:end]
        require(words(passage["text"]) == expected, "contexte: texte différent de la source")
        positions.update((doc_id, i) for i in range(start, end))
    details = []
    for fact_id in required:
        require(fact_id in spans, "évaluation: fait inconnu")
        span = spans[fact_id]
        covered = sum((span["document_id"], i) in positions for i in range(span["start"], span["end"]))
        total = span["end"] - span["start"]
        details.append({"id": fact_id, "covered": covered == total, "covered_words": covered, "reference_words": total})
    return details, len(positions)


def run(data):
    docs, spans = validate(data)
    report = {"schema_version": 1, "executed_at": datetime.now(timezone.utc).isoformat(),
              "environment": {"python": platform.python_version(), "sqlite": sqlite3.sqlite_version, "fts5": True},
              "method": {"search": "SQLite FTS5 unicode61 remove_diacritics 2, OR, bm25 asc puis id asc", "filters": FILTERS,
                         "top_k": TOP_K, "word_budget": WORD_BUDGET, "word_definition": "str.split(): séparation par blancs",
                         "assembly": "concaténation selon le rang, dernier passage coupé aux mots, aucun titre préfixé et aucune déduplication",
                         "fact_coverage": "tous les mots d'une citation de référence présents via leurs positions source dans le contexte transmis",
                         "generation_executed": False},
              "fact_spans": spans, "variants": []}
    for variant in VARIANTS:
        chunk_list = [chunk for doc in docs.values() for chunk in split_document(doc, variant)]
        chunks = {chunk["id"]: chunk for chunk in chunk_list}
        with sqlite3.connect(":memory:") as conn:
            try:
                conn.execute("CREATE VIRTUAL TABLE chunks USING fts5(id UNINDEXED, country UNINDEXED, status UNINDEXED, text, tokenize='unicode61 remove_diacritics 2')")
            except sqlite3.OperationalError as error:
                raise RuntimeError("Cette distribution SQLite ne fournit pas FTS5. Aucune substitution silencieuse n'est effectuée.") from error
            conn.executemany("INSERT INTO chunks VALUES (:id, :country, :status, :text)", chunk_list)
            rows = []
            for question in data["questions"]:
                query = lexical_query(question["text"])
                hits = search(conn, query)
                # Trace complète avant budget : le classement reste inchangé.
                hits = [{**hit, "rank": rank, "document_id": chunks[hit["id"]]["document_id"],
                         "start": chunks[hit["id"]]["start"], "end": chunks[hit["id"]]["end"],
                         "words": chunks[hit["id"]]["end"] - chunks[hit["id"]]["start"],
                         "text": chunks[hit["id"]]["text"]}
                        for rank, hit in enumerate(hits, 1)]
                selected = assemble(hits, chunks)
                details, unique_words = coverage(question["required_facts"], selected, docs, spans)
                first, _ = coverage(question["required_facts"], selected[:1], docs, spans)
                sent = sum(p["words"] for p in selected)
                rows.append({"question_id": question["id"], "question": question["text"], "query": query,
                             "answerable": question["answerable"], "facts": details,
                             "covered_facts": sum(f["covered"] for f in details), "required_facts": len(details),
                             "all_required_covered": all(f["covered"] for f in details) if question["answerable"] else None,
                             "first_passage_covered_fact_ids": [f["id"] for f in first if f["covered"]],
                             "words_sent": sent, "unique_source_words": unique_words,
                             "redundant_words": sent - unique_words, "hits_before_budget": hits, "context": selected})
        answerable = [row for row in rows if row["answerable"]]
        report["variants"].append({"configuration": variant, "chunks": len(chunk_list),
            "indexed_words": sum(len(words(c["text"])) for c in chunk_list),
            "summary": {"answerable_questions": len(answerable),
                        "complete_contexts": sum(row["all_required_covered"] for row in answerable),
                        "covered_facts": sum(row["covered_facts"] for row in answerable),
                        "required_facts": sum(row["required_facts"] for row in answerable),
                        "words_sent_answerable": sum(row["words_sent"] for row in answerable),
                        "redundant_words_answerable": sum(row["redundant_words"] for row in answerable)},
            "results": rows})
    return report


def self_test(data):
    docs, spans = validate(data)
    checks = []
    def rejects(name, action):
        try:
            action()
        except ValueError:
            checks.append({"test": name, "passed": True})
        else:
            raise AssertionError(name)
    for name, mutate in (
        ("identifiant de document dupliqué", lambda d: d["documents"].append(copy.deepcopy(d["documents"][0]))),
        ("document de fait inconnu", lambda d: d["facts"][0].update(document_id="inconnu")),
        ("citation absente", lambda d: d["facts"][0].update(quote="Cette citation ne figure pas dans le corpus.")),
        ("fait requis inconnu", lambda d: d["questions"][0].update(required_facts=["inconnu"])),
        ("faits requis dupliqués", lambda d: d["questions"][0].update(required_facts=["hotel_normal", "hotel_normal"])),
        ("répondable sans référence", lambda d: d["questions"][0].update(required_facts=[])),
        ("absence avec référence", lambda d: d["questions"][-1].update(required_facts=["hotel_normal"])),
        ("document vide", lambda d: d["documents"][0].update(text="")),
    ):
        malformed = copy.deepcopy(data)
        mutate(malformed)
        rejects(name, lambda: validate(malformed))
    fact = spans["hotel_normal"]
    only_rule = [{**fact, "text": " ".join(words(docs[fact["document_id"]]["text"])[fact["start"]:fact["end"]])}]
    details, _ = coverage(["hotel_normal", "hotel_exception"], only_rule, docs, spans)
    assert [f["covered"] for f in details] == [True, False]
    checks.append({"test": "règle présente et exception absente donnent un fait couvert sur deux", "passed": True})
    truncated = copy.deepcopy(only_rule)
    truncated[0]["end"] -= 1
    truncated[0]["text"] = " ".join(words(truncated[0]["text"])[:-1])
    assert not coverage(["hotel_normal"], truncated, docs, spans)[0][0]["covered"]
    checks.append({"test": "citation tronquée non couverte", "passed": True})
    wrong = copy.deepcopy(only_rule)
    wrong[0]["document_id"] = "inconnu"
    rejects("document de contexte inconnu", lambda: coverage(["hotel_normal"], wrong, docs, spans))
    rejects("passage retrouvé inconnu", lambda: assemble([{"id": "inconnu", "bm25": 0}], {}))
    rejects("overlap égal à la taille", lambda: split_document(data["documents"][0], {"size_words": 48, "overlap_words": 48}))
    return {"passed": True, "checks": len(checks), "details": checks}


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--data", type=Path, default=Path(__file__).with_name("cas_fr.json"))
    parser.add_argument("--output", type=Path, default=Path(__file__).with_name("resultats.json"))
    parser.add_argument("--self-test", action="store_true")
    args = parser.parse_args()
    data = json.loads(args.data.read_text(encoding="utf-8"))
    if args.self_test:
        print(json.dumps(self_test(data), ensure_ascii=False, indent=2))
        return
    report = run(data)
    report["input_sha256"] = hashlib.sha256(args.data.read_bytes()).hexdigest()
    report["script_sha256"] = hashlib.sha256(Path(__file__).read_bytes()).hexdigest()
    args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    print(f"Python {report['environment']['python']}, SQLite {report['environment']['sqlite']}, FTS5 disponible")
    for variant in report["variants"]:
        s = variant["summary"]
        print(f"{variant['configuration']['id']}: {s['complete_contexts']}/{s['answerable_questions']} contextes complets ; {s['covered_facts']}/{s['required_facts']} faits ; {s['words_sent_answerable']} mots transmis ; {s['redundant_words_answerable']} mots redondants")
    print(f"Rapport : {args.output}")


if __name__ == "__main__":
    main()